From 0e063316e9e8607772be68fc71accce3eb68cf5f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:01:08 +0100 Subject: [PATCH 01/83] refactor(ports): split process-runner.ts into parts --- src/ports/node-process-runner.ts | 581 ++++++++++++ src/ports/process-line-buffer.ts | 112 +++ src/ports/process-runner-types.ts | 159 ++++ src/ports/process-runner.ts | 1266 +------------------------- src/ports/process-tree-kill.ts | 91 ++ src/ports/scripted-process-runner.ts | 333 +++++++ 6 files changed, 1297 insertions(+), 1245 deletions(-) create mode 100644 src/ports/node-process-runner.ts create mode 100644 src/ports/process-line-buffer.ts create mode 100644 src/ports/process-runner-types.ts create mode 100644 src/ports/process-tree-kill.ts create mode 100644 src/ports/scripted-process-runner.ts diff --git a/src/ports/node-process-runner.ts b/src/ports/node-process-runner.ts new file mode 100644 index 000000000..a7e0a4300 --- /dev/null +++ b/src/ports/node-process-runner.ts @@ -0,0 +1,581 @@ +import { spawn as spawnChildProcess, type ChildProcess } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; +import type { Writable } from "node:stream"; + +import { LineBuffer, captureLines } from "./process-line-buffer.js"; +import { isRunning, killProcessTree, runLimits } from "./process-tree-kill.js"; +import { childLaunch } from "./windows-command.js"; +import { + InputStreamFailedError, + ProcessSpawnError, + type LineEnd, + type NodeProcessRunnerOptions, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ProcessStreamOptions, + type StreamingProcessHandle, + type StreamingProcessResult, +} from "./process-runner-types.js"; + +// why: `close` normally follows `exit` within the same tick, once the child's own stdio +// pipes report EOF. But a child that forks a grandchild before it dies leaves that +// grandchild holding the inherited write end of our pipe open -- it was handed the +// same descriptors, and nothing takes them back -- so `close` never fires even +// though the process we actually care about is long gone. (`detached: true` is not +// the cause; it is why the grandchild is reachable to kill, since it puts the whole +// tree in one process group.) After `exit`, `wait()` therefore stops waiting for +// `close` once the pipes have gone quiet for this long. +// +// Quiet, not merely elapsed: settling on a bare timer would truncate the output of a +// process that exited while its pipe was still draining (a large `simctl list --json` +// under a loaded event loop), and a truncated capture surfaces as a parse error far +// from its cause. So the window restarts whenever more output arrives, which +// distinguishes "still draining" from "held open by something that has nothing to +// say" -- the only case this needs to escape. +const EXIT_TO_CLOSE_GRACE_MS = 1_000; + +// why: A grandchild that inherits the pipe *and* chatters on it would otherwise restart the +// grace window forever, so the deferral is capped. Reaching this cap means the output +// may be incomplete; nothing else is waiting on it by then. +const EXIT_TO_CLOSE_MAX_DEFERRAL_MS = 5_000; + +// why: A hard bound on how long `run()` waits after a timeout-triggered SIGTERM before escalating to +// SIGKILL. A child that ignores SIGTERM (or is itself stuck in an uninterruptible wait) would +// otherwise hold the caller's `await process.wait()` open forever -- exactly the unbounded wait +// this constant exists to cap. Fixed rather than derived from `timeoutMs`: it is a +// termination-cleanup budget, not a scaled fraction of the operation's own timeout. +const SIGTERM_TO_SIGKILL_GRACE_MS = 10_000; + +export class NodeProcessRunner implements ProcessRunner { + readonly sigkillGraceMs: number; + readonly exitDeferralCapMs: number; + readonly exitGraceMs: number; + + constructor(options: NodeProcessRunnerOptions = {}) { + this.sigkillGraceMs = options.sigkillGraceMs ?? SIGTERM_TO_SIGKILL_GRACE_MS; + this.exitDeferralCapMs = options.exitDeferralCapMs ?? EXIT_TO_CLOSE_MAX_DEFERRAL_MS; + this.exitGraceMs = options.exitGraceMs ?? EXIT_TO_CLOSE_GRACE_MS; + } + + async run( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): Promise { + const process = this.#launch(command, args, options); + let killTimeout: NodeJS.Timeout | undefined; + const timeout = + options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + try { + process.kill("SIGTERM", "timeout"); + } catch { + // why: The child may have exited between the timer firing and the kill. + } + // why: SIGTERM is a request, not a guarantee -- a child that ignores it (or is itself + // hung) must not be able to keep this `run()` call waiting indefinitely. Escalate to + // SIGKILL if it hasn't exited within the grace period; cleared below like `timeout` + // itself the moment `process.wait()` actually settles, so a child that does exit + // promptly after SIGTERM never sees the follow-up signal. + killTimeout = setTimeout(() => { + try { + process.kill("SIGKILL"); + } catch { + // why: The child may have exited between the timer firing and the kill. + } + }, this.sigkillGraceMs); + }, options.timeoutMs); + + try { + return await process.wait(); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + if (killTimeout !== undefined) { + clearTimeout(killTimeout); + } + } + } + + spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { + return this.#launch(command, args, options); + } + + #launch(command: string, args: readonly string[], options: ProcessRunOptions): NodeProcessHandle { + const launch = childLaunch(command, args); + const logFd = + typeof options.stdio === "object" ? openSync(options.stdio.file, "a", 0o600) : undefined; + const child = spawnChildProcess(launch.command, launch.args, { + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.env === undefined ? {} : { env: options.env }), + detached: process.platform !== "win32", + stdio: childStdio(options.stdio, logFd), + // why: the daemon has no console, so on Windows a console program it starts (adb, the + // emulator, node) would otherwise get a new, visible one. + windowsHide: true, + windowsVerbatimArguments: launch.windowsVerbatimArguments, + }); + // why: the child holds its own copy of the descriptor; keeping ours would leak one per start. + if (logFd !== undefined) closeSync(logFd); + + if (child.pid === undefined) { + // why: The spawn failed, and Node will say why by emitting `error` on the next tick. An + // `error` with no listener is a hard, uncatchable process exit, so the listener is + // attached *before* this function leaves -- throwing first would take the daemon + // down over an unreadable `adb`. The reason is lost with it (it does not exist yet), + // which is the price of turning an asynchronous crash into a catchable failure. + child.once("error", () => undefined); + throw new ProcessSpawnError(command, args); + } + + if (options.input !== undefined) { + child.stdin?.end(options.input); + } + + return new NodeProcessHandle( + child, + child.pid, + options.lineEnd ?? "newline", + this.exitDeferralCapMs, + this.exitGraceMs, + launch.windowsVerbatimArguments, + runLimits(options), + ); + } + + spawnStreaming( + command: string, + args: readonly string[], + options: ProcessStreamOptions, + ): StreamingProcessHandle { + const launch = childLaunch(command, args); + const child = spawnChildProcess(launch.command, launch.args, { + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.env === undefined ? {} : { env: options.env }), + detached: process.platform !== "win32", + stdio: "pipe", + // why: as in `spawn`: no visible console window for a child of a daemon that has none. + windowsHide: true, + windowsVerbatimArguments: launch.windowsVerbatimArguments, + }); + + if (child.pid === undefined) { + // why: Same reasoning as `spawn` above: attach a listener before throwing, or Node's + // asynchronous `error` takes the daemon down instead of rejecting one operation. + child.once("error", () => undefined); + throw new ProcessSpawnError(command, args); + } + + const handle = new NodeStreamingProcessHandle( + child, + child.pid, + options.onChunk, + this.exitDeferralCapMs, + this.exitGraceMs, + launch.windowsVerbatimArguments, + ); + if (options.input !== undefined) { + // why: One shot, then closed (ADR 0005 §19c): a command reading stdin sees exactly this and + // then EOF. `end` is safe even if the child already exited -- the EPIPE that produces is + // swallowed below rather than left to surface as an unhandled `error` on the stream. + child.stdin?.on("error", () => undefined); + child.stdin?.end(options.input); + } else if (options.inputStream !== undefined && child.stdin) { + const inputFailed = pumpInput(options.inputStream, child.stdin); + // why: a caller that never looks at `inputFailed` must not turn a read error into an unhandled rejection that ends the daemon. + inputFailed.catch(() => undefined); + handle.inputFailed = inputFailed; + } + return handle; + } +} + +/** + * why: writes the stream to the child's stdin chunk by chunk, waiting for `drain` so a slow child holds the read back (nothing is buffered, ADR 0005 §19e), then closes stdin. A child that closes its stdin early (EPIPE) cancels the read quietly; only a read error rejects the returned promise, which never resolves otherwise. + */ +function pumpInput(source: ReadableStream, stdin: Writable): Promise { + return new Promise((_resolve, reject) => { + const reader = source.getReader(); + let written = 0; + let closed = false; + const onClosed = (): void => { + closed = true; + void reader.cancel().catch(() => undefined); + }; + stdin.on("error", onClosed); + stdin.on("close", onClosed); + void (async () => { + for (;;) { + let result: ReadableStreamReadResult; + try { + result = await reader.read(); + } catch (error) { + if (!closed) reject(new InputStreamFailedError(written, error)); + return; + } + if (closed) return; + if (result.done) { + stdin.end(); + return; + } + written += result.value.byteLength; + if (!stdin.write(result.value) && !closed) { + await new Promise((resolve) => { + const done = (): void => { + stdin.off("drain", done); + stdin.off("close", done); + resolve(); + }; + stdin.once("drain", done); + stdin.once("close", done); + }); + } + } + })(); + }); +} + +/** + * why: Thrown by `NodeStreamingProcessHandle#wait()` when the child exited but a chunk's delivery + * -- the promise `onChunk` returned, which this handle was waiting on before resuming the + * pipe for backpressure (ADR 0005 §19e) -- was still outstanding `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` + * later. A paused readable never reaches `close` and stops emitting `data`, so nothing here can + * *prove* the remaining output was delivered; resolving `{ exitCode }` as if it had been would + * be exactly the silent truncation streaming without buffering exists to avoid. The caller sees + * this as `device.exec`'s terminal error, the same shape a `PASSTHROUGH_REFUSED` or + * `EXEC_TIMEOUT` already is on both transports -- not a clean exit that quietly drops a tail. + */ +export class ExecOutputDeliveryStalledError extends Error { + constructor(deferralCapMs: number = EXIT_TO_CLOSE_MAX_DEFERRAL_MS) { + super( + "The command exited, but delivering one of its output chunks was still unresolved " + + `${String(deferralCapMs)}ms later; treating this as a clean exit could ` + + "have reported an exit code while silently dropping the tail of the command's output.", + ); + this.name = "ExecOutputDeliveryStalledError"; + } +} + +/** + * why: `spawnStreaming`'s handle. Holds no output: every chunk is handed to `onChunk` the moment + * the stream emits it and is then forgotten, which is what lets `device.exec` stream + * arbitrarily large output through a daemon that never grows for it. + * + * Settling repeats `NodeProcessHandle`'s `exit`-then-quiet-window dance for the same reason + * (see `EXIT_TO_CLOSE_GRACE_MS`): a grandchild holding the inherited pipe open (`adb + * start-server` is exactly this) means `close` may never fire even though the process the + * caller asked about is gone. + * + * The quiet window that dance relies on only means anything when the pipe is idle because the + * child has nothing left to write. `#forward` below pauses the readable while a chunk's + * delivery to the consumer is outstanding (the backpressure ADR 0005 §19e asks for): a paused + * readable emits no `data`, so `#chunkCount` stops moving for a reason that has nothing to do + * with the child being gone. Settling under that -- as this class once did -- resolves `wait()` + * while a chunk still sits undelivered in the paused pipe, and the caller returns `{ exitCode }` + * having silently dropped it. + * + * `close` itself is not the all-clear it looks like, either: verified against real Node (a + * child that writes two small chunks in quick succession and calls `process.exit()`), the + * stdio pipe's `close` can fire in the very same tick as a `data` event delivering a chunk + * this handle just paused on -- Node drains whatever the OS pipe was already holding as part + * of noticing EOF, pause or no pause. So both `close` and the exit grace window fail closed + * through the same `#settleWhenDrained`: `#pendingDeliveries` is this handle's own state (it is + * the one that called `pause()`), checked directly rather than inferred from the chunk counter + * the grandchild case already overloads, and only `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` with a + * delivery still outstanding at that deadline ends the wait -- as a rejection + * (`ExecOutputDeliveryStalledError`), not a settle, because at that point something really may + * have been dropped and pretending otherwise is the one thing this class must not do. A chatty + * grandchild with **no** pending delivery hitting the same deadline still settles as before: + * nothing of the command's own output is waiting on anyone in that case, only unrelated noise + * this handle was never going to forward to a live consumer anyway (see `#execDevice` in + * `daemon/server.ts`, torn down by then). + */ +class NodeStreamingProcessHandle implements StreamingProcessHandle { + readonly pid: number; + inputFailed?: Promise; + readonly #result: Promise; + #chunkCount = 0; + #pendingDeliveries = 0; + /** why: Per-stream half of `#pendingDeliveries` -- the total is what the exit-settle path needs + * ("is anything at all still outstanding"), this is what `#forward`'s own `resume()` needs + * ("is *this* stream's own last outstanding delivery the one that just settled"). A stream + * can hold more than one outstanding delivery at once (a paused readable still drains + * whatever it already had at EOF -- see the class doc), so resuming on any settle rather + * than this stream's own count reaching zero would resume it while another delivery on the + * same stream is still unaccounted for. */ + #pendingByStream: Record<"stdout" | "stderr", number> = { stderr: 0, stdout: 0 }; + #deliveryWaiters: Array<() => void> = []; + #killedWith: NodeJS.Signals | undefined; + + constructor( + private readonly child: ChildProcess, + pid: number, + onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, + private readonly exitDeferralCapMs: number, + private readonly exitGraceMs: number, + private readonly wrapped: boolean, + ) { + this.pid = pid; + this.#forward(child.stdout, "stdout", onChunk); + this.#forward(child.stderr, "stderr", onChunk); + this.#result = new Promise((resolve, reject) => { + child.once("error", reject); + + let settled = false; + const settle = (code: number | null, signal: NodeJS.Signals | null): void => { + if (settled) return; + settled = true; + const killedWith = this.#killedWith; + resolve( + wrapped && killedWith !== undefined + ? { code: null, signal: killedWith } + : { code, signal }, + ); + }; + const fail = (error: Error): void => { + if (settled) return; + settled = true; + reject(error); + }; + // why: Neither `close` nor the exit grace window may resolve this promise while a chunk's + // delivery is still outstanding -- both funnel through here instead of settling + // directly, each with its own budget for how long it will wait on a delivery that + // never finishes before giving up and failing loudly instead. Re-checked against a + // timer at the deadline, not only woken by a delivery settling: a delivery that never + // settles at all -- the case this exists for -- would otherwise leave nothing to wake + // this back up and notice the deadline has passed. + const settleWhenDrained = (deadline: number, onDrained: () => void): void => { + if (settled) return; + if (this.#pendingDeliveries === 0) { + onDrained(); + return; + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + fail(new ExecOutputDeliveryStalledError(this.exitDeferralCapMs)); + return; + } + let deadlineTimer: NodeJS.Timeout | undefined; + const timedOut = new Promise((resolve) => { + deadlineTimer = setTimeout(resolve, remainingMs); + deadlineTimer.unref(); + }); + void Promise.race([this.#nextDeliverySettled(), timedOut]).then(() => { + if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); + settleWhenDrained(deadline, onDrained); + }); + }; + + child.once("close", (code, signal) => { + settleWhenDrained(Date.now() + this.exitDeferralCapMs, () => settle(code, signal)); + }); + child.once("exit", (code, signal) => { + const deadline = Date.now() + this.exitDeferralCapMs; + const armGrace = (chunksAtArm: number): void => { + const timer = setTimeout(() => { + if (settled) return; + // why: A pending delivery always wins over "chunk count still changing": it is this + // handle's own state, not an inference, and it is checked first so a chattering + // stream with one stuck delivery does not keep re-arming past it forever. + if ( + this.#pendingDeliveries === 0 && + this.#chunkCount !== chunksAtArm && + Date.now() < deadline + ) { + armGrace(this.#chunkCount); + return; + } + settleWhenDrained(deadline, () => settle(code, signal)); + }, this.exitGraceMs); + timer.unref(); + }; + armGrace(this.#chunkCount); + }); + }); + // why: A late `error` on a handle whose `wait()` nobody kept (a caller that killed and walked + // away) must not become an unhandled rejection and end the daemon; a caller that does + // await `wait()` still sees the rejection. + this.#result.catch(() => undefined); + } + + kill(signal: NodeJS.Signals = "SIGTERM"): void { + if (isRunning(this.child)) this.#killedWith ??= signal; + killProcessTree(this.child, this.pid, signal, this.wrapped); + } + + wait(): Promise { + return this.#result; + } + + #nextDeliverySettled(): Promise { + return new Promise((resolve) => { + this.#deliveryWaiters.push(resolve); + }); + } + + #forward( + stream: NodeJS.ReadableStream | null, + name: "stdout" | "stderr", + onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, + ): void { + if (stream === null) return; + stream.setEncoding("utf8"); + stream.on("data", (chunk: string) => { + this.#chunkCount += 1; + // why: `onChunk` is typed `void | Promise`, but nothing enforces that a transport's + // implementation never throws synchronously (a bug in a socket write, say). Called bare + // inside a `data` listener, that throw would become an uncaught exception and take the + // whole daemon down over one bad chunk delivery -- caught here and treated exactly like + // a delivery whose returned promise rejected: a failed delivery, not a crash. + let delivered: void | Promise; + try { + delivered = onChunk(name, chunk); + } catch (error) { + delivered = Promise.reject(error instanceof Error ? error : new Error(String(error))); + } + if (delivered === undefined) return; + // why: The consumer is not ready for more yet. Pausing the readable stops this child at the + // pipe -- it fills the OS buffer and then blocks in its own `write` -- which is the only + // place backpressure can be applied without holding the bytes somewhere. Resumed on + // either outcome: a failed delivery (a dead socket) must not wedge the process, and the + // exec timeout still bounds a child nobody drains. Tracked in `#pendingDeliveries` so the + // exit settle path can tell "paused waiting on the consumer" apart from "child is gone", + // and separately in `#pendingByStream` so `resume()` fires only once *this* stream's own + // outstanding count reaches zero -- not on any delivery settling, which would resume a + // stream while one of its own other deliveries (two chunks landing before either + // resolves, e.g. at EOF) is still unaccounted for. + this.#pendingDeliveries += 1; + this.#pendingByStream[name] += 1; + stream.pause(); + const settleDelivery = (): void => { + this.#pendingDeliveries -= 1; + this.#pendingByStream[name] -= 1; + if (this.#pendingByStream[name] === 0) stream.resume(); + const waiters = this.#deliveryWaiters; + this.#deliveryWaiters = []; + for (const waiter of waiters) waiter(); + }; + void Promise.resolve(delivered).then(settleDelivery, settleDelivery); + }); + } +} + +class NodeProcessHandle implements ProcessHandle { + readonly pid: number; + readonly stdout = new LineBuffer(); + readonly stderr = new LineBuffer(); + readonly #stdoutChunks: string[] = []; + readonly #stderrChunks: string[] = []; + readonly #result: Promise; + #killedWith: NodeJS.Signals | undefined; + #endedBy: ProcessResult["endedBy"]; + + constructor( + private readonly child: ChildProcess, + pid: number, + lineEnd: LineEnd, + private readonly exitDeferralCapMs: number, + private readonly exitGraceMs: number, + private readonly wrapped: boolean, + limits: Pick = {}, + ) { + this.pid = pid; + captureLines(child.stdout, this.stdout, this.#stdoutChunks, lineEnd); + captureLines(child.stderr, this.stderr, this.#stderrChunks, lineEnd); + this.#watchLimits(limits); + this.#result = new Promise((resolve, reject) => { + child.once("error", reject); + + let settled = false; + const settle = (code: number | null, signal: NodeJS.Signals | null): void => { + if (settled) { + return; + } + settled = true; + resolve({ + code: wrapped && this.#killedWith !== undefined ? null : code, + stderr: this.#stderrChunks.join(""), + stdout: this.#stdoutChunks.join(""), + ...(this.#endedBy === undefined ? {} : { endedBy: this.#endedBy }), + ...(signal === null ? {} : { signal }), + }); + }; + + child.once("close", (code, signal) => { + settle(code, signal); + }); + child.once("exit", (code, signal) => { + const deadline = Date.now() + this.exitDeferralCapMs; + const capturedSoFar = (): number => this.#stdoutChunks.length + this.#stderrChunks.length; + const armGrace = (chunksAtArm: number): void => { + const timer = setTimeout(() => { + if (capturedSoFar() !== chunksAtArm && Date.now() < deadline) { + armGrace(capturedSoFar()); + return; + } + settle(code, signal); + }, this.exitGraceMs); + // why: A pending grace timer must never keep the Node process alive on its own. + timer.unref(); + }; + armGrace(capturedSoFar()); + }); + }); + // why: A child nobody waits on (the supervised adb server) would otherwise turn a late + // `error` into an unhandled rejection, which ends the daemon. Marking the promise + // handled here changes nothing for a caller that does await `wait()`. + this.#result.catch(() => undefined); + } + + /** + * why: the output bound counts bytes as they arrive, and an aborted signal ends the child the + * same way; both kill with SIGKILL, since the caller has no use for a graceful end of a child + * that is printing without end or whose answer nobody wants. + */ + #watchLimits({ + maxStdoutBytes, + signal, + }: Pick): void { + if (maxStdoutBytes !== undefined) { + let received = 0; + this.child.stdout?.on("data", (chunk: string | Buffer) => { + received += Buffer.byteLength(chunk); + if (received > maxStdoutBytes) this.kill("SIGKILL", "output-limit"); + }); + } + if (signal !== undefined) { + const abort = (): void => this.kill("SIGKILL", "aborted"); + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + this.child.once("close", () => signal.removeEventListener("abort", abort)); + } + } + + kill(signal: NodeJS.Signals = "SIGTERM", endedBy?: ProcessResult["endedBy"]): void { + if (isRunning(this.child)) { + this.#killedWith ??= signal; + this.#endedBy ??= endedBy; + } + killProcessTree(this.child, this.pid, signal, this.wrapped); + } + + unref(): void { + this.child.unref(); + } + + wait(): Promise { + return this.#result; + } +} + +function childStdio( + stdio: ProcessRunOptions["stdio"], + logFd: number | undefined, +): "pipe" | "ignore" | ["ignore", number, number] { + if (logFd !== undefined) return ["ignore", logFd, logFd]; + return stdio === undefined || typeof stdio === "object" ? "pipe" : stdio; +} diff --git a/src/ports/process-line-buffer.ts b/src/ports/process-line-buffer.ts new file mode 100644 index 000000000..a179f0f3d --- /dev/null +++ b/src/ports/process-line-buffer.ts @@ -0,0 +1,112 @@ +import type { LineEnd } from "./process-runner-types.js"; + +export class LineBuffer implements AsyncIterable { + readonly #lines: string[] = []; + readonly #waiters: Array<(line: string | undefined) => void> = []; + #closed = false; + + push(line: string): void { + const waiter = this.#waiters.shift(); + if (waiter !== undefined) { + waiter(line); + return; + } + + this.#lines.push(line); + } + + close(): void { + this.#closed = true; + + while (this.#waiters.length > 0) { + this.#waiters.shift()?.(undefined); + } + } + + async *[Symbol.asyncIterator](): AsyncGenerator { + while (true) { + const line = await this.#next(); + if (line === undefined) { + return; + } + + yield line; + } + } + + #next(): Promise { + const line = this.#lines.shift(); + if (line !== undefined) { + return Promise.resolve(line); + } + + if (this.#closed) { + return Promise.resolve(undefined); + } + + return new Promise((resolve) => { + this.#waiters.push(resolve); + }); + } +} + +/** + * why: The one place process output is cut into lines, for the real handle and the scripted one. + * `\r\n` is always a single line end. Under `"carriage-return-too"` a bare `\r` ends a line as + * well, and empty lines are dropped -- which is also what keeps a `\r\n` pair split across two + * chunks one line end rather than a line plus an empty one. + */ +export class LineSplitter { + #remainder = ""; + + constructor(private readonly lineEnd: LineEnd) {} + + feed(chunk: string): string[] { + const parts = `${this.#remainder}${chunk}`.split( + this.lineEnd === "carriage-return-too" ? /\r\n|\r|\n/ : /\r?\n/, + ); + this.#remainder = parts.pop() ?? ""; + return this.#kept(parts); + } + + end(): string[] { + const tail = this.#remainder; + this.#remainder = ""; + return tail === "" ? [] : this.#kept([tail]); + } + + #kept(lines: string[]): string[] { + return this.lineEnd === "carriage-return-too" ? lines.filter((line) => line !== "") : lines; + } +} + +export function captureLines( + stream: NodeJS.ReadableStream | null, + destination: LineBuffer, + chunks: string[], + lineEnd: LineEnd, +): void { + // why: No stream means the child was spawned with its output detached (`stdio: "ignore"`, or a + // log file taking both descriptors). Closing the buffer + // straight away is what keeps `for await (const line of handle.stdout)` a loop that + // ends immediately rather than one that never yields and never returns. + if (stream === null) { + destination.close(); + return; + } + + const splitter = new LineSplitter(lineEnd); + stream.setEncoding("utf8"); + stream.on("data", (chunk: string) => { + chunks.push(chunk); + for (const line of splitter.feed(chunk)) { + destination.push(line); + } + }); + stream.on("end", () => { + for (const line of splitter.end()) { + destination.push(line); + } + destination.close(); + }); +} diff --git a/src/ports/process-runner-types.ts b/src/ports/process-runner-types.ts new file mode 100644 index 000000000..4d8440d1b --- /dev/null +++ b/src/ports/process-runner-types.ts @@ -0,0 +1,159 @@ +export interface ProcessRunOptions { + readonly timeoutMs?: number; + readonly env?: NodeJS.ProcessEnv; + readonly cwd?: string; + /** + * why: `"ignore"` detaches the child's output entirely: nothing is captured and nothing is + * iterable. A long-lived supervised process would otherwise accumulate every line it ever + * wrote in this handle's buffers for as long as it runs. `{ file }` appends the child's + * stdout and stderr to that file (created 0600 when missing, whose directory must exist), + * with the same effect on the handle: nothing is captured. + */ + readonly stdio?: "pipe" | "ignore" | { readonly file: string }; + readonly input?: string; + /** + * why: ends the child when the caller no longer wants its answer: aborting kills the child's + * whole process group, and `run` answers `endedBy: "aborted"`. + */ + readonly signal?: AbortSignal; + /** + * why: a bound on a child's stdout, counted in bytes as it arrives: past it the runner kills the + * child and answers `endedBy: "output-limit"`. Output is kept in memory until the child ends, so + * a caller that reads an answer from an untrusted program sets it. + */ + readonly maxStdoutBytes?: number; + /** + * why: Where `spawn`'s `stdout` and `stderr` iterables end a line. `"newline"` (the default) ends + * one only at `\n`. `"carriage-return-too"` also ends one at a bare `\r`, and yields no empty + * lines: an installer such as `sdkmanager` redraws its progress bar in place with `\r`, so + * without this every update waits in one unfinished line until the process prints its final + * newline. The captured `ProcessResult` text is the same either way. + */ + readonly lineEnd?: LineEnd; +} + +export type LineEnd = "newline" | "carriage-return-too"; + +export interface ProcessResult { + readonly code: number | null; + readonly stdout: string; + readonly stderr: string; + /** why: set when the runner itself killed the child, and why: its `timeoutMs`, `maxStdoutBytes` or `signal`. */ + readonly endedBy?: "timeout" | "output-limit" | "aborted"; + /** why: the name of the signal that ended the child, whoever sent it; absent when it exited by itself. */ + readonly signal?: NodeJS.Signals; +} + +/** + * why: ADR 0005 §19a's spawn: a child whose output is handed to the caller chunk by chunk as it + * arrives and is never accumulated anywhere in this process. + * + * Deliberately a second method rather than an option on `spawn`: `ProcessHandle` exposes its + * output as line-delimited `AsyncIterable`s *and* keeps every chunk so `wait()` can hand back + * a complete `stdout`/`stderr` -- exactly right for the driver calls that parse a tool's whole + * answer, and exactly wrong for `device.exec`, which must forward bytes as they appear and + * hold none of them (§19e: "Output is streamed, never buffered, so there is no size cap"). + * Bolting a "don't buffer" flag onto `spawn` would have left `ProcessResult`'s `stdout`/`stderr` + * silently empty for that mode; a separate return type says so in the type system instead. + * + * Chunks are whatever the child wrote, decoded UTF-8, unsplit: no line assembly, so a caller + * that wants lines does that itself. + */ +export interface ProcessStreamOptions { + readonly env?: NodeJS.ProcessEnv; + readonly cwd?: string; + readonly input?: string; + /** + * why: Piped to the child's stdin and then closed, for input too big to hold as a string. + * Given together with `input`, `inputStream` is ignored. + */ + readonly inputStream?: ReadableStream; + /** + * why: Called with each chunk as it arrives. **May return a promise**, and if it does, this + * runner stops reading that stream until it resolves -- which is how backpressure reaches + * the child: a consumer that cannot place a chunk yet (an SSE client that is not reading, a + * socket that has not drained) slows the command down instead of accumulating its output + * somewhere. ADR 0005 §19e says output is streamed and never buffered; that is only true + * end to end if the slow end can push back, so the callback's promise is the seam that + * makes it true rather than a claim about the fast path only. + */ + readonly onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise; +} + +export interface StreamingProcessResult { + readonly code: number | null; + readonly signal: NodeJS.Signals | null; +} + +export interface StreamingProcessHandle { + readonly pid: number; + /** + * why: Only on a child spawned with `inputStream`: rejects when that stream fails to be read + * to its end, with the bytes written to the child by then. A child that closes its stdin + * early is not a failure and never rejects it. + */ + inputFailed?: Promise; + kill(signal?: NodeJS.Signals): void; + wait(): Promise; +} + +export interface ProcessHandle { + readonly pid: number; + readonly stdout: AsyncIterable; + readonly stderr: AsyncIterable; + kill(signal?: NodeJS.Signals): void; + /** + * why: Stops this child from keeping the parent's event loop alive. Only for a process that + * is meant to outlive the daemon and is reaped by pid instead of by exit (Simlock's adb + * server): everything else is awaited through `wait()` and must keep the loop open. + */ + unref(): void; + wait(): Promise; +} + +/** + * why: A child that could not be started at all -- a binary that exists but is not executable, + * `ETXTBSY` while it is being rewritten, `EAGAIN` under memory pressure. Node reports this + * asynchronously, so it is raised here as a synchronous throw (and therefore a rejected + * promise out of `run`) rather than left to surface as an unhandled `error` event. + */ +export class ProcessSpawnError extends Error { + constructor( + readonly command: string, + readonly args: readonly string[], + ) { + super(`Failed to spawn ${command} ${args.join(" ")}`); + this.name = "ProcessSpawnError"; + } +} + +export interface ProcessRunner { + run( + command: string, + args: readonly string[], + options?: ProcessRunOptions, + ): Promise; + spawn(command: string, args: readonly string[], options?: ProcessRunOptions): ProcessHandle; + spawnStreaming( + command: string, + args: readonly string[], + options: ProcessStreamOptions, + ): StreamingProcessHandle; +} + +export interface NodeProcessRunnerOptions { + readonly sigkillGraceMs?: number; + readonly exitDeferralCapMs?: number; + readonly exitGraceMs?: number; +} + +/** why: the input stream of a streamed child failed to be read to its end; `bytesWritten` is what the child's stdin had been handed by then. */ +export class InputStreamFailedError extends Error { + constructor( + readonly bytesWritten: number, + cause: unknown, + ) { + super(`Reading the input stream failed after ${String(bytesWritten)} bytes`, { cause }); + this.name = "InputStreamFailedError"; + } +} diff --git a/src/ports/process-runner.ts b/src/ports/process-runner.ts index 48173d6cd..d9020632a 100644 --- a/src/ports/process-runner.ts +++ b/src/ports/process-runner.ts @@ -1,137 +1,26 @@ -import { spawn as spawnChildProcess, type ChildProcess } from "node:child_process"; -import { closeSync, openSync } from "node:fs"; import { constants } from "node:os"; -import type { Writable } from "node:stream"; -import { childLaunch } from "./windows-command.js"; - -// why: `close` normally follows `exit` within the same tick, once the child's own stdio -// pipes report EOF. But a child that forks a grandchild before it dies leaves that -// grandchild holding the inherited write end of our pipe open -- it was handed the -// same descriptors, and nothing takes them back -- so `close` never fires even -// though the process we actually care about is long gone. (`detached: true` is not -// the cause; it is why the grandchild is reachable to kill, since it puts the whole -// tree in one process group.) After `exit`, `wait()` therefore stops waiting for -// `close` once the pipes have gone quiet for this long. -// -// Quiet, not merely elapsed: settling on a bare timer would truncate the output of a -// process that exited while its pipe was still draining (a large `simctl list --json` -// under a loaded event loop), and a truncated capture surfaces as a parse error far -// from its cause. So the window restarts whenever more output arrives, which -// distinguishes "still draining" from "held open by something that has nothing to -// say" -- the only case this needs to escape. -const EXIT_TO_CLOSE_GRACE_MS = 1_000; - -// why: A grandchild that inherits the pipe *and* chatters on it would otherwise restart the -// grace window forever, so the deferral is capped. Reaching this cap means the output -// may be incomplete; nothing else is waiting on it by then. -const EXIT_TO_CLOSE_MAX_DEFERRAL_MS = 5_000; - -// why: A hard bound on how long `run()` waits after a timeout-triggered SIGTERM before escalating to -// SIGKILL. A child that ignores SIGTERM (or is itself stuck in an uninterruptible wait) would -// otherwise hold the caller's `await process.wait()` open forever -- exactly the unbounded wait -// this constant exists to cap. Fixed rather than derived from `timeoutMs`: it is a -// termination-cleanup budget, not a scaled fraction of the operation's own timeout. -const SIGTERM_TO_SIGKILL_GRACE_MS = 10_000; - -export interface ProcessRunOptions { - readonly timeoutMs?: number; - readonly env?: NodeJS.ProcessEnv; - readonly cwd?: string; - /** - * why: `"ignore"` detaches the child's output entirely: nothing is captured and nothing is - * iterable. A long-lived supervised process would otherwise accumulate every line it ever - * wrote in this handle's buffers for as long as it runs. `{ file }` appends the child's - * stdout and stderr to that file (created 0600 when missing, whose directory must exist), - * with the same effect on the handle: nothing is captured. - */ - readonly stdio?: "pipe" | "ignore" | { readonly file: string }; - readonly input?: string; - /** - * why: ends the child when the caller no longer wants its answer: aborting kills the child's - * whole process group, and `run` answers `endedBy: "aborted"`. - */ - readonly signal?: AbortSignal; - /** - * why: a bound on a child's stdout, counted in bytes as it arrives: past it the runner kills the - * child and answers `endedBy: "output-limit"`. Output is kept in memory until the child ends, so - * a caller that reads an answer from an untrusted program sets it. - */ - readonly maxStdoutBytes?: number; - /** - * why: Where `spawn`'s `stdout` and `stderr` iterables end a line. `"newline"` (the default) ends - * one only at `\n`. `"carriage-return-too"` also ends one at a bare `\r`, and yields no empty - * lines: an installer such as `sdkmanager` redraws its progress bar in place with `\r`, so - * without this every update waits in one unfinished line until the process prints its final - * newline. The captured `ProcessResult` text is the same either way. - */ - readonly lineEnd?: LineEnd; -} - -export type LineEnd = "newline" | "carriage-return-too"; - -export interface ProcessResult { - readonly code: number | null; - readonly stdout: string; - readonly stderr: string; - /** why: set when the runner itself killed the child, and why: its `timeoutMs`, `maxStdoutBytes` or `signal`. */ - readonly endedBy?: "timeout" | "output-limit" | "aborted"; - /** why: the name of the signal that ended the child, whoever sent it; absent when it exited by itself. */ - readonly signal?: NodeJS.Signals; -} - -/** - * why: ADR 0005 §19a's spawn: a child whose output is handed to the caller chunk by chunk as it - * arrives and is never accumulated anywhere in this process. - * - * Deliberately a second method rather than an option on `spawn`: `ProcessHandle` exposes its - * output as line-delimited `AsyncIterable`s *and* keeps every chunk so `wait()` can hand back - * a complete `stdout`/`stderr` -- exactly right for the driver calls that parse a tool's whole - * answer, and exactly wrong for `device.exec`, which must forward bytes as they appear and - * hold none of them (§19e: "Output is streamed, never buffered, so there is no size cap"). - * Bolting a "don't buffer" flag onto `spawn` would have left `ProcessResult`'s `stdout`/`stderr` - * silently empty for that mode; a separate return type says so in the type system instead. - * - * Chunks are whatever the child wrote, decoded UTF-8, unsplit: no line assembly, so a caller - * that wants lines does that itself. - */ -export interface ProcessStreamOptions { - readonly env?: NodeJS.ProcessEnv; - readonly cwd?: string; - readonly input?: string; - /** - * why: Piped to the child's stdin and then closed, for input too big to hold as a string. - * Given together with `input`, `inputStream` is ignored. - */ - readonly inputStream?: ReadableStream; - /** - * why: Called with each chunk as it arrives. **May return a promise**, and if it does, this - * runner stops reading that stream until it resolves -- which is how backpressure reaches - * the child: a consumer that cannot place a chunk yet (an SSE client that is not reading, a - * socket that has not drained) slows the command down instead of accumulating its output - * somewhere. ADR 0005 §19e says output is streamed and never buffered; that is only true - * end to end if the slow end can push back, so the callback's promise is the seam that - * makes it true rather than a claim about the fast path only. - */ - readonly onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise; -} - -export interface StreamingProcessResult { - readonly code: number | null; - readonly signal: NodeJS.Signals | null; -} - -export interface StreamingProcessHandle { - readonly pid: number; - /** - * why: Only on a child spawned with `inputStream`: rejects when that stream fails to be read - * to its end, with the bytes written to the child by then. A child that closes its stdin - * early is not a failure and never rejects it. - */ - inputFailed?: Promise; - kill(signal?: NodeJS.Signals): void; - wait(): Promise; -} +import type { StreamingProcessResult } from "./process-runner-types.js"; + +export type { + LineEnd, + NodeProcessRunnerOptions, + ProcessHandle, + ProcessResult, + ProcessRunner, + ProcessRunOptions, + ProcessStreamOptions, + StreamingProcessHandle, + StreamingProcessResult, +} from "./process-runner-types.js"; +export { InputStreamFailedError, ProcessSpawnError } from "./process-runner-types.js"; +export { ExecOutputDeliveryStalledError, NodeProcessRunner } from "./node-process-runner.js"; +export type { + ProcessInvocation, + ProcessMatcher, + ScriptedProcessExpectation, +} from "./scripted-process-runner.js"; +export { ScriptedProcessRunner } from "./scripted-process-runner.js"; /** * why: The single number a shell reports for a finished process: its own exit code, or @@ -146,1116 +35,3 @@ export function exitCodeOf(result: StreamingProcessResult): number { if (result.signal === null) return 1; return 128 + (constants.signals[result.signal] ?? 0); } - -export interface ProcessHandle { - readonly pid: number; - readonly stdout: AsyncIterable; - readonly stderr: AsyncIterable; - kill(signal?: NodeJS.Signals): void; - /** - * why: Stops this child from keeping the parent's event loop alive. Only for a process that - * is meant to outlive the daemon and is reaped by pid instead of by exit (Simlock's adb - * server): everything else is awaited through `wait()` and must keep the loop open. - */ - unref(): void; - wait(): Promise; -} - -/** - * why: A child that could not be started at all -- a binary that exists but is not executable, - * `ETXTBSY` while it is being rewritten, `EAGAIN` under memory pressure. Node reports this - * asynchronously, so it is raised here as a synchronous throw (and therefore a rejected - * promise out of `run`) rather than left to surface as an unhandled `error` event. - */ -export class ProcessSpawnError extends Error { - constructor( - readonly command: string, - readonly args: readonly string[], - ) { - super(`Failed to spawn ${command} ${args.join(" ")}`); - this.name = "ProcessSpawnError"; - } -} - -export interface ProcessRunner { - run( - command: string, - args: readonly string[], - options?: ProcessRunOptions, - ): Promise; - spawn(command: string, args: readonly string[], options?: ProcessRunOptions): ProcessHandle; - spawnStreaming( - command: string, - args: readonly string[], - options: ProcessStreamOptions, - ): StreamingProcessHandle; -} - -export interface NodeProcessRunnerOptions { - readonly sigkillGraceMs?: number; - readonly exitDeferralCapMs?: number; - readonly exitGraceMs?: number; -} - -export class NodeProcessRunner implements ProcessRunner { - readonly sigkillGraceMs: number; - readonly exitDeferralCapMs: number; - readonly exitGraceMs: number; - - constructor(options: NodeProcessRunnerOptions = {}) { - this.sigkillGraceMs = options.sigkillGraceMs ?? SIGTERM_TO_SIGKILL_GRACE_MS; - this.exitDeferralCapMs = options.exitDeferralCapMs ?? EXIT_TO_CLOSE_MAX_DEFERRAL_MS; - this.exitGraceMs = options.exitGraceMs ?? EXIT_TO_CLOSE_GRACE_MS; - } - - async run( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): Promise { - const process = this.#launch(command, args, options); - let killTimeout: NodeJS.Timeout | undefined; - const timeout = - options.timeoutMs === undefined - ? undefined - : setTimeout(() => { - try { - process.kill("SIGTERM", "timeout"); - } catch { - // why: The child may have exited between the timer firing and the kill. - } - // why: SIGTERM is a request, not a guarantee -- a child that ignores it (or is itself - // hung) must not be able to keep this `run()` call waiting indefinitely. Escalate to - // SIGKILL if it hasn't exited within the grace period; cleared below like `timeout` - // itself the moment `process.wait()` actually settles, so a child that does exit - // promptly after SIGTERM never sees the follow-up signal. - killTimeout = setTimeout(() => { - try { - process.kill("SIGKILL"); - } catch { - // why: The child may have exited between the timer firing and the kill. - } - }, this.sigkillGraceMs); - }, options.timeoutMs); - - try { - return await process.wait(); - } finally { - if (timeout !== undefined) { - clearTimeout(timeout); - } - if (killTimeout !== undefined) { - clearTimeout(killTimeout); - } - } - } - - spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { - return this.#launch(command, args, options); - } - - #launch(command: string, args: readonly string[], options: ProcessRunOptions): NodeProcessHandle { - const launch = childLaunch(command, args); - const logFd = - typeof options.stdio === "object" ? openSync(options.stdio.file, "a", 0o600) : undefined; - const child = spawnChildProcess(launch.command, launch.args, { - ...(options.cwd === undefined ? {} : { cwd: options.cwd }), - ...(options.env === undefined ? {} : { env: options.env }), - detached: process.platform !== "win32", - stdio: childStdio(options.stdio, logFd), - // why: the daemon has no console, so on Windows a console program it starts (adb, the - // emulator, node) would otherwise get a new, visible one. - windowsHide: true, - windowsVerbatimArguments: launch.windowsVerbatimArguments, - }); - // why: the child holds its own copy of the descriptor; keeping ours would leak one per start. - if (logFd !== undefined) closeSync(logFd); - - if (child.pid === undefined) { - // why: The spawn failed, and Node will say why by emitting `error` on the next tick. An - // `error` with no listener is a hard, uncatchable process exit, so the listener is - // attached *before* this function leaves -- throwing first would take the daemon - // down over an unreadable `adb`. The reason is lost with it (it does not exist yet), - // which is the price of turning an asynchronous crash into a catchable failure. - child.once("error", () => undefined); - throw new ProcessSpawnError(command, args); - } - - if (options.input !== undefined) { - child.stdin?.end(options.input); - } - - return new NodeProcessHandle( - child, - child.pid, - options.lineEnd ?? "newline", - this.exitDeferralCapMs, - this.exitGraceMs, - launch.windowsVerbatimArguments, - runLimits(options), - ); - } - - spawnStreaming( - command: string, - args: readonly string[], - options: ProcessStreamOptions, - ): StreamingProcessHandle { - const launch = childLaunch(command, args); - const child = spawnChildProcess(launch.command, launch.args, { - ...(options.cwd === undefined ? {} : { cwd: options.cwd }), - ...(options.env === undefined ? {} : { env: options.env }), - detached: process.platform !== "win32", - stdio: "pipe", - // why: as in `spawn`: no visible console window for a child of a daemon that has none. - windowsHide: true, - windowsVerbatimArguments: launch.windowsVerbatimArguments, - }); - - if (child.pid === undefined) { - // why: Same reasoning as `spawn` above: attach a listener before throwing, or Node's - // asynchronous `error` takes the daemon down instead of rejecting one operation. - child.once("error", () => undefined); - throw new ProcessSpawnError(command, args); - } - - const handle = new NodeStreamingProcessHandle( - child, - child.pid, - options.onChunk, - this.exitDeferralCapMs, - this.exitGraceMs, - launch.windowsVerbatimArguments, - ); - if (options.input !== undefined) { - // why: One shot, then closed (ADR 0005 §19c): a command reading stdin sees exactly this and - // then EOF. `end` is safe even if the child already exited -- the EPIPE that produces is - // swallowed below rather than left to surface as an unhandled `error` on the stream. - child.stdin?.on("error", () => undefined); - child.stdin?.end(options.input); - } else if (options.inputStream !== undefined && child.stdin) { - const inputFailed = pumpInput(options.inputStream, child.stdin); - // why: a caller that never looks at `inputFailed` must not turn a read error into an unhandled rejection that ends the daemon. - inputFailed.catch(() => undefined); - handle.inputFailed = inputFailed; - } - return handle; - } -} - -/** why: the input stream of a streamed child failed to be read to its end; `bytesWritten` is what the child's stdin had been handed by then. */ -export class InputStreamFailedError extends Error { - constructor( - readonly bytesWritten: number, - cause: unknown, - ) { - super(`Reading the input stream failed after ${String(bytesWritten)} bytes`, { cause }); - this.name = "InputStreamFailedError"; - } -} - -/** - * why: writes the stream to the child's stdin chunk by chunk, waiting for `drain` so a slow child holds the read back (nothing is buffered, ADR 0005 §19e), then closes stdin. A child that closes its stdin early (EPIPE) cancels the read quietly; only a read error rejects the returned promise, which never resolves otherwise. - */ -function pumpInput(source: ReadableStream, stdin: Writable): Promise { - return new Promise((_resolve, reject) => { - const reader = source.getReader(); - let written = 0; - let closed = false; - const onClosed = (): void => { - closed = true; - void reader.cancel().catch(() => undefined); - }; - stdin.on("error", onClosed); - stdin.on("close", onClosed); - void (async () => { - for (;;) { - let result: ReadableStreamReadResult; - try { - result = await reader.read(); - } catch (error) { - if (!closed) reject(new InputStreamFailedError(written, error)); - return; - } - if (closed) return; - if (result.done) { - stdin.end(); - return; - } - written += result.value.byteLength; - if (!stdin.write(result.value) && !closed) { - await new Promise((resolve) => { - const done = (): void => { - stdin.off("drain", done); - stdin.off("close", done); - resolve(); - }; - stdin.once("drain", done); - stdin.once("close", done); - }); - } - } - })(); - }); -} - -/** - * why: Thrown by `NodeStreamingProcessHandle#wait()` when the child exited but a chunk's delivery - * -- the promise `onChunk` returned, which this handle was waiting on before resuming the - * pipe for backpressure (ADR 0005 §19e) -- was still outstanding `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` - * later. A paused readable never reaches `close` and stops emitting `data`, so nothing here can - * *prove* the remaining output was delivered; resolving `{ exitCode }` as if it had been would - * be exactly the silent truncation streaming without buffering exists to avoid. The caller sees - * this as `device.exec`'s terminal error, the same shape a `PASSTHROUGH_REFUSED` or - * `EXEC_TIMEOUT` already is on both transports -- not a clean exit that quietly drops a tail. - */ -export class ExecOutputDeliveryStalledError extends Error { - constructor(deferralCapMs: number = EXIT_TO_CLOSE_MAX_DEFERRAL_MS) { - super( - "The command exited, but delivering one of its output chunks was still unresolved " + - `${String(deferralCapMs)}ms later; treating this as a clean exit could ` + - "have reported an exit code while silently dropping the tail of the command's output.", - ); - this.name = "ExecOutputDeliveryStalledError"; - } -} - -/** - * why: `spawnStreaming`'s handle. Holds no output: every chunk is handed to `onChunk` the moment - * the stream emits it and is then forgotten, which is what lets `device.exec` stream - * arbitrarily large output through a daemon that never grows for it. - * - * Settling repeats `NodeProcessHandle`'s `exit`-then-quiet-window dance for the same reason - * (see `EXIT_TO_CLOSE_GRACE_MS`): a grandchild holding the inherited pipe open (`adb - * start-server` is exactly this) means `close` may never fire even though the process the - * caller asked about is gone. - * - * The quiet window that dance relies on only means anything when the pipe is idle because the - * child has nothing left to write. `#forward` below pauses the readable while a chunk's - * delivery to the consumer is outstanding (the backpressure ADR 0005 §19e asks for): a paused - * readable emits no `data`, so `#chunkCount` stops moving for a reason that has nothing to do - * with the child being gone. Settling under that -- as this class once did -- resolves `wait()` - * while a chunk still sits undelivered in the paused pipe, and the caller returns `{ exitCode }` - * having silently dropped it. - * - * `close` itself is not the all-clear it looks like, either: verified against real Node (a - * child that writes two small chunks in quick succession and calls `process.exit()`), the - * stdio pipe's `close` can fire in the very same tick as a `data` event delivering a chunk - * this handle just paused on -- Node drains whatever the OS pipe was already holding as part - * of noticing EOF, pause or no pause. So both `close` and the exit grace window fail closed - * through the same `#settleWhenDrained`: `#pendingDeliveries` is this handle's own state (it is - * the one that called `pause()`), checked directly rather than inferred from the chunk counter - * the grandchild case already overloads, and only `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` with a - * delivery still outstanding at that deadline ends the wait -- as a rejection - * (`ExecOutputDeliveryStalledError`), not a settle, because at that point something really may - * have been dropped and pretending otherwise is the one thing this class must not do. A chatty - * grandchild with **no** pending delivery hitting the same deadline still settles as before: - * nothing of the command's own output is waiting on anyone in that case, only unrelated noise - * this handle was never going to forward to a live consumer anyway (see `#execDevice` in - * `daemon/server.ts`, torn down by then). - */ -class NodeStreamingProcessHandle implements StreamingProcessHandle { - readonly pid: number; - inputFailed?: Promise; - readonly #result: Promise; - #chunkCount = 0; - #pendingDeliveries = 0; - /** why: Per-stream half of `#pendingDeliveries` -- the total is what the exit-settle path needs - * ("is anything at all still outstanding"), this is what `#forward`'s own `resume()` needs - * ("is *this* stream's own last outstanding delivery the one that just settled"). A stream - * can hold more than one outstanding delivery at once (a paused readable still drains - * whatever it already had at EOF -- see the class doc), so resuming on any settle rather - * than this stream's own count reaching zero would resume it while another delivery on the - * same stream is still unaccounted for. */ - #pendingByStream: Record<"stdout" | "stderr", number> = { stderr: 0, stdout: 0 }; - #deliveryWaiters: Array<() => void> = []; - #killedWith: NodeJS.Signals | undefined; - - constructor( - private readonly child: ChildProcess, - pid: number, - onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, - private readonly exitDeferralCapMs: number, - private readonly exitGraceMs: number, - private readonly wrapped: boolean, - ) { - this.pid = pid; - this.#forward(child.stdout, "stdout", onChunk); - this.#forward(child.stderr, "stderr", onChunk); - this.#result = new Promise((resolve, reject) => { - child.once("error", reject); - - let settled = false; - const settle = (code: number | null, signal: NodeJS.Signals | null): void => { - if (settled) return; - settled = true; - const killedWith = this.#killedWith; - resolve( - wrapped && killedWith !== undefined - ? { code: null, signal: killedWith } - : { code, signal }, - ); - }; - const fail = (error: Error): void => { - if (settled) return; - settled = true; - reject(error); - }; - // why: Neither `close` nor the exit grace window may resolve this promise while a chunk's - // delivery is still outstanding -- both funnel through here instead of settling - // directly, each with its own budget for how long it will wait on a delivery that - // never finishes before giving up and failing loudly instead. Re-checked against a - // timer at the deadline, not only woken by a delivery settling: a delivery that never - // settles at all -- the case this exists for -- would otherwise leave nothing to wake - // this back up and notice the deadline has passed. - const settleWhenDrained = (deadline: number, onDrained: () => void): void => { - if (settled) return; - if (this.#pendingDeliveries === 0) { - onDrained(); - return; - } - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { - fail(new ExecOutputDeliveryStalledError(this.exitDeferralCapMs)); - return; - } - let deadlineTimer: NodeJS.Timeout | undefined; - const timedOut = new Promise((resolve) => { - deadlineTimer = setTimeout(resolve, remainingMs); - deadlineTimer.unref(); - }); - void Promise.race([this.#nextDeliverySettled(), timedOut]).then(() => { - if (deadlineTimer !== undefined) clearTimeout(deadlineTimer); - settleWhenDrained(deadline, onDrained); - }); - }; - - child.once("close", (code, signal) => { - settleWhenDrained(Date.now() + this.exitDeferralCapMs, () => settle(code, signal)); - }); - child.once("exit", (code, signal) => { - const deadline = Date.now() + this.exitDeferralCapMs; - const armGrace = (chunksAtArm: number): void => { - const timer = setTimeout(() => { - if (settled) return; - // why: A pending delivery always wins over "chunk count still changing": it is this - // handle's own state, not an inference, and it is checked first so a chattering - // stream with one stuck delivery does not keep re-arming past it forever. - if ( - this.#pendingDeliveries === 0 && - this.#chunkCount !== chunksAtArm && - Date.now() < deadline - ) { - armGrace(this.#chunkCount); - return; - } - settleWhenDrained(deadline, () => settle(code, signal)); - }, this.exitGraceMs); - timer.unref(); - }; - armGrace(this.#chunkCount); - }); - }); - // why: A late `error` on a handle whose `wait()` nobody kept (a caller that killed and walked - // away) must not become an unhandled rejection and end the daemon; a caller that does - // await `wait()` still sees the rejection. - this.#result.catch(() => undefined); - } - - kill(signal: NodeJS.Signals = "SIGTERM"): void { - if (isRunning(this.child)) this.#killedWith ??= signal; - killProcessTree(this.child, this.pid, signal, this.wrapped); - } - - wait(): Promise { - return this.#result; - } - - #nextDeliverySettled(): Promise { - return new Promise((resolve) => { - this.#deliveryWaiters.push(resolve); - }); - } - - #forward( - stream: NodeJS.ReadableStream | null, - name: "stdout" | "stderr", - onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, - ): void { - if (stream === null) return; - stream.setEncoding("utf8"); - stream.on("data", (chunk: string) => { - this.#chunkCount += 1; - // why: `onChunk` is typed `void | Promise`, but nothing enforces that a transport's - // implementation never throws synchronously (a bug in a socket write, say). Called bare - // inside a `data` listener, that throw would become an uncaught exception and take the - // whole daemon down over one bad chunk delivery -- caught here and treated exactly like - // a delivery whose returned promise rejected: a failed delivery, not a crash. - let delivered: void | Promise; - try { - delivered = onChunk(name, chunk); - } catch (error) { - delivered = Promise.reject(error instanceof Error ? error : new Error(String(error))); - } - if (delivered === undefined) return; - // why: The consumer is not ready for more yet. Pausing the readable stops this child at the - // pipe -- it fills the OS buffer and then blocks in its own `write` -- which is the only - // place backpressure can be applied without holding the bytes somewhere. Resumed on - // either outcome: a failed delivery (a dead socket) must not wedge the process, and the - // exec timeout still bounds a child nobody drains. Tracked in `#pendingDeliveries` so the - // exit settle path can tell "paused waiting on the consumer" apart from "child is gone", - // and separately in `#pendingByStream` so `resume()` fires only once *this* stream's own - // outstanding count reaches zero -- not on any delivery settling, which would resume a - // stream while one of its own other deliveries (two chunks landing before either - // resolves, e.g. at EOF) is still unaccounted for. - this.#pendingDeliveries += 1; - this.#pendingByStream[name] += 1; - stream.pause(); - const settleDelivery = (): void => { - this.#pendingDeliveries -= 1; - this.#pendingByStream[name] -= 1; - if (this.#pendingByStream[name] === 0) stream.resume(); - const waiters = this.#deliveryWaiters; - this.#deliveryWaiters = []; - for (const waiter of waiters) waiter(); - }; - void Promise.resolve(delivered).then(settleDelivery, settleDelivery); - }); - } -} - -const TASKKILL_TIMEOUT_MS = 10_000; - -function runLimits({ - maxStdoutBytes, - signal, -}: ProcessRunOptions): Pick { - return { - ...(maxStdoutBytes === undefined ? {} : { maxStdoutBytes }), - ...(signal === undefined ? {} : { signal }), - }; -} - -/** - * why: Windows has no process groups, and a batch file runs under a `cmd.exe` wrapper - * (`windows-command.ts`): signalling the wrapper alone would leave the script's own processes, - * `sdkmanager`'s Java, running. `taskkill /T` walks the tree from the wrapper's pid, so it runs - * while the wrapper is still alive; the wrapper outlives what it runs, since a script runs its - * commands in the foreground. Windows ends a process on any signal, so `/F` is no harsher than - * `child.kill`. `taskkill` makes the wrapper exit with a code where `child.kill` reports a - * signal, so the handle reports the signal it sent instead. Should `taskkill` itself fail, the - * wrapper alone is killed as before. - */ -function killWindowsProcessTree(child: ChildProcess, pid: number, signal: NodeJS.Signals): void { - const fallback = () => { - if (isRunning(child)) child.kill(signal); - }; - const taskkill = spawnChildProcess("taskkill", ["/pid", String(pid), "/T", "/F"], { - stdio: "ignore", - timeout: TASKKILL_TIMEOUT_MS, - windowsHide: true, - }); - taskkill.once("error", fallback); - taskkill.once("exit", (code) => { - if (code !== 0) fallback(); - }); -} - -/** - * why: Signals the child's whole process group (children are spawned `detached`, so the group is - * the unit that actually dies), or on Windows a batch file's whole tree and any other child - * alone, treating an already-exited process as success. Extracted so `ProcessHandle` and `StreamingProcessHandle` - * kill identically -- a timeout that only reached the direct child would leave a tool's own - * subprocesses running. - * - * Guarded on `child.exitCode`/`child.signalCode` -- both still `null` only while the child is - * actually running -- before ever calling `process.kill(-pid, signal)` directly: that call - * bypasses `ChildProcess.kill`'s own "already exited" guard, and `wait()` can defer settling - * up to `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` past the real `exit` event (a chatty grandchild holding - * stdio open, or a stalled delivery). Node frees a reaped pid for the kernel to recycle, so a - * timeout firing in that window without this guard could signal a process group the kernel has - * already handed to someone else -- latent before this PR, but `device.exec` is what makes the - * timing remotely triggerable. - */ -function killProcessTree( - child: ChildProcess, - pid: number, - signal: NodeJS.Signals, - wrapped: boolean, -): void { - if (!isRunning(child)) return; - - if (process.platform === "win32") { - if (wrapped) killWindowsProcessTree(child, pid, signal); - else child.kill(signal); - return; - } - - try { - process.kill(-pid, signal); - } catch (error: unknown) { - if (isNoSuchProcessError(error)) { - return; - } - - throw error; - } -} - -// why: a wrapper that already exited, its output still held open by a grandchild, is never -// signalled, so its own exit code stands rather than a kill nobody sent. -function isRunning(child: ChildProcess): boolean { - return child.exitCode === null && child.signalCode === null; -} - -class NodeProcessHandle implements ProcessHandle { - readonly pid: number; - readonly stdout = new LineBuffer(); - readonly stderr = new LineBuffer(); - readonly #stdoutChunks: string[] = []; - readonly #stderrChunks: string[] = []; - readonly #result: Promise; - #killedWith: NodeJS.Signals | undefined; - #endedBy: ProcessResult["endedBy"]; - - constructor( - private readonly child: ChildProcess, - pid: number, - lineEnd: LineEnd, - private readonly exitDeferralCapMs: number, - private readonly exitGraceMs: number, - private readonly wrapped: boolean, - limits: Pick = {}, - ) { - this.pid = pid; - captureLines(child.stdout, this.stdout, this.#stdoutChunks, lineEnd); - captureLines(child.stderr, this.stderr, this.#stderrChunks, lineEnd); - this.#watchLimits(limits); - this.#result = new Promise((resolve, reject) => { - child.once("error", reject); - - let settled = false; - const settle = (code: number | null, signal: NodeJS.Signals | null): void => { - if (settled) { - return; - } - settled = true; - resolve({ - code: wrapped && this.#killedWith !== undefined ? null : code, - stderr: this.#stderrChunks.join(""), - stdout: this.#stdoutChunks.join(""), - ...(this.#endedBy === undefined ? {} : { endedBy: this.#endedBy }), - ...(signal === null ? {} : { signal }), - }); - }; - - child.once("close", (code, signal) => { - settle(code, signal); - }); - child.once("exit", (code, signal) => { - const deadline = Date.now() + this.exitDeferralCapMs; - const capturedSoFar = (): number => this.#stdoutChunks.length + this.#stderrChunks.length; - const armGrace = (chunksAtArm: number): void => { - const timer = setTimeout(() => { - if (capturedSoFar() !== chunksAtArm && Date.now() < deadline) { - armGrace(capturedSoFar()); - return; - } - settle(code, signal); - }, this.exitGraceMs); - // why: A pending grace timer must never keep the Node process alive on its own. - timer.unref(); - }; - armGrace(capturedSoFar()); - }); - }); - // why: A child nobody waits on (the supervised adb server) would otherwise turn a late - // `error` into an unhandled rejection, which ends the daemon. Marking the promise - // handled here changes nothing for a caller that does await `wait()`. - this.#result.catch(() => undefined); - } - - /** - * why: the output bound counts bytes as they arrive, and an aborted signal ends the child the - * same way; both kill with SIGKILL, since the caller has no use for a graceful end of a child - * that is printing without end or whose answer nobody wants. - */ - #watchLimits({ - maxStdoutBytes, - signal, - }: Pick): void { - if (maxStdoutBytes !== undefined) { - let received = 0; - this.child.stdout?.on("data", (chunk: string | Buffer) => { - received += Buffer.byteLength(chunk); - if (received > maxStdoutBytes) this.kill("SIGKILL", "output-limit"); - }); - } - if (signal !== undefined) { - const abort = (): void => this.kill("SIGKILL", "aborted"); - if (signal.aborted) abort(); - else signal.addEventListener("abort", abort, { once: true }); - this.child.once("close", () => signal.removeEventListener("abort", abort)); - } - } - - kill(signal: NodeJS.Signals = "SIGTERM", endedBy?: ProcessResult["endedBy"]): void { - if (isRunning(this.child)) { - this.#killedWith ??= signal; - this.#endedBy ??= endedBy; - } - killProcessTree(this.child, this.pid, signal, this.wrapped); - } - - unref(): void { - this.child.unref(); - } - - wait(): Promise { - return this.#result; - } -} - -export interface ProcessInvocation { - readonly command: string; - readonly args: readonly string[]; - readonly options: ProcessRunOptions; -} - -export interface ProcessMatcher { - readonly command: string | RegExp; - readonly args: readonly (string | RegExp)[]; -} - -export interface ScriptedProcessExpectation { - readonly match: ProcessMatcher; - readonly result?: ProcessResult; - readonly stdoutLines?: readonly string[]; - readonly stderrLines?: readonly string[]; - readonly hangs?: boolean; - /** - * why: Raw chunks for `spawnStreaming`, emitted in this exact order across both streams -- - * which `stdoutLines`/`stderrLines` cannot express, since they are two separate lists with - * no ordering between them. A test asserting that interleaved output reaches a caller in - * arrival order needs that, so it is stated here rather than inferred. When absent, - * `spawnStreaming` falls back to the line lists (stdout then stderr, newline-terminated). - * `spawn` replays them too, in place of the line lists, through the same line splitting a - * real child's output gets -- so a test can say exactly where one chunk ends. - */ - readonly chunks?: readonly { readonly stream: "stdout" | "stderr"; readonly chunk: string }[]; - /** - * why: Models a child that ignores `SIGTERM` -- only `SIGKILL` ends it. Without this a scripted - * `hangs` process dies on the first signal, so the escalation every timeout path depends on - * (`SIGTERM`, then `SIGKILL` after a grace window) is never actually exercised: the code that - * sends the second signal would be dead and nothing would notice. - */ - readonly ignoresSigterm?: boolean; - readonly onSpawn?: (args: readonly string[]) => void | Promise; -} - -export class ScriptedProcessRunner implements ProcessRunner { - readonly calls: ProcessInvocation[] = []; - /** - * why: The handle each `spawn` returned, so a test can assert what the caller did *with* - * the child rather than only how it was started. `unref` is the case - * that needs it: nothing about the argv or the options says whether the event loop was - * released, and a spawned child that stays referenced keeps the daemon alive after - * `daemon stop`. - */ - readonly handles: (ProcessHandle & { readonly unreffed: boolean })[] = []; - #nextPid = 1; - readonly #expectations: ScriptedProcessExpectation[]; - - constructor(expectations: readonly ScriptedProcessExpectation[]) { - this.#expectations = [...expectations]; - } - - async run( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): Promise { - return this.spawn(command, args, options).wait(); - } - - spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { - const invocation = { args: [...args], command, options }; - this.calls.push(invocation); - const expectation = this.#expectations.shift(); - - if (expectation === undefined || !matches(expectation.match, invocation)) { - throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); - } - - void expectation.onSpawn?.([...args]); - const handle = new ScriptedProcessHandle( - this.#nextPid++, - expectation, - options.lineEnd ?? "newline", - ); - this.handles.push(handle); - return handle; - } - - spawnStreaming( - command: string, - args: readonly string[], - options: ProcessStreamOptions, - ): StreamingProcessHandle { - const invocation = { - args: [...args], - command, - // why: Recorded as a `ProcessRunOptions` so `calls` stays one list a test can read uniformly; - // `onChunk` is a callback, not configuration, and nothing would assert on it. - options: { - ...(options.cwd === undefined ? {} : { cwd: options.cwd }), - ...(options.env === undefined ? {} : { env: options.env }), - ...(options.input === undefined ? {} : { input: options.input }), - }, - }; - this.calls.push(invocation); - const expectation = this.#expectations.shift(); - - if (expectation === undefined || !matches(expectation.match, invocation)) { - throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); - } - - return new ScriptedStreamingProcessHandle( - this.#nextPid++, - expectation, - options.onChunk, - options.inputStream, - ); - } -} - -/** - * why: The scripted counterpart of `NodeStreamingProcessHandle`: emits the expectation's chunks - * synchronously (so a test needs no timers to see them), then settles unless the expectation - * `hangs` -- in which case only `kill()` settles it, reporting the signal the way a real - * killed child does. That is what makes the daemon's exec timeout testable without a real - * process or a real clock. - */ -class ScriptedStreamingProcessHandle implements StreamingProcessHandle { - inputFailed?: Promise; - readonly #result: Promise; - #resolve!: (result: StreamingProcessResult) => void; - #finished = false; - - readonly #ignoresSigterm: boolean; - - constructor( - readonly pid: number, - expectation: ScriptedProcessExpectation, - onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, - inputStream?: ReadableStream, - ) { - this.#ignoresSigterm = expectation.ignoresSigterm ?? false; - if (inputStream !== undefined) this.inputFailed = scriptedInputFailure(inputStream); - this.#result = new Promise((resolve) => { - this.#resolve = resolve; - }); - void this.#deliver(expectation, onChunk); - } - - /** - * why: One chunk at a time, awaiting each delivery before the next -- the same discipline - * `NodeStreamingProcessHandle` gets by pausing a readable, so a test that stalls a delivery - * sees the scripted child stall too rather than racing ahead of it. The command settles only - * once its output is delivered, which keeps "output, then exit code" true here as well. - */ - async #deliver( - expectation: ScriptedProcessExpectation, - onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, - ): Promise { - for (const { chunk, stream } of scriptedChunks(expectation)) { - if (this.#finished) return; - await onChunk(stream, chunk); - } - if (!expectation.hangs) { - this.#finish({ code: expectation.result?.code ?? 0, signal: null }); - } - } - - kill(signal: NodeJS.Signals = "SIGTERM"): void { - if (this.#ignoresSigterm && signal !== "SIGKILL") return; - this.#finish({ code: null, signal }); - } - - wait(): Promise { - return this.#result; - } - - #finish(result: StreamingProcessResult): void { - if (this.#finished) return; - this.#finished = true; - this.#resolve(result); - } -} - -/** why: the scripted child reads its whole input at once, so a test sees a failing stream reject `inputFailed` as the real adapter does. */ -function scriptedInputFailure(source: ReadableStream): Promise { - const failed = new Promise((_resolve, reject) => { - void (async () => { - const reader = source.getReader(); - let read = 0; - try { - for (;;) { - const result = await reader.read(); - if (result.done) return; - read += result.value.byteLength; - } - } catch (error) { - reject(new InputStreamFailedError(read, error)); - } - })(); - }); - failed.catch(() => undefined); - return failed; -} - -function scriptedChunks( - expectation: ScriptedProcessExpectation, -): readonly { readonly stream: "stdout" | "stderr"; readonly chunk: string }[] { - if (expectation.chunks !== undefined) return expectation.chunks; - return [ - ...(expectation.stdoutLines ?? []).map((line) => ({ - chunk: `${line}\n`, - stream: "stdout" as const, - })), - ...(expectation.stderrLines ?? []).map((line) => ({ - chunk: `${line}\n`, - stream: "stderr" as const, - })), - ]; -} - -class ScriptedProcessHandle implements ProcessHandle { - unreffed = false; - readonly stdout = new LineBuffer(); - readonly stderr = new LineBuffer(); - readonly #result: Promise; - #resolve!: (result: ProcessResult) => void; - #finished = false; - - constructor( - readonly pid: number, - private readonly expectation: ScriptedProcessExpectation, - lineEnd: LineEnd, - ) { - this.#result = new Promise((resolve) => { - this.#resolve = resolve; - }); - if (expectation.chunks === undefined) { - this.#writeLines(expectation.stdoutLines, this.stdout, lineEnd); - this.#writeLines(expectation.stderrLines, this.stderr, lineEnd); - } else { - this.#writeChunks(expectation.chunks, lineEnd); - } - - if (!expectation.hangs) { - this.#finish(expectation.result ?? defaultResult(expectation)); - } - } - - kill(signal: NodeJS.Signals = "SIGTERM"): void { - if (this.expectation.ignoresSigterm === true && signal !== "SIGKILL") return; - this.#finish({ code: null, stderr: "", stdout: "" }); - } - - unref(): void { - this.unreffed = true; - } - - wait(): Promise { - return this.#result; - } - - /** - * why: Each scripted line is replayed as the child would have printed it, `\n`-terminated, through - * the same splitter the real handle uses -- so a scripted line holding `\r` redraws splits - * exactly as real output would under the caller's `lineEnd`. - */ - #writeLines( - lines: readonly string[] | undefined, - destination: LineBuffer, - lineEnd: LineEnd, - ): void { - const splitter = new LineSplitter(lineEnd); - for (const line of lines ?? []) { - for (const split of splitter.feed(`${line}\n`)) destination.push(split); - } - } - - #writeChunks(chunks: NonNullable, lineEnd: LineEnd): void { - const splitters = { stderr: new LineSplitter(lineEnd), stdout: new LineSplitter(lineEnd) }; - for (const { chunk, stream } of chunks) { - for (const line of splitters[stream].feed(chunk)) this[stream].push(line); - } - for (const stream of ["stdout", "stderr"] as const) { - for (const line of splitters[stream].end()) this[stream].push(line); - } - } - - #finish(result: ProcessResult): void { - if (this.#finished) { - return; - } - - this.#finished = true; - this.stdout.close(); - this.stderr.close(); - this.#resolve(result); - } -} - -class LineBuffer implements AsyncIterable { - readonly #lines: string[] = []; - readonly #waiters: Array<(line: string | undefined) => void> = []; - #closed = false; - - push(line: string): void { - const waiter = this.#waiters.shift(); - if (waiter !== undefined) { - waiter(line); - return; - } - - this.#lines.push(line); - } - - close(): void { - this.#closed = true; - - while (this.#waiters.length > 0) { - this.#waiters.shift()?.(undefined); - } - } - - async *[Symbol.asyncIterator](): AsyncGenerator { - while (true) { - const line = await this.#next(); - if (line === undefined) { - return; - } - - yield line; - } - } - - #next(): Promise { - const line = this.#lines.shift(); - if (line !== undefined) { - return Promise.resolve(line); - } - - if (this.#closed) { - return Promise.resolve(undefined); - } - - return new Promise((resolve) => { - this.#waiters.push(resolve); - }); - } -} - -/** - * why: The one place process output is cut into lines, for the real handle and the scripted one. - * `\r\n` is always a single line end. Under `"carriage-return-too"` a bare `\r` ends a line as - * well, and empty lines are dropped -- which is also what keeps a `\r\n` pair split across two - * chunks one line end rather than a line plus an empty one. - */ -class LineSplitter { - #remainder = ""; - - constructor(private readonly lineEnd: LineEnd) {} - - feed(chunk: string): string[] { - const parts = `${this.#remainder}${chunk}`.split( - this.lineEnd === "carriage-return-too" ? /\r\n|\r|\n/ : /\r?\n/, - ); - this.#remainder = parts.pop() ?? ""; - return this.#kept(parts); - } - - end(): string[] { - const tail = this.#remainder; - this.#remainder = ""; - return tail === "" ? [] : this.#kept([tail]); - } - - #kept(lines: string[]): string[] { - return this.lineEnd === "carriage-return-too" ? lines.filter((line) => line !== "") : lines; - } -} - -function captureLines( - stream: NodeJS.ReadableStream | null, - destination: LineBuffer, - chunks: string[], - lineEnd: LineEnd, -): void { - // why: No stream means the child was spawned with its output detached (`stdio: "ignore"`, or a - // log file taking both descriptors). Closing the buffer - // straight away is what keeps `for await (const line of handle.stdout)` a loop that - // ends immediately rather than one that never yields and never returns. - if (stream === null) { - destination.close(); - return; - } - - const splitter = new LineSplitter(lineEnd); - stream.setEncoding("utf8"); - stream.on("data", (chunk: string) => { - chunks.push(chunk); - for (const line of splitter.feed(chunk)) { - destination.push(line); - } - }); - stream.on("end", () => { - for (const line of splitter.end()) { - destination.push(line); - } - destination.close(); - }); -} - -function matches(matcher: ProcessMatcher, invocation: ProcessInvocation): boolean { - return ( - matchesPart(matcher.command, invocation.command) && - matcher.args.length === invocation.args.length && - matcher.args.every((argument, index) => matchesPart(argument, invocation.args[index] ?? "")) - ); -} - -function matchesPart(matcher: string | RegExp, value: string): boolean { - return typeof matcher === "string" ? matcher === value : matcher.test(value); -} - -function defaultResult(expectation: ScriptedProcessExpectation): ProcessResult { - if (expectation.chunks !== undefined) { - const text = (stream: "stdout" | "stderr"): string => - expectation.chunks - ?.filter((chunk) => chunk.stream === stream) - .map(({ chunk }) => chunk) - .join("") ?? ""; - return { code: 0, stderr: text("stderr"), stdout: text("stdout") }; - } - return { - code: 0, - stderr: (expectation.stderrLines ?? []).join("\n"), - stdout: (expectation.stdoutLines ?? []).join("\n"), - }; -} - -function isNoSuchProcessError(error: unknown): error is NodeJS.ErrnoException { - return typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH"; -} - -function childStdio( - stdio: ProcessRunOptions["stdio"], - logFd: number | undefined, -): "pipe" | "ignore" | ["ignore", number, number] { - if (logFd !== undefined) return ["ignore", logFd, logFd]; - return stdio === undefined || typeof stdio === "object" ? "pipe" : stdio; -} diff --git a/src/ports/process-tree-kill.ts b/src/ports/process-tree-kill.ts new file mode 100644 index 000000000..df656417b --- /dev/null +++ b/src/ports/process-tree-kill.ts @@ -0,0 +1,91 @@ +import { spawn as spawnChildProcess, type ChildProcess } from "node:child_process"; + +import type { ProcessRunOptions } from "./process-runner-types.js"; + +const TASKKILL_TIMEOUT_MS = 10_000; + +export function runLimits({ + maxStdoutBytes, + signal, +}: ProcessRunOptions): Pick { + return { + ...(maxStdoutBytes === undefined ? {} : { maxStdoutBytes }), + ...(signal === undefined ? {} : { signal }), + }; +} + +/** + * why: Windows has no process groups, and a batch file runs under a `cmd.exe` wrapper + * (`windows-command.ts`): signalling the wrapper alone would leave the script's own processes, + * `sdkmanager`'s Java, running. `taskkill /T` walks the tree from the wrapper's pid, so it runs + * while the wrapper is still alive; the wrapper outlives what it runs, since a script runs its + * commands in the foreground. Windows ends a process on any signal, so `/F` is no harsher than + * `child.kill`. `taskkill` makes the wrapper exit with a code where `child.kill` reports a + * signal, so the handle reports the signal it sent instead. Should `taskkill` itself fail, the + * wrapper alone is killed as before. + */ +function killWindowsProcessTree(child: ChildProcess, pid: number, signal: NodeJS.Signals): void { + const fallback = () => { + if (isRunning(child)) child.kill(signal); + }; + const taskkill = spawnChildProcess("taskkill", ["/pid", String(pid), "/T", "/F"], { + stdio: "ignore", + timeout: TASKKILL_TIMEOUT_MS, + windowsHide: true, + }); + taskkill.once("error", fallback); + taskkill.once("exit", (code) => { + if (code !== 0) fallback(); + }); +} + +/** + * why: Signals the child's whole process group (children are spawned `detached`, so the group is + * the unit that actually dies), or on Windows a batch file's whole tree and any other child + * alone, treating an already-exited process as success. Extracted so `ProcessHandle` and `StreamingProcessHandle` + * kill identically -- a timeout that only reached the direct child would leave a tool's own + * subprocesses running. + * + * Guarded on `child.exitCode`/`child.signalCode` -- both still `null` only while the child is + * actually running -- before ever calling `process.kill(-pid, signal)` directly: that call + * bypasses `ChildProcess.kill`'s own "already exited" guard, and `wait()` can defer settling + * up to `EXIT_TO_CLOSE_MAX_DEFERRAL_MS` past the real `exit` event (a chatty grandchild holding + * stdio open, or a stalled delivery). Node frees a reaped pid for the kernel to recycle, so a + * timeout firing in that window without this guard could signal a process group the kernel has + * already handed to someone else -- latent before this PR, but `device.exec` is what makes the + * timing remotely triggerable. + */ +export function killProcessTree( + child: ChildProcess, + pid: number, + signal: NodeJS.Signals, + wrapped: boolean, +): void { + if (!isRunning(child)) return; + + if (process.platform === "win32") { + if (wrapped) killWindowsProcessTree(child, pid, signal); + else child.kill(signal); + return; + } + + try { + process.kill(-pid, signal); + } catch (error: unknown) { + if (isNoSuchProcessError(error)) { + return; + } + + throw error; + } +} + +// why: a wrapper that already exited, its output still held open by a grandchild, is never +// signalled, so its own exit code stands rather than a kill nobody sent. +export function isRunning(child: ChildProcess): boolean { + return child.exitCode === null && child.signalCode === null; +} + +function isNoSuchProcessError(error: unknown): error is NodeJS.ErrnoException { + return typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH"; +} diff --git a/src/ports/scripted-process-runner.ts b/src/ports/scripted-process-runner.ts new file mode 100644 index 000000000..a5fc6e32e --- /dev/null +++ b/src/ports/scripted-process-runner.ts @@ -0,0 +1,333 @@ +import { LineBuffer, LineSplitter } from "./process-line-buffer.js"; +import { + InputStreamFailedError, + type LineEnd, + type ProcessHandle, + type ProcessResult, + type ProcessRunOptions, + type ProcessRunner, + type ProcessStreamOptions, + type StreamingProcessHandle, + type StreamingProcessResult, +} from "./process-runner-types.js"; + +export interface ProcessInvocation { + readonly command: string; + readonly args: readonly string[]; + readonly options: ProcessRunOptions; +} + +export interface ProcessMatcher { + readonly command: string | RegExp; + readonly args: readonly (string | RegExp)[]; +} + +export interface ScriptedProcessExpectation { + readonly match: ProcessMatcher; + readonly result?: ProcessResult; + readonly stdoutLines?: readonly string[]; + readonly stderrLines?: readonly string[]; + readonly hangs?: boolean; + /** + * why: Raw chunks for `spawnStreaming`, emitted in this exact order across both streams -- + * which `stdoutLines`/`stderrLines` cannot express, since they are two separate lists with + * no ordering between them. A test asserting that interleaved output reaches a caller in + * arrival order needs that, so it is stated here rather than inferred. When absent, + * `spawnStreaming` falls back to the line lists (stdout then stderr, newline-terminated). + * `spawn` replays them too, in place of the line lists, through the same line splitting a + * real child's output gets -- so a test can say exactly where one chunk ends. + */ + readonly chunks?: readonly { readonly stream: "stdout" | "stderr"; readonly chunk: string }[]; + /** + * why: Models a child that ignores `SIGTERM` -- only `SIGKILL` ends it. Without this a scripted + * `hangs` process dies on the first signal, so the escalation every timeout path depends on + * (`SIGTERM`, then `SIGKILL` after a grace window) is never actually exercised: the code that + * sends the second signal would be dead and nothing would notice. + */ + readonly ignoresSigterm?: boolean; + readonly onSpawn?: (args: readonly string[]) => void | Promise; +} + +export class ScriptedProcessRunner implements ProcessRunner { + readonly calls: ProcessInvocation[] = []; + /** + * why: The handle each `spawn` returned, so a test can assert what the caller did *with* + * the child rather than only how it was started. `unref` is the case + * that needs it: nothing about the argv or the options says whether the event loop was + * released, and a spawned child that stays referenced keeps the daemon alive after + * `daemon stop`. + */ + readonly handles: (ProcessHandle & { readonly unreffed: boolean })[] = []; + #nextPid = 1; + readonly #expectations: ScriptedProcessExpectation[]; + + constructor(expectations: readonly ScriptedProcessExpectation[]) { + this.#expectations = [...expectations]; + } + + async run( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): Promise { + return this.spawn(command, args, options).wait(); + } + + spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { + const invocation = { args: [...args], command, options }; + this.calls.push(invocation); + const expectation = this.#expectations.shift(); + + if (expectation === undefined || !matches(expectation.match, invocation)) { + throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); + } + + void expectation.onSpawn?.([...args]); + const handle = new ScriptedProcessHandle( + this.#nextPid++, + expectation, + options.lineEnd ?? "newline", + ); + this.handles.push(handle); + return handle; + } + + spawnStreaming( + command: string, + args: readonly string[], + options: ProcessStreamOptions, + ): StreamingProcessHandle { + const invocation = { + args: [...args], + command, + // why: Recorded as a `ProcessRunOptions` so `calls` stays one list a test can read uniformly; + // `onChunk` is a callback, not configuration, and nothing would assert on it. + options: { + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(options.env === undefined ? {} : { env: options.env }), + ...(options.input === undefined ? {} : { input: options.input }), + }, + }; + this.calls.push(invocation); + const expectation = this.#expectations.shift(); + + if (expectation === undefined || !matches(expectation.match, invocation)) { + throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); + } + + return new ScriptedStreamingProcessHandle( + this.#nextPid++, + expectation, + options.onChunk, + options.inputStream, + ); + } +} + +/** + * why: The scripted counterpart of `NodeStreamingProcessHandle`: emits the expectation's chunks + * synchronously (so a test needs no timers to see them), then settles unless the expectation + * `hangs` -- in which case only `kill()` settles it, reporting the signal the way a real + * killed child does. That is what makes the daemon's exec timeout testable without a real + * process or a real clock. + */ +class ScriptedStreamingProcessHandle implements StreamingProcessHandle { + inputFailed?: Promise; + readonly #result: Promise; + #resolve!: (result: StreamingProcessResult) => void; + #finished = false; + + readonly #ignoresSigterm: boolean; + + constructor( + readonly pid: number, + expectation: ScriptedProcessExpectation, + onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, + inputStream?: ReadableStream, + ) { + this.#ignoresSigterm = expectation.ignoresSigterm ?? false; + if (inputStream !== undefined) this.inputFailed = scriptedInputFailure(inputStream); + this.#result = new Promise((resolve) => { + this.#resolve = resolve; + }); + void this.#deliver(expectation, onChunk); + } + + /** + * why: One chunk at a time, awaiting each delivery before the next -- the same discipline + * `NodeStreamingProcessHandle` gets by pausing a readable, so a test that stalls a delivery + * sees the scripted child stall too rather than racing ahead of it. The command settles only + * once its output is delivered, which keeps "output, then exit code" true here as well. + */ + async #deliver( + expectation: ScriptedProcessExpectation, + onChunk: (stream: "stdout" | "stderr", chunk: string) => void | Promise, + ): Promise { + for (const { chunk, stream } of scriptedChunks(expectation)) { + if (this.#finished) return; + await onChunk(stream, chunk); + } + if (!expectation.hangs) { + this.#finish({ code: expectation.result?.code ?? 0, signal: null }); + } + } + + kill(signal: NodeJS.Signals = "SIGTERM"): void { + if (this.#ignoresSigterm && signal !== "SIGKILL") return; + this.#finish({ code: null, signal }); + } + + wait(): Promise { + return this.#result; + } + + #finish(result: StreamingProcessResult): void { + if (this.#finished) return; + this.#finished = true; + this.#resolve(result); + } +} + +/** why: the scripted child reads its whole input at once, so a test sees a failing stream reject `inputFailed` as the real adapter does. */ +function scriptedInputFailure(source: ReadableStream): Promise { + const failed = new Promise((_resolve, reject) => { + void (async () => { + const reader = source.getReader(); + let read = 0; + try { + for (;;) { + const result = await reader.read(); + if (result.done) return; + read += result.value.byteLength; + } + } catch (error) { + reject(new InputStreamFailedError(read, error)); + } + })(); + }); + failed.catch(() => undefined); + return failed; +} + +function scriptedChunks( + expectation: ScriptedProcessExpectation, +): readonly { readonly stream: "stdout" | "stderr"; readonly chunk: string }[] { + if (expectation.chunks !== undefined) return expectation.chunks; + return [ + ...(expectation.stdoutLines ?? []).map((line) => ({ + chunk: `${line}\n`, + stream: "stdout" as const, + })), + ...(expectation.stderrLines ?? []).map((line) => ({ + chunk: `${line}\n`, + stream: "stderr" as const, + })), + ]; +} + +class ScriptedProcessHandle implements ProcessHandle { + unreffed = false; + readonly stdout = new LineBuffer(); + readonly stderr = new LineBuffer(); + readonly #result: Promise; + #resolve!: (result: ProcessResult) => void; + #finished = false; + + constructor( + readonly pid: number, + private readonly expectation: ScriptedProcessExpectation, + lineEnd: LineEnd, + ) { + this.#result = new Promise((resolve) => { + this.#resolve = resolve; + }); + if (expectation.chunks === undefined) { + this.#writeLines(expectation.stdoutLines, this.stdout, lineEnd); + this.#writeLines(expectation.stderrLines, this.stderr, lineEnd); + } else { + this.#writeChunks(expectation.chunks, lineEnd); + } + + if (!expectation.hangs) { + this.#finish(expectation.result ?? defaultResult(expectation)); + } + } + + kill(signal: NodeJS.Signals = "SIGTERM"): void { + if (this.expectation.ignoresSigterm === true && signal !== "SIGKILL") return; + this.#finish({ code: null, stderr: "", stdout: "" }); + } + + unref(): void { + this.unreffed = true; + } + + wait(): Promise { + return this.#result; + } + + /** + * why: Each scripted line is replayed as the child would have printed it, `\n`-terminated, through + * the same splitter the real handle uses -- so a scripted line holding `\r` redraws splits + * exactly as real output would under the caller's `lineEnd`. + */ + #writeLines( + lines: readonly string[] | undefined, + destination: LineBuffer, + lineEnd: LineEnd, + ): void { + const splitter = new LineSplitter(lineEnd); + for (const line of lines ?? []) { + for (const split of splitter.feed(`${line}\n`)) destination.push(split); + } + } + + #writeChunks(chunks: NonNullable, lineEnd: LineEnd): void { + const splitters = { stderr: new LineSplitter(lineEnd), stdout: new LineSplitter(lineEnd) }; + for (const { chunk, stream } of chunks) { + for (const line of splitters[stream].feed(chunk)) this[stream].push(line); + } + for (const stream of ["stdout", "stderr"] as const) { + for (const line of splitters[stream].end()) this[stream].push(line); + } + } + + #finish(result: ProcessResult): void { + if (this.#finished) { + return; + } + + this.#finished = true; + this.stdout.close(); + this.stderr.close(); + this.#resolve(result); + } +} + +function matches(matcher: ProcessMatcher, invocation: ProcessInvocation): boolean { + return ( + matchesPart(matcher.command, invocation.command) && + matcher.args.length === invocation.args.length && + matcher.args.every((argument, index) => matchesPart(argument, invocation.args[index] ?? "")) + ); +} + +function matchesPart(matcher: string | RegExp, value: string): boolean { + return typeof matcher === "string" ? matcher === value : matcher.test(value); +} + +function defaultResult(expectation: ScriptedProcessExpectation): ProcessResult { + if (expectation.chunks !== undefined) { + const text = (stream: "stdout" | "stderr"): string => + expectation.chunks + ?.filter((chunk) => chunk.stream === stream) + .map(({ chunk }) => chunk) + .join("") ?? ""; + return { code: 0, stderr: text("stderr"), stdout: text("stdout") }; + } + return { + code: 0, + stderr: (expectation.stderrLines ?? []).join("\n"), + stdout: (expectation.stdoutLines ?? []).join("\n"), + }; +} From 60321517335850125b08da7579749fec45a089ab Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:03:34 +0100 Subject: [PATCH 02/83] refactor(drivers): split android testing.ts into test-* parts --- .dependency-cruiser.mjs | 5 +- src/drivers/android/test-driver-harness.ts | 230 ++++ src/drivers/android/test-emulator-host.ts | 205 ++++ src/drivers/android/test-sdk-fixtures.ts | 270 +++++ src/drivers/android/test-seed-harness.ts | 566 +++++++++ src/drivers/android/testing.ts | 1278 +------------------- 6 files changed, 1341 insertions(+), 1213 deletions(-) create mode 100644 src/drivers/android/test-driver-harness.ts create mode 100644 src/drivers/android/test-emulator-host.ts create mode 100644 src/drivers/android/test-sdk-fixtures.ts create mode 100644 src/drivers/android/test-seed-harness.ts diff --git a/.dependency-cruiser.mjs b/.dependency-cruiser.mjs index ae5353ad7..6085f060b 100644 --- a/.dependency-cruiser.mjs +++ b/.dependency-cruiser.mjs @@ -68,9 +68,10 @@ export default { }, { name: "testing-only-in-tests", - comment: "only a *.test.ts file or a test-*.ts helper imports a testing.ts or test-*.ts file", + comment: + "only a *.test.ts file, a test-*.ts helper or a testing.ts entry imports a testing.ts or test-*.ts file", severity: "error", - from: { pathNot: testFile }, + from: { pathNot: linkTestFile }, to: { path: "(^|/)(testing|test-[^/]+)\\.tsx?$" }, }, { diff --git a/src/drivers/android/test-driver-harness.ts b/src/drivers/android/test-driver-harness.ts new file mode 100644 index 000000000..0d42862b6 --- /dev/null +++ b/src/drivers/android/test-driver-harness.ts @@ -0,0 +1,230 @@ +import { + MemoryFilesystem, + ScriptedProcessRunner, + type ProcessResult, + type ProcessRunner, + type ScriptedProcessExpectation, + FakeClock, + FakeProcessSupervisor, + FakeTcpListenerFactory, + FakeTcpProbe, + FakeTcpStreamConnector, + type Filesystem, +} from "../../ports/index.js"; +import { type DeviceProfileSource } from "./device-profile-source.js"; +import { + AndroidDriver, + type AndroidDriverDiagnostic, + type AndroidEmulatorLaunchOptions, +} from "./index.js"; +import { + sdk, + home, + memoryAndroidSourceDatabase, + processResult, + pixelDevices, + simlockHome, + instanceId, + avdDirectory, + binaries, +} from "./test-sdk-fixtures.js"; +import { + EmulatorExitingRunner, + makingTheSeed, + writeSeedFiles, + androidFilesystem, + copyBootExpectations, + markWriteExpectation, +} from "./test-seed-harness.js"; + +export function bootProbes(runner: ScriptedProcessRunner): number { + return runner.calls.filter((call) => call.args.includes("sys.boot_completed")).length; +} + +export const adbServerPort = 5038; + +export const adbServerPid = 4242; + +export const adbRecordPath = `${simlockHome}/adb-server.json`; + +export const scopedEnv = { + ANDROID_ADB_SERVER_PORT: String(adbServerPort), + ANDROID_AVD_HOME: avdDirectory, + ANDROID_HOME: sdk, +} as const; + +export const PHYSICAL_DEVICE = { + address: "R58M123ABC", + deviceId: "R58M123ABC", + driverData: {}, + enrolledApps: ["com.example.kept"], + physical: true, +} as const; + +/** why: the arguments of every emulator start, in order. */ +export function emulatorLaunchArgs(runner: ScriptedProcessRunner): (readonly string[])[] { + return runner.calls + .filter((call) => call.command === binaries.emulator && call.args.includes("-avd")) + .map((call) => call.args); +} + +/** + * why: a driver and the copy it provisioned: the seed is made (scripted) and copied, so what a test + * scripts after `afterwards` is about that copy. The seed's id is the second the generator gives + * and the copy's the first, so a copy's folder is `/simlock_one/` and the mark token of its + * first boot is the third id, `device-3`. + */ +// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. +export async function provisionedHarness( + options: { + readonly bootCompleted?: string; + readonly config?: string; + readonly emulator?: AndroidEmulatorLaunchOptions; + readonly filesystem?: MemoryFilesystem; + readonly emulatorFlags?: readonly string[]; + readonly ids?: readonly string[]; + readonly initialAdbFailures?: number; + readonly tcpListener?: FakeTcpListenerFactory; + readonly unsettledPmAnswers?: readonly ProcessResult[]; + readonly bootTimeoutMs?: number; + readonly firstMarkToken?: string; + readonly afterwards?: readonly ScriptedProcessExpectation[]; + } = {}, +) { + const filesystem = await androidFilesystem({}, options.filesystem); + const ids = options.ids ?? ["one", "two"]; + const seedHome = `${avdDirectory}/sources/seed_${ids[1] ?? "two"}`; + const copyHome = `${avdDirectory}/simlock_${ids[0] ?? "one"}`; + await writeSeedFiles(filesystem, seedHome, options.config); + const tcpProbe = new FakeTcpProbe([adbServerPort]); + const clock = new FakeClock(); + const expectations: ScriptedProcessExpectation[] = [ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...makingTheSeed(seedHome, "Android emulator version 36.1.9", options.emulatorFlags), + ...copyBootExpectations({ ...options, copyHome }), + ]; + if ((options.bootCompleted ?? "1\n").trim() === "1") { + expectations.push(markWriteExpectation("emulator-5586", options.firstMarkToken ?? "device-3")); + } + expectations.push(...(options.afterwards ?? [])); + const runner = new EmulatorExitingRunner(expectations); + const driver = await createDriver(filesystem, runner, { + clock, + ...(options.emulator === undefined ? {} : { emulator: options.emulator }), + ids, + ...(options.tcpListener === undefined ? {} : { tcpListener: options.tcpListener }), + ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), + tcpProbe, + }); + const spec = await driver.resolveSpec({ + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + const device = await driver.provision(spec); + + return { clock, copyHome, device, driver, filesystem, runner, seedHome, tcpProbe }; +} + +export async function createDriver( + filesystem: Filesystem, + processRunner: ProcessRunner, + options: { + readonly acceptAndroidLicenses?: boolean; + /** why: `ANDROID_AVD_HOME` in the environment the daemon started with; unset by default. */ + readonly avdHome?: string; + readonly clock?: FakeClock; + /** why: a second driver over the same table is the daemon after a restart. */ + readonly database?: Awaited>; + readonly deviceProfileSources?: readonly DeviceProfileSource[]; + readonly driverConfig?: Readonly>; + readonly emulator?: AndroidEmulatorLaunchOptions; + readonly extraEnv?: Readonly>; + /** why: the ABI this host runs natively; `arm64-v8a` unless a test says otherwise. */ + readonly hostAbi?: string; + readonly ids?: readonly string[]; + readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; + readonly bootTimeoutMs?: number; + readonly scrcpyServerPath?: string; + readonly tcpListener?: FakeTcpListenerFactory; + readonly tcpProbe?: FakeTcpProbe; + readonly tcpStreamConnector?: FakeTcpStreamConnector; + } = {}, +) { + let nextId = 0; + const driverConfig = options.driverConfig ?? {}; + const configuredPort = + typeof driverConfig["adbServerPort"] === "number" + ? driverConfig["adbServerPort"] + : adbServerPort; + await recordRunningAdbServer(filesystem, configuredPort); + return AndroidDriver.create({ + ...onlyProvided(options), + clock: options.clock ?? new FakeClock(), + database: options.database ?? (await memoryAndroidSourceDatabase()), + driverConfig, + env: { + ANDROID_HOME: sdk, + ...options.extraEnv, + ...(options.avdHome === undefined ? {} : { ANDROID_AVD_HOME: options.avdHome }), + }, + filesystem, + homeDirectory: home, + hostAbi: options.hostAbi ?? "arm64-v8a", + idGenerator: { + generate: () => options.ids?.[nextId++] ?? `device-${nextId}`, + }, + instanceId, + processRunner, + processSupervisor: new FakeProcessSupervisor([adbServerPid]), + simlockHome, + tcpProbe: options.tcpProbe ?? new FakeTcpProbe([configuredPort]), + }); +} + +/** why: the options this helper only forwards when a test actually set one -- `AndroidDriver.create` + * declares them optional under `exactOptionalPropertyTypes`, so an absent one must be absent + * rather than `undefined`. Split out of `createDriver` so that the list can keep growing + * without the helper itself turning into a pile of conditionals. */ +function onlyProvided(options: { + readonly acceptAndroidLicenses?: boolean; + readonly deviceProfileSources?: readonly DeviceProfileSource[]; + readonly emulator?: AndroidEmulatorLaunchOptions; + readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; + readonly bootTimeoutMs?: number; + readonly scrcpyServerPath?: string; + readonly tcpListener?: FakeTcpListenerFactory; + readonly tcpStreamConnector?: FakeTcpStreamConnector; +}) { + return { + ...(options.tcpListener === undefined ? {} : { tcpListener: options.tcpListener }), + ...(options.scrcpyServerPath === undefined + ? {} + : { scrcpyServerPath: options.scrcpyServerPath }), + ...(options.tcpStreamConnector === undefined + ? {} + : { tcpStreamConnector: options.tcpStreamConnector }), + ...(options.acceptAndroidLicenses === undefined + ? {} + : { acceptAndroidLicenses: options.acceptAndroidLicenses }), + ...(options.deviceProfileSources === undefined + ? {} + : { deviceProfileSources: options.deviceProfileSources }), + config: { + ...(options.emulator === undefined ? {} : { emulator: options.emulator }), + ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), + }, + ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }), + }; +} + +export async function recordRunningAdbServer( + filesystem: Filesystem, + port = adbServerPort, +): Promise { + await filesystem.mkdirp(simlockHome); + await filesystem.writeFileAtomic( + adbRecordPath, + JSON.stringify({ pid: adbServerPid, port, startedAt: 1 }), + ); +} diff --git a/src/drivers/android/test-emulator-host.ts b/src/drivers/android/test-emulator-host.ts new file mode 100644 index 000000000..be8194e63 --- /dev/null +++ b/src/drivers/android/test-emulator-host.ts @@ -0,0 +1,205 @@ +import { + MemoryFilesystem, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ScriptedProcessExpectation, +} from "../../ports/index.js"; +import { basename } from "node:path"; +import { processResult, pixelDevices, avdDirectory, binaries, ok } from "./test-sdk-fixtures.js"; +import { emulatorHelpListing, writeSeedFiles } from "./test-seed-harness.js"; + +/** + * why: The emulators on one machine, which outlive any one daemon: as much of `avdmanager`, + * `emulator` and `adb` as provisioning, booting, saving a snapshot and stopping use. Each + * `runner()` is one daemon process's view of it -- the driver keeps its port reservations per + * runner, as a real daemon keeps them per process. + * + * Two behaviours are the real tools', not Simlock's. An emulator told to take a console port + * another one holds exits at once: `emulator -help` (36.1.9) says that "if any of these ports + * is already used, the emulator will fail to start". And a serial is answered by whichever + * emulator is listening on its port. + */ +export class EmulatorHost { + /** why: the AVD folder each console port's emulator runs, as its `emu avd path` answers it. */ + readonly #listening = new Map(); + readonly #exits = new Map void>(); + #nextPid = 1; + + constructor(private readonly filesystem: MemoryFilesystem) {} + + pathAnswering(serial: string | undefined): string | undefined { + return this.#listening.get(consolePortOf(serial)); + } + + /** why: the id of the device whose AVD answers: a copy's folder name, or an old-layout AVD's name. */ + avdAnswering(serial: string | undefined): string | undefined { + const folder = this.#listening.get(consolePortOf(serial)); + if (folder === undefined) return undefined; + const home = folder.slice(0, folder.lastIndexOf("/")); + return home === avdDirectory ? basename(folder, ".avd") : basename(home); + } + + runner(): ProcessRunner { + return { + run: (command, args, options) => this.#run(command, args, options?.env), + spawn: (command, args, options) => this.#spawn(command, args, options?.env), + spawnStreaming: (command, args) => { + throw new Error(`Unexpected streaming invocation: ${command} ${args.join(" ")}`); + }, + }; + } + + async #run( + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv | undefined, + ): Promise { + const result = + command === binaries.avdmanager + ? await this.#avdmanager(args, env?.["ANDROID_AVD_HOME"] ?? avdDirectory) + : command === binaries.adb + ? this.#adb(args) + : command === binaries.emulator + ? emulatorQuery(args) + : undefined; + if (result === undefined) { + throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); + } + return result; + } + + async #avdmanager(args: readonly string[], home: string): Promise { + if (args[0] === "list") return ok(pixelDevices); + const avdName = args[args.indexOf("-n") + 1] ?? ""; + if (args[0] === "delete") { + await this.filesystem.rm(`${home}/${avdName}.avd`); + return ok(); + } + if (args[0] !== "create") return undefined; + await writeSeedFiles(this.filesystem, home, "hw.ramSize=2048\n", avdName); + return ok(); + } + + // fallow-ignore-next-line complexity -- one fake adb: each branch answers one command of the table. + #adb(args: readonly string[]): ProcessResult | undefined { + if (args[0] === "devices") { + const lines = [...this.#listening.keys()].map((port) => `emulator-${port}\tdevice\n`); + return ok(`List of devices attached\n${lines.join("")}`); + } + if (args[0] !== "-s") return undefined; + const port = consolePortOf(args[1]); + if (!this.#listening.has(port)) { + return { code: 1, stderr: `adb: device '${args[1]}' not found`, stdout: "" }; + } + const rest = args.slice(2).join(" "); + if (rest === "emu kill" || rest === "shell reboot -p") { + this.#exit(port); + return ok("OK: killing emulator, bye bye\n"); + } + if (rest.startsWith("shell echo ")) return ok(); + if (rest.startsWith("shell getprop ro.boot.qemu.avd_name; head -c ")) { + return ok(`${basename(this.#listening.get(port) ?? "", ".avd")}\n`); + } + if (rest === "emu avd path") { + return ok(`${this.#listening.get(port) ?? ""}\r\nOK\r\n`); + } + const answer = ADB_ANSWERS.get(rest); + return answer === undefined ? undefined : ok(answer); + } + + #spawn( + command: string, + args: readonly string[], + env: NodeJS.ProcessEnv | undefined, + ): ProcessHandle { + if (command !== binaries.emulator || args[0] !== "-avd") { + throw new Error(`Unexpected spawn: ${command} ${args.join(" ")}`); + } + const avdName = `${env?.["ANDROID_AVD_HOME"] ?? avdDirectory}/${args[1] ?? ""}.avd`; + const port = Number(args[args.indexOf("-port") + 1]); + let exit!: (result: ProcessResult) => void; + const exited = new Promise((resolve) => { + exit = resolve; + }); + const bound = !this.#listening.has(port); + if (bound) { + this.#listening.set(port, avdName); + this.#exits.set(port, () => exit({ code: 0, stderr: "", stdout: "" })); + } else { + exit({ code: 1, stderr: `console port ${port} is already used`, stdout: "" }); + } + return { + // why: Killing an emulator that never bound must not stop the one that did. + kill: () => { + if (bound && this.#listening.get(port) === avdName) this.#exit(port); + }, + pid: this.#nextPid++, + stderr: emptyLines(), + stdout: emptyLines(), + unref: () => undefined, + wait: () => exited, + }; + } + + #exit(port: number): void { + this.#listening.delete(port); + this.#exits.get(port)?.(); + this.#exits.delete(port); + } +} + +export class RecordingRunner implements ProcessRunner { + readonly calls: { args: readonly string[]; command: string; options: ProcessRunOptions }[] = []; + + constructor(private readonly inner: ProcessRunner) {} + + run(command: string, args: readonly string[], options: ProcessRunOptions = {}) { + this.calls.push({ args, command, options }); + return this.inner.run(command, args, options); + } + + spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}) { + this.calls.push({ args, command, options }); + return this.inner.spawn(command, args, options); + } + + spawnStreaming(...args: Parameters) { + return this.inner.spawnStreaming(...args); + } +} + +const ADB_ANSWERS = new Map([ + ["shell getprop sys.boot_completed", "1\n"], + ["shell getprop init.svc.bootanim", "stopped\n"], + ["shell pm path android", "package:/system/framework/framework-res.apk\n"], + ["shell sync", ""], + ["emu avd snapshot save simlock_boot", "OK\n"], + ["emu avd snapshot list", "simlock_boot\n"], +]); + +export function consolePortOf(serial: string | undefined): number { + return Number(/^emulator-(\d+)$/.exec(serial ?? "")?.[1]); +} + +export async function* emptyLines(): AsyncIterable {} + +/** why: what a running copy answers to `emu avd path`: the AVD inside its own folder. */ +export function copyPathExpectation( + serial: string, + folder = "simlock_one", +): ScriptedProcessExpectation { + return processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${avdDirectory}/${folder}/simlock.avd\r\nOK\r\n`, + ); +} + +/** why: the two read-only `emulator` calls the fake answers; any other is unexpected. */ +function emulatorQuery(args: readonly string[]): ProcessResult | undefined { + if (args[0] === "-version") return ok("Android emulator version 36.1.9"); + if (args[0] === "-help") return ok(emulatorHelpListing); + return undefined; +} diff --git a/src/drivers/android/test-sdk-fixtures.ts b/src/drivers/android/test-sdk-fixtures.ts new file mode 100644 index 000000000..be4f12291 --- /dev/null +++ b/src/drivers/android/test-sdk-fixtures.ts @@ -0,0 +1,270 @@ +import { + MemoryFilesystem, + NodeSqliteOpener, + ScriptedProcessRunner, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ScriptedProcessExpectation, + type SqliteDatabase, +} from "../../ports/index.js"; +import { type LibraryDeviceProfile } from "./profiles/index.js"; +import { androidSourceDataOwner } from "./source-data.js"; + +export const ANDROID_SDK_ROOT = "/android-sdk"; +export const ANDROID_HOME_DIRECTORY = "/home/simlock"; +export const sdk = ANDROID_SDK_ROOT; +export const home = ANDROID_HOME_DIRECTORY; + +export async function memoryAndroidSourceDatabase(): Promise { + const database = await new NodeSqliteOpener().open(":memory:", { readOnly: false }); + for (const migration of androidSourceDataOwner.migrations) database.exec(migration); + return database; +} + +export function processResult(command: string, args: readonly (string | RegExp)[], stdout = "") { + return { + match: { args, command }, + result: { code: 0, stderr: "", stdout }, + }; +} + +function imageDirectory(packageName: string | undefined): string | undefined { + const match = /^system-images;android-(.+);(.+);(.+)$/.exec(packageName ?? ""); + if (match === null) return undefined; + const [, api, tag, abi] = match; + return `${ANDROID_SDK_ROOT}/system-images/android-${api ?? ""}/${tag ?? ""}/${abi ?? ""}`; +} + +/** + * why: An AVD in the user's own AVD home, as `avdmanager` writes it: a pointer file beside a folder + * whose `config.ini` names the system image (`image.sysdir.1`). + */ +export async function userAvd( + filesystem: MemoryFilesystem, + name: string, + sysdir: string, + options: { readonly avdHome?: string; readonly avdPath?: string } = {}, +): Promise { + const avdHome = options.avdHome ?? `${ANDROID_HOME_DIRECTORY}/.android/avd`; + const avdPath = options.avdPath ?? `${avdHome}/${name}.avd`; + await filesystem.mkdirp(avdPath); + await filesystem.writeFileAtomic( + `${avdHome}/${name}.ini`, + `avd.ini.encoding=UTF-8\npath=${avdPath}\n`, + ); + await filesystem.writeFileAtomic( + `${avdPath}/config.ini`, + `hw.ramSize=2048\nimage.sysdir.1=${sysdir}\ntag.id=google_apis\n`, + ); +} + +/** + * why: A `ScriptedProcessRunner` that also mirrors what real `sdkmanager` does on disk: a + * successful (`code: 0`, no unaccepted-license text) `--install ` call creates the + * corresponding `system-images/android-//` directory in `filesystem`, and a + * successful `--uninstall` removes it. A bare `ScriptedProcessRunner` only scripts the process's own + * stdout/stderr/exit code, never a filesystem side effect, so the scan after an install would not + * find the image it just "installed". + */ +export class InstallReflectingProcessRunner extends ScriptedProcessRunner { + readonly #filesystem: MemoryFilesystem; + + constructor(expectations: readonly ScriptedProcessExpectation[], filesystem: MemoryFilesystem) { + super(expectations); + this.#filesystem = filesystem; + } + + override spawn( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): ProcessHandle { + const handle = super.spawn(command, args, options); + const uninstalled = args[0] === "--uninstall" ? imageDirectory(args[1]) : undefined; + if (uninstalled !== undefined) { + void handle.wait().then(async (result) => { + if (result.code === 0) await this.#filesystem.rm(uninstalled); + }); + } + if (args[0] === "--install" && typeof args[1] === "string") { + const packageName = args[1]; + void handle.wait().then((result) => { + const combined = `${result.stdout}\n${result.stderr}`; + const licenseNotAccepted = + /licen[cs]e/i.test(combined) && /not (?:been )?accepted/i.test(combined); + if (result.code === 0 && !licenseNotAccepted) { + const match = /^system-images;android-(.+);(.+);(.+)$/.exec(packageName); + if (match !== null) { + const [, api, tag, abi] = match; + const imagePath = `${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}`; + void this.#filesystem + .mkdirp(imagePath) + .then(() => + this.#filesystem.writeFileAtomic( + `${imagePath}/source.properties`, + "Pkg.Revision=1\n", + ), + ); + } + } + }); + } + return handle; + } +} + +export const pixelDevices = `Available devices:\nid: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n`; +export const twoPixelDevices = + `Available devices:\nid: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n` + + `---------\nid: 1 or "pixel_9"\n Name: Pixel 9\n OEM : Google\n`; + +export async function writeDevicesXml( + filesystem: MemoryFilesystem, + deviceBodies: string, +): Promise { + await filesystem.mkdirp(`${ANDROID_HOME_DIRECTORY}/.android`); + await filesystem.writeFileAtomic( + `${ANDROID_HOME_DIRECTORY}/.android/devices.xml`, + `${deviceBodies}`, + ); +} + +export function customDeviceXml(name: string, ramMiB: number): string { + return ( + `${name}` + + `${ramMiB}` + ); +} + +export type FitImage = readonly [api: string, tag: string, abi: string, properties?: string]; + +export const fitDevices = + `Available devices:\n` + + `id: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n---------\n` + + `id: 1 or "pixel_9_pro"\n Name: Pixel 9 Pro\n OEM : Google\n---------\n` + + `id: 2 or "tv_720p"\n Name: Television (720p)\n OEM : Google\n Tag : android-tv\n---------\n` + + `id: 3 or "My Tablet"\n Name: My Tablet\n OEM : User\n---------\n` + + `id: 4 or "odd"\n Name: Odd Device\n OEM : Google\n Tag : foo-device\n---------\n` + + `id: 5 or "xr_headset"\n Name: XR Headset\n OEM : Google\n Tag : android-xr\n---------\n` + + `id: 6 or "glasses"\n Name: AI Glasses\n OEM : Google\n Tag : ai-glasses\n`; + +export type Refusal = Error & { readonly component?: string; readonly downloadable?: boolean }; + +export const HOST_IMAGES: readonly FitImage[] = [ + ["35", "google_apis", "arm64-v8a"], + ["36", "google_apis_playstore", "arm64-v8a"], + ["34", "android-tv", "arm64-v8a"], +]; + +export function testLibraryProfile( + overrides: Partial = {}, +): LibraryDeviceProfile { + return { + apiRange: { min: 1 }, + bootProperties: {}, + copiedFrom: "20.0", + deviceClass: "phone", + hardware: { "hw.lcd.height": "100", "hw.ramSize": "2G" }, + id: "test_phone", + imageKind: undefined, + kind: "library", + name: "Test Phone", + names: ["Test Phone", "test_phone"], + playStore: true, + tablet: false, + ...overrides, + }; +} + +export interface CustomFitProfile { + readonly apiLevel?: string; + readonly diagonal?: string; + readonly folded?: boolean; + readonly name: string; + readonly playStore?: string; + readonly tagId?: string; +} + +export function customFitXml(profile: CustomFitProfile): string { + const screen = + profile.diagonal === undefined && profile.folded !== true + ? "" + : `${profile.diagonal === undefined ? "" : `${profile.diagonal}`}` + + (profile.folded === true + ? "10802400" + : "") + + ""; + return ( + `${profile.name}` + + (profile.tagId === undefined ? "" : `${profile.tagId}`) + + `${screen}` + + (profile.apiLevel === undefined + ? "" + : `${profile.apiLevel}`) + + (profile.playStore === undefined + ? "" + : `${profile.playStore}`) + + "" + ); +} + +export function userListing( + ...entries: readonly (readonly [name: string, tag?: string])[] +): string { + return ( + "Available devices:\n" + + entries + .map( + ([name, tag]) => + `id: 9 or "${name}"\n Name: ${name}\n OEM : User\n` + + (tag === undefined ? "" : ` Tag : ${tag}\n`) + + "---------\n", + ) + .join("") + ); +} + +/** why: answers every `adb -s emu avd path` with `answer` and leaves each other call to `inner`, so a test lies about where an emulator runs. */ +export function answeringAvdPath( + adb: string, + inner: ProcessRunner, + answer: ProcessResult, +): ProcessRunner { + return { + run: async (command, args, options) => + command === adb && args[0] === "-s" && args.slice(2).join(" ") === "emu avd path" + ? answer + : inner.run(command, args, options), + spawn: (command, args, options) => inner.spawn(command, args, options), + spawnStreaming: (command, args, options) => inner.spawnStreaming(command, args, options), + }; +} + +export const simlockHome = "/home/simlock/.simlock"; + +export const instanceId = "instance-1"; + +export const avdDirectory = `${simlockHome}/devices/android`; + +export const binaries = { + adb: `${ANDROID_SDK_ROOT}/platform-tools/adb`, + avdmanager: `${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/avdmanager`, + emulator: `${ANDROID_SDK_ROOT}/emulator/emulator`, + sdkmanager: `${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager`, +} as const; + +export function ok(stdout = ""): ProcessResult { + return { code: 0, stderr: "", stdout }; +} + +export class WorkingDirectoryFilesystem extends MemoryFilesystem { + constructor(private readonly workingDirectory: string) { + super(); + } + + override async realpath(path: string): Promise { + return super.realpath(path.startsWith("/") ? path : `${this.workingDirectory}/${path}`); + } +} diff --git a/src/drivers/android/test-seed-harness.ts b/src/drivers/android/test-seed-harness.ts new file mode 100644 index 000000000..45246cb08 --- /dev/null +++ b/src/drivers/android/test-seed-harness.ts @@ -0,0 +1,566 @@ +import { + type DriverCatalogEntry, + type DriverDevice, + type ExactDeviceRequest, + OWNED_ROOT_MARKER_FILE, + type VirtualSpec, +} from "../../core/index.js"; +import { + MemoryFilesystem, + ScriptedProcessRunner, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ScriptedProcessExpectation, +} from "../../ports/index.js"; +import { type DeviceProfileSource } from "./device-profile-source.js"; +import { type LibraryDeviceProfile } from "./profiles/index.js"; +import { + ANDROID_SDK_ROOT, + processResult, + pixelDevices, + writeDevicesXml, + type FitImage, + fitDevices, + type Refusal, + instanceId, + avdDirectory, + binaries, + ok, +} from "./test-sdk-fixtures.js"; + +/** + * why: A scripted emulator stays up until it is told to stop, as a real one does, so a boot that + * sees it exit has failed; `emu kill` and the seed's `reboot -p` are what stop it. + */ +export class EmulatorExitingRunner extends ScriptedProcessRunner { + /** why: a test of an emulator that ignores `emu kill` sets this. */ + ignoresEmuKill = false; + + /** why: the flag probe is not one of the calls a boot test scripts, so it is answered here and never enters `calls` or the script's order. */ + override run( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): Promise { + if (command === binaries.emulator && args[0] === "-help") { + return Promise.resolve(ok(emulatorHelpListing)); + } + return super.run(command, args, options); + } + + override spawn( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): ProcessHandle { + const handle = super.spawn(command, args, options); + const stopsTheEmulator = /(?:emu kill|shell reboot -p)$/.test(args.join(" ")); + if (!this.ignoresEmuKill && stopsTheEmulator) { + for (const [index, call] of this.calls.entries()) { + if (call.command === binaries.emulator && call.args.includes("-avd")) { + this.handles[index]?.kill("SIGKILL"); + } + } + } + return handle; + } +} + +/** why: an emulator new enough to list every flag Simlock passes, `-crash-report-mode` included. */ +export const emulatorHelpListing = [ + "-avd", + "-port", + "-no-snapshot-save", + "-grpc", + "-grpc-use-token", + "-crash-report-mode", +] + .map((flag) => ` ${flag} described`) + .join("\n"); + +const defaultSeedHome = `${avdDirectory}/sources/seed_two`; + +/** why: the calls of making and booting a seed, shared by the seed tests and the harness every driver test starts from. */ +export const seedEmulatorLine = [ + "-port", + "5586", + "-no-snapshot-save", + "-grpc", + "8586", + "-grpc-use-token", + "-crash-report-mode", + "never", +]; + +export function bootingAt( + folder: string, + launch: readonly string[], + hangs: boolean, + notReadyPolls = 0, + flags: readonly string[] = [], +) { + const serial = "emulator-5586"; + const notReady = Array.from({ length: notReadyPolls }, () => + processResult(binaries.adb, ["-s", serial, "shell", "getprop", "sys.boot_completed"], "0\n"), + ); + return [ + { + hangs, + match: { + args: ["-avd", "simlock", ...seedEmulatorLine, ...flags, ...launch], + command: binaries.emulator, + }, + }, + ...notReady, + processResult(binaries.adb, ["-s", serial, "shell", "getprop", "sys.boot_completed"], "1\n"), + processResult(binaries.adb, ["-s", serial, "shell", "getprop", "init.svc.bootanim"], ""), + processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${folder}/simlock.avd\r\nOK\r\n`, + ), + processResult( + binaries.adb, + ["-s", serial, "shell", "pm", "path", "android"], + "package:/system/framework/framework-res.apk\n", + ), + ] satisfies ScriptedProcessExpectation[]; +} + +/** why: the calls of making a seed, by stage, so a test can stop it at the step that fails. */ +export function seedStages( + home = defaultSeedHome, + emulatorVersion = "Android emulator version 36.1.9", + flags: readonly string[] = [], +) { + const serial = "emulator-5586"; + return { + create: processResult(binaries.avdmanager, [ + "create", + "avd", + "-n", + "simlock", + "-k", + "system-images;android-34;google_apis;arm64-v8a", + "-d", + "pixel_5", + ]), + devices: processResult(binaries.adb, ["devices"], "List of devices attached\n"), + firstBoot: bootingAt(home, ["-no-snapshot-load"], true, 0, flags), + kill: processResult(binaries.adb, ["-s", serial, "emu", "kill"]), + list: processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "snapshot", "list"], + "simlock_boot\n", + ), + powerOff: processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), + probe: processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${home}/simlock.avd\r\nOK\r\n`, + ), + save: processResult(binaries.adb, [ + "-s", + serial, + "emu", + "avd", + "snapshot", + "save", + "simlock_boot", + ]), + secondBoot: bootingAt(home, ["-no-snapshot-load"], true, 0, flags), + sync: processResult(binaries.adb, ["-s", serial, "shell", "sync"]), + version: processResult(binaries.emulator, ["-version"], emulatorVersion), + }; +} + +export function makingTheSeed( + home = defaultSeedHome, + emulatorVersion = "Android emulator version 36.1.9", + flags: readonly string[] = [], +): ScriptedProcessExpectation[] { + const stage = seedStages(home, emulatorVersion, flags); + return [ + stage.version, + stage.create, + stage.devices, + ...stage.firstBoot, + stage.sync, + stage.powerOff, + ...stage.secondBoot, + stage.save, + stage.list, + stage.probe, + stage.kill, + ]; +} + +export async function writeSeedFiles( + filesystem: MemoryFilesystem, + home: string, + config = "hw.ramSize=2048\n", + avdName = "simlock", +): Promise { + await filesystem.mkdirp(`${home}/${avdName}.avd`); + await filesystem.writeFileAtomic(`${home}/${avdName}.avd/config.ini`, config); + await filesystem.writeFileAtomic( + `${home}/${avdName}.ini`, + `avd.ini.encoding=UTF-8\npath=${home}/${avdName}.avd\ntarget=android-34\n`, + ); +} + +export async function androidFilesystem( + options: { + readonly config?: string; + readonly freeDiskBytes?: number; + readonly images?: readonly (readonly [string, string, string])[]; + readonly withDeviceRoot?: boolean; + } = {}, + filesystem: MemoryFilesystem = new MemoryFilesystem(options.freeDiskBytes), +): Promise { + for (const binary of Object.values(binaries)) { + await filesystem.mkdirp(binary.slice(0, binary.lastIndexOf("/"))); + await filesystem.writeFileAtomic(binary, "binary"); + } + // why: Marked, so `ensureOwnedRoot` adopts it rather than creating one -- a root Simlock + // created in an earlier run is the ordinary case, and it keeps the id generator's first + // value available for the AVD name the tests assert on. + if (options.withDeviceRoot !== false) { + await filesystem.mkdirp(avdDirectory); + await filesystem.writeFileAtomic( + `${avdDirectory}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "android", schemaVersion: 1 }), + ); + } + for (const [api, tag, abi] of options.images ?? [["34", "google_apis", "arm64-v8a"]]) { + await filesystem.mkdirp(`${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}`); + } + if (options.config !== undefined) { + await filesystem.mkdirp(`${avdDirectory}/simlock_one.avd`); + await filesystem.writeFileAtomic(`${avdDirectory}/simlock_one.avd/config.ini`, options.config); + } + return filesystem; +} + +/** why: making a seed from whichever image and device profile the test's request resolves to; `device` pins the profile when the test cares. */ +export function makingASeedFrom( + home = defaultSeedHome, + device: string | RegExp = /.+/, +): ScriptedProcessExpectation[] { + const making = makingTheSeed(home); + making[1] = processResult(binaries.avdmanager, [ + "create", + "avd", + "-n", + "simlock", + "-k", + /.+/, + "-d", + device, + ]); + return making; +} + +/** + * why: the boot of a copy, as the driver asks the emulator for it: `adb devices` for the port, + * the launch, then the readiness polls. `launch` is `-snapshot simlock_boot` for a prepare boot. + */ +export function copyBootExpectations(options: { + readonly bootCompleted?: string | undefined; + readonly copyHome: string; + readonly emulatorFlags?: readonly string[] | undefined; + readonly initialAdbFailures?: number | undefined; + readonly launch?: readonly string[]; + readonly unsettledPmAnswers?: readonly ProcessResult[] | undefined; +}): ScriptedProcessExpectation[] { + const bootCompleted = options.bootCompleted ?? "1\n"; + const serial = "emulator-5586"; + const poll = (property: string, answer: string) => + processResult(binaries.adb, ["-s", serial, "shell", "getprop", property], answer); + const path = processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${options.copyHome}/simlock.avd\r\nOK\r\n`, + ); + const pm = (answer: ProcessResult): ScriptedProcessExpectation => ({ + match: { args: ["-s", serial, "shell", "pm", "path", "android"], command: binaries.adb }, + result: answer, + }); + const expectations: ScriptedProcessExpectation[] = [ + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + { + hangs: true, + match: { + args: [ + "-avd", + "simlock", + ...seedEmulatorLine, + ...(options.emulatorFlags ?? []), + ...(options.launch ?? ["-snapshot", "simlock_boot"]), + ], + command: binaries.emulator, + }, + }, + ]; + for (let failure = 0; failure < (options.initialAdbFailures ?? 0); failure += 1) { + expectations.push({ + match: { + args: ["-s", serial, "shell", "getprop", "sys.boot_completed"], + command: binaries.adb, + }, + result: { code: 1, stderr: "connection refused", stdout: "" }, + }); + } + expectations.push(poll("sys.boot_completed", bootCompleted)); + if (bootCompleted.trim() !== "1") return expectations; + for (const answer of options.unsettledPmAnswers ?? []) { + expectations.push( + poll("init.svc.bootanim", ""), + path, + pm(answer), + poll("sys.boot_completed", "1\n"), + ); + } + expectations.push( + poll("init.svc.bootanim", ""), + path, + pm({ code: 0, stderr: "", stdout: "package:/system/framework/framework-res.apk\n" }), + ); + return expectations; +} + +class ListingRunner extends EmulatorExitingRunner { + constructor( + private readonly listing: string, + expectations: readonly ScriptedProcessExpectation[], + ) { + super(expectations); + } + + override spawn( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): ProcessHandle { + if (command === binaries.avdmanager && args[0] === "list") { + return new EmulatorExitingRunner([processResult(command, args, this.listing)]).spawn( + command, + args, + options, + ); + } + return super.spawn(command, args, options); + } +} + +/** why: what the seed and library tests need of whatever makes a seed: a driver at the root, or the part with the catalog beside it. */ +export interface SeedProvisioner { + listCatalog(): Promise; + provision(spec: VirtualSpec): Promise; + resolveSpec(request: ExactDeviceRequest): Promise; +} + +export type BuildProvisioner = ( + filesystem: MemoryFilesystem, + processRunner: ProcessRunner, + options: { + readonly deviceProfileSources?: readonly DeviceProfileSource[]; + readonly extraEnv?: Readonly>; + readonly hostAbi?: string; + readonly ids: readonly string[]; + }, +) => Promise; + +export async function fitHarness( + build: BuildProvisioner, + images: readonly FitImage[], + options: { + readonly devicesXml?: string; + readonly extraEnv?: Readonly>; + readonly hostAbi?: string; + readonly listing?: string; + readonly profiles?: readonly LibraryDeviceProfile[]; + } = {}, +) { + const filesystem = await androidFilesystem({ + images: images.map(([api, tag, abi]) => [api, tag, abi]), + }); + for (const [api, tag, abi, properties] of images) { + if (properties !== undefined) { + await filesystem.writeFileAtomic( + `${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}/source.properties`, + properties, + ); + } + } + if (options.devicesXml !== undefined) await writeDevicesXml(filesystem, options.devicesXml); + await writeSeedFiles(filesystem, defaultSeedHome); + const runner = new ListingRunner(options.listing ?? fitDevices, makingASeedFrom()); + const driver = await build(filesystem, runner, { + hostAbi: options.hostAbi ?? "arm64-v8a", + ids: ["one", "two"], + ...(options.extraEnv === undefined ? {} : { extraEnv: options.extraEnv }), + ...(options.profiles === undefined + ? {} + : { deviceProfileSources: [{ profiles: async () => options.profiles ?? [] }] }), + }); + const resolve = (model: string, osVersion?: string, imageTag?: string) => + driver.resolveSpec({ + model, + platform: "android", + ...(osVersion === undefined ? {} : { osVersion }), + ...(imageTag === undefined ? {} : { imageTag }), + }); + const refusal = (model: string, osVersion?: string, imageTag?: string) => + resolve(model, osVersion, imageTag).then( + () => { + throw new Error(`${model} ${osVersion ?? ""} ${imageTag ?? ""} resolved`); + }, + (error: unknown) => error as Refusal, + ); + const createdPackage = () => { + const create = runner.calls.find((call) => call.args[0] === "create"); + return create?.args[create.args.indexOf("-k") + 1]; + }; + return { createdPackage, driver, filesystem, refusal, resolve, runner }; +} + +const SEED_LINES = [ + "avd.ini.encoding=UTF-8", + "hw.cpu.arch=arm64", + "hw.ramSize=1G", + "hw.lcd.density=1", + "hw.seedOnly=yes", + "image.sysdir.1=system-images/android-35/google_apis/arm64-v8a/", + "tag.id=google_apis", + "abi.type=arm64-v8a", + "target=android-35", + "disk.dataPartition.size=6G", + "showDeviceFrame=yes", + "skin.name=pixel_8", + "skin.path=skins/pixel_8", + "vm.heapSize=64M", + "PlayStore.enabled=no", +]; + +export function seedConfig(extra: readonly string[]): string { + return `${[...SEED_LINES.filter((line) => !extra.some((added) => added.split("=")[0] === line.split("=")[0])), ...extra].join("\n")}\n`; +} + +// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. +export async function libraryHarness( + build: BuildProvisioner, + options: { + readonly devicesXml?: string; + readonly extraImages?: readonly (readonly [string, string, string])[]; + readonly filesystem?: MemoryFilesystem; + readonly imageProperties?: string; + readonly imageTag?: string; + readonly listing?: string; + readonly profiles?: readonly LibraryDeviceProfile[]; + readonly seed?: string; + readonly seedBootProp?: string; + readonly sources?: readonly DeviceProfileSource[]; + } = {}, +) { + const tag = options.imageTag ?? "google_apis"; + const filesystem = options.filesystem ?? new MemoryFilesystem(); + await androidFilesystem( + { images: [["35", tag, "arm64-v8a"], ...(options.extraImages ?? [])] }, + filesystem, + ); + const imageProperties = options.imageProperties ?? `Pkg.Revision=1\nSystemImage.TagId=${tag}\n`; + if (imageProperties !== "") { + await filesystem.writeFileAtomic( + `${ANDROID_SDK_ROOT}/system-images/android-35/${tag}/arm64-v8a/source.properties`, + imageProperties, + ); + } + if (options.devicesXml !== undefined) { + await writeDevicesXml(filesystem, options.devicesXml); + } + // why: the profile is applied to the seed's AVD, which `avdmanager create` writes in the seed's own folder. + const seedFolder = `${avdDirectory}/sources/seed_two`; + const avd = `${seedFolder}/simlock.avd`; + const configPath = `${avd}/config.ini`; + const bootPropPath = `${avd}/boot.prop`; + await writeSeedFiles(filesystem, seedFolder, options.seed ?? seedConfig([])); + if (options.seedBootProp !== undefined) { + await filesystem.writeFileAtomic(bootPropPath, options.seedBootProp); + } + const listing = options.listing ?? pixelDevices; + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], listing), + ...makingASeedFrom(seedFolder), + ]); + const sources = + options.profiles === undefined + ? options.sources + : [{ profiles: async () => options.profiles ?? [] }]; + const driver = await build(filesystem, runner, { + ids: ["one", "two"], + ...(sources === undefined ? {} : { deviceProfileSources: sources }), + }); + const resolve = (model: string) => + driver.resolveSpec({ model, osVersion: "35", platform: "android" }); + const readLines = async () => + (await filesystem.readFile(configPath)).split("\n").filter((line) => line !== ""); + return { + bootPropPath, + config: async () => + new Map( + (await readLines()).map((line) => [ + line.slice(0, line.indexOf("=")), + line.slice(line.indexOf("=") + 1), + ]), + ), + configPath, + createCalls: () => runner.calls.filter((call) => call.args[0] === "create"), + driver, + filesystem, + lines: readLines, + provision: async (model: string) => driver.provision(await resolve(model)), + resolve, + runner, + seedDeviceId: () => { + const create = runner.calls.find((call) => call.args[0] === "create"); + return create?.args[create.args.indexOf("-d") + 1]; + }, + }; +} + +// why: the erasable mark a boot writes into the running device. +export function markWriteExpectation(serial: string, token: string): ScriptedProcessExpectation { + return processResult(binaries.adb, [ + "-s", + serial, + "shell", + `echo '${JSON.stringify({ token })}' > /data/local/tmp/simlock-mark.json`, + ]); +} + +export function createdFromPackage(runner: ScriptedProcessRunner): string | undefined { + const create = runner.calls.find( + (call) => call.command === binaries.avdmanager && call.args[0] === "create", + ); + return create?.args[create.args.indexOf("-k") + 1]; +} + +export async function taggedHarness( + build: BuildProvisioner, + images: readonly (readonly [string, string, string])[], + hostAbi = "arm64-v8a", +) { + const filesystem = await androidFilesystem({ images }); + await writeSeedFiles(filesystem, defaultSeedHome); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...makingASeedFrom(defaultSeedHome, "pixel_5"), + ...copyBootExpectations({ copyHome: `${avdDirectory}/simlock_one` }), + markWriteExpectation("emulator-5586", "device-3"), + ]); + const driver = await build(filesystem, runner, { hostAbi, ids: ["one", "two"] }); + return { driver, filesystem, runner }; +} diff --git a/src/drivers/android/testing.ts b/src/drivers/android/testing.ts index 80c5ae032..335c75204 100644 --- a/src/drivers/android/testing.ts +++ b/src/drivers/android/testing.ts @@ -1,1211 +1,67 @@ -import { - type DriverCatalogEntry, - type DriverDevice, - type ExactDeviceRequest, - OWNED_ROOT_MARKER_FILE, - type VirtualSpec, -} from "../../core/index.js"; -import { type DeviceProfileSource } from "./device-profile-source.js"; -import { - MemoryFilesystem, - NodeSqliteOpener, - ScriptedProcessRunner, - type ProcessHandle, - type ProcessResult, - type ProcessRunner, - type ProcessRunOptions, - type ScriptedProcessExpectation, - type SqliteDatabase, - FakeClock, - FakeProcessSupervisor, - FakeTcpListenerFactory, - FakeTcpProbe, - FakeTcpStreamConnector, - type Filesystem, -} from "../../ports/index.js"; -import { type LibraryDeviceProfile } from "./profiles/index.js"; -import { androidSourceDataOwner } from "./source-data.js"; -import { basename } from "node:path"; -import { - AndroidDriver, - type AndroidDriverDiagnostic, - type AndroidEmulatorLaunchOptions, -} from "./index.js"; - -export const ANDROID_SDK_ROOT = "/android-sdk"; -export const ANDROID_HOME_DIRECTORY = "/home/simlock"; -const sdk = ANDROID_SDK_ROOT; -const home = ANDROID_HOME_DIRECTORY; - -export async function memoryAndroidSourceDatabase(): Promise { - const database = await new NodeSqliteOpener().open(":memory:", { readOnly: false }); - for (const migration of androidSourceDataOwner.migrations) database.exec(migration); - return database; -} - -export function processResult(command: string, args: readonly (string | RegExp)[], stdout = "") { - return { - match: { args, command }, - result: { code: 0, stderr: "", stdout }, - }; -} - -function imageDirectory(packageName: string | undefined): string | undefined { - const match = /^system-images;android-(.+);(.+);(.+)$/.exec(packageName ?? ""); - if (match === null) return undefined; - const [, api, tag, abi] = match; - return `${ANDROID_SDK_ROOT}/system-images/android-${api ?? ""}/${tag ?? ""}/${abi ?? ""}`; -} - -/** - * why: An AVD in the user's own AVD home, as `avdmanager` writes it: a pointer file beside a folder - * whose `config.ini` names the system image (`image.sysdir.1`). - */ -export async function userAvd( - filesystem: MemoryFilesystem, - name: string, - sysdir: string, - options: { readonly avdHome?: string; readonly avdPath?: string } = {}, -): Promise { - const avdHome = options.avdHome ?? `${ANDROID_HOME_DIRECTORY}/.android/avd`; - const avdPath = options.avdPath ?? `${avdHome}/${name}.avd`; - await filesystem.mkdirp(avdPath); - await filesystem.writeFileAtomic( - `${avdHome}/${name}.ini`, - `avd.ini.encoding=UTF-8\npath=${avdPath}\n`, - ); - await filesystem.writeFileAtomic( - `${avdPath}/config.ini`, - `hw.ramSize=2048\nimage.sysdir.1=${sysdir}\ntag.id=google_apis\n`, - ); -} - -/** - * why: A `ScriptedProcessRunner` that also mirrors what real `sdkmanager` does on disk: a - * successful (`code: 0`, no unaccepted-license text) `--install ` call creates the - * corresponding `system-images/android-//` directory in `filesystem`, and a - * successful `--uninstall` removes it. A bare `ScriptedProcessRunner` only scripts the process's own - * stdout/stderr/exit code, never a filesystem side effect, so the scan after an install would not - * find the image it just "installed". - */ -export class InstallReflectingProcessRunner extends ScriptedProcessRunner { - readonly #filesystem: MemoryFilesystem; - - constructor(expectations: readonly ScriptedProcessExpectation[], filesystem: MemoryFilesystem) { - super(expectations); - this.#filesystem = filesystem; - } - - override spawn( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): ProcessHandle { - const handle = super.spawn(command, args, options); - const uninstalled = args[0] === "--uninstall" ? imageDirectory(args[1]) : undefined; - if (uninstalled !== undefined) { - void handle.wait().then(async (result) => { - if (result.code === 0) await this.#filesystem.rm(uninstalled); - }); - } - if (args[0] === "--install" && typeof args[1] === "string") { - const packageName = args[1]; - void handle.wait().then((result) => { - const combined = `${result.stdout}\n${result.stderr}`; - const licenseNotAccepted = - /licen[cs]e/i.test(combined) && /not (?:been )?accepted/i.test(combined); - if (result.code === 0 && !licenseNotAccepted) { - const match = /^system-images;android-(.+);(.+);(.+)$/.exec(packageName); - if (match !== null) { - const [, api, tag, abi] = match; - const imagePath = `${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}`; - void this.#filesystem - .mkdirp(imagePath) - .then(() => - this.#filesystem.writeFileAtomic( - `${imagePath}/source.properties`, - "Pkg.Revision=1\n", - ), - ); - } - } - }); - } - return handle; - } -} - -export const pixelDevices = `Available devices:\nid: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n`; -export const twoPixelDevices = - `Available devices:\nid: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n` + - `---------\nid: 1 or "pixel_9"\n Name: Pixel 9\n OEM : Google\n`; - -export async function writeDevicesXml( - filesystem: MemoryFilesystem, - deviceBodies: string, -): Promise { - await filesystem.mkdirp(`${ANDROID_HOME_DIRECTORY}/.android`); - await filesystem.writeFileAtomic( - `${ANDROID_HOME_DIRECTORY}/.android/devices.xml`, - `${deviceBodies}`, - ); -} - -export function customDeviceXml(name: string, ramMiB: number): string { - return ( - `${name}` + - `${ramMiB}` - ); -} - -export type FitImage = readonly [api: string, tag: string, abi: string, properties?: string]; - -export const fitDevices = - `Available devices:\n` + - `id: 0 or "pixel_5"\n Name: Pixel 5\n OEM : Google\n---------\n` + - `id: 1 or "pixel_9_pro"\n Name: Pixel 9 Pro\n OEM : Google\n---------\n` + - `id: 2 or "tv_720p"\n Name: Television (720p)\n OEM : Google\n Tag : android-tv\n---------\n` + - `id: 3 or "My Tablet"\n Name: My Tablet\n OEM : User\n---------\n` + - `id: 4 or "odd"\n Name: Odd Device\n OEM : Google\n Tag : foo-device\n---------\n` + - `id: 5 or "xr_headset"\n Name: XR Headset\n OEM : Google\n Tag : android-xr\n---------\n` + - `id: 6 or "glasses"\n Name: AI Glasses\n OEM : Google\n Tag : ai-glasses\n`; - -export type Refusal = Error & { readonly component?: string; readonly downloadable?: boolean }; - -export const HOST_IMAGES: readonly FitImage[] = [ - ["35", "google_apis", "arm64-v8a"], - ["36", "google_apis_playstore", "arm64-v8a"], - ["34", "android-tv", "arm64-v8a"], -]; - -export function testLibraryProfile( - overrides: Partial = {}, -): LibraryDeviceProfile { - return { - apiRange: { min: 1 }, - bootProperties: {}, - copiedFrom: "20.0", - deviceClass: "phone", - hardware: { "hw.lcd.height": "100", "hw.ramSize": "2G" }, - id: "test_phone", - imageKind: undefined, - kind: "library", - name: "Test Phone", - names: ["Test Phone", "test_phone"], - playStore: true, - tablet: false, - ...overrides, - }; -} - -export interface CustomFitProfile { - readonly apiLevel?: string; - readonly diagonal?: string; - readonly folded?: boolean; - readonly name: string; - readonly playStore?: string; - readonly tagId?: string; -} - -export function customFitXml(profile: CustomFitProfile): string { - const screen = - profile.diagonal === undefined && profile.folded !== true - ? "" - : `${profile.diagonal === undefined ? "" : `${profile.diagonal}`}` + - (profile.folded === true - ? "10802400" - : "") + - ""; - return ( - `${profile.name}` + - (profile.tagId === undefined ? "" : `${profile.tagId}`) + - `${screen}` + - (profile.apiLevel === undefined - ? "" - : `${profile.apiLevel}`) + - (profile.playStore === undefined - ? "" - : `${profile.playStore}`) + - "" - ); -} - -export function userListing( - ...entries: readonly (readonly [name: string, tag?: string])[] -): string { - return ( - "Available devices:\n" + - entries - .map( - ([name, tag]) => - `id: 9 or "${name}"\n Name: ${name}\n OEM : User\n` + - (tag === undefined ? "" : ` Tag : ${tag}\n`) + - "---------\n", - ) - .join("") - ); -} - -/** why: answers every `adb -s emu avd path` with `answer` and leaves each other call to `inner`, so a test lies about where an emulator runs. */ -export function answeringAvdPath( - adb: string, - inner: ProcessRunner, - answer: ProcessResult, -): ProcessRunner { - return { - run: async (command, args, options) => - command === adb && args[0] === "-s" && args.slice(2).join(" ") === "emu avd path" - ? answer - : inner.run(command, args, options), - spawn: (command, args, options) => inner.spawn(command, args, options), - spawnStreaming: (command, args, options) => inner.spawnStreaming(command, args, options), - }; -} - -export const simlockHome = "/home/simlock/.simlock"; - -export const instanceId = "instance-1"; - -export const avdDirectory = `${simlockHome}/devices/android`; - -export const binaries = { - adb: `${ANDROID_SDK_ROOT}/platform-tools/adb`, - avdmanager: `${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/avdmanager`, - emulator: `${ANDROID_SDK_ROOT}/emulator/emulator`, - sdkmanager: `${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager`, -} as const; - -/** - * why: A scripted emulator stays up until it is told to stop, as a real one does, so a boot that - * sees it exit has failed; `emu kill` and the seed's `reboot -p` are what stop it. - */ -export class EmulatorExitingRunner extends ScriptedProcessRunner { - /** why: a test of an emulator that ignores `emu kill` sets this. */ - ignoresEmuKill = false; - - /** why: the flag probe is not one of the calls a boot test scripts, so it is answered here and never enters `calls` or the script's order. */ - override run( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): Promise { - if (command === binaries.emulator && args[0] === "-help") { - return Promise.resolve(ok(emulatorHelpListing)); - } - return super.run(command, args, options); - } - - override spawn( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): ProcessHandle { - const handle = super.spawn(command, args, options); - const stopsTheEmulator = /(?:emu kill|shell reboot -p)$/.test(args.join(" ")); - if (!this.ignoresEmuKill && stopsTheEmulator) { - for (const [index, call] of this.calls.entries()) { - if (call.command === binaries.emulator && call.args.includes("-avd")) { - this.handles[index]?.kill("SIGKILL"); - } - } - } - return handle; - } -} - -/** why: an emulator new enough to list every flag Simlock passes, `-crash-report-mode` included. */ -export const emulatorHelpListing = [ - "-avd", - "-port", - "-no-snapshot-save", - "-grpc", - "-grpc-use-token", - "-crash-report-mode", -] - .map((flag) => ` ${flag} described`) - .join("\n"); - -const defaultSeedHome = `${avdDirectory}/sources/seed_two`; - -/** why: the calls of making and booting a seed, shared by the seed tests and the harness every driver test starts from. */ -export const seedEmulatorLine = [ - "-port", - "5586", - "-no-snapshot-save", - "-grpc", - "8586", - "-grpc-use-token", - "-crash-report-mode", - "never", -]; - -export function bootingAt( - folder: string, - launch: readonly string[], - hangs: boolean, - notReadyPolls = 0, - flags: readonly string[] = [], -) { - const serial = "emulator-5586"; - const notReady = Array.from({ length: notReadyPolls }, () => - processResult(binaries.adb, ["-s", serial, "shell", "getprop", "sys.boot_completed"], "0\n"), - ); - return [ - { - hangs, - match: { - args: ["-avd", "simlock", ...seedEmulatorLine, ...flags, ...launch], - command: binaries.emulator, - }, - }, - ...notReady, - processResult(binaries.adb, ["-s", serial, "shell", "getprop", "sys.boot_completed"], "1\n"), - processResult(binaries.adb, ["-s", serial, "shell", "getprop", "init.svc.bootanim"], ""), - processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${folder}/simlock.avd\r\nOK\r\n`, - ), - processResult( - binaries.adb, - ["-s", serial, "shell", "pm", "path", "android"], - "package:/system/framework/framework-res.apk\n", - ), - ] satisfies ScriptedProcessExpectation[]; -} - -/** why: the calls of making a seed, by stage, so a test can stop it at the step that fails. */ -export function seedStages( - home = defaultSeedHome, - emulatorVersion = "Android emulator version 36.1.9", - flags: readonly string[] = [], -) { - const serial = "emulator-5586"; - return { - create: processResult(binaries.avdmanager, [ - "create", - "avd", - "-n", - "simlock", - "-k", - "system-images;android-34;google_apis;arm64-v8a", - "-d", - "pixel_5", - ]), - devices: processResult(binaries.adb, ["devices"], "List of devices attached\n"), - firstBoot: bootingAt(home, ["-no-snapshot-load"], true, 0, flags), - kill: processResult(binaries.adb, ["-s", serial, "emu", "kill"]), - list: processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "snapshot", "list"], - "simlock_boot\n", - ), - powerOff: processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), - probe: processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${home}/simlock.avd\r\nOK\r\n`, - ), - save: processResult(binaries.adb, [ - "-s", - serial, - "emu", - "avd", - "snapshot", - "save", - "simlock_boot", - ]), - secondBoot: bootingAt(home, ["-no-snapshot-load"], true, 0, flags), - sync: processResult(binaries.adb, ["-s", serial, "shell", "sync"]), - version: processResult(binaries.emulator, ["-version"], emulatorVersion), - }; -} - -export function makingTheSeed( - home = defaultSeedHome, - emulatorVersion = "Android emulator version 36.1.9", - flags: readonly string[] = [], -): ScriptedProcessExpectation[] { - const stage = seedStages(home, emulatorVersion, flags); - return [ - stage.version, - stage.create, - stage.devices, - ...stage.firstBoot, - stage.sync, - stage.powerOff, - ...stage.secondBoot, - stage.save, - stage.list, - stage.probe, - stage.kill, - ]; -} - -export async function writeSeedFiles( - filesystem: MemoryFilesystem, - home: string, - config = "hw.ramSize=2048\n", - avdName = "simlock", -): Promise { - await filesystem.mkdirp(`${home}/${avdName}.avd`); - await filesystem.writeFileAtomic(`${home}/${avdName}.avd/config.ini`, config); - await filesystem.writeFileAtomic( - `${home}/${avdName}.ini`, - `avd.ini.encoding=UTF-8\npath=${home}/${avdName}.avd\ntarget=android-34\n`, - ); -} - -export async function androidFilesystem( - options: { - readonly config?: string; - readonly freeDiskBytes?: number; - readonly images?: readonly (readonly [string, string, string])[]; - readonly withDeviceRoot?: boolean; - } = {}, - filesystem: MemoryFilesystem = new MemoryFilesystem(options.freeDiskBytes), -): Promise { - for (const binary of Object.values(binaries)) { - await filesystem.mkdirp(binary.slice(0, binary.lastIndexOf("/"))); - await filesystem.writeFileAtomic(binary, "binary"); - } - // why: Marked, so `ensureOwnedRoot` adopts it rather than creating one -- a root Simlock - // created in an earlier run is the ordinary case, and it keeps the id generator's first - // value available for the AVD name the tests assert on. - if (options.withDeviceRoot !== false) { - await filesystem.mkdirp(avdDirectory); - await filesystem.writeFileAtomic( - `${avdDirectory}/${OWNED_ROOT_MARKER_FILE}`, - JSON.stringify({ instanceId, owner: "simlock", platform: "android", schemaVersion: 1 }), - ); - } - for (const [api, tag, abi] of options.images ?? [["34", "google_apis", "arm64-v8a"]]) { - await filesystem.mkdirp(`${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}`); - } - if (options.config !== undefined) { - await filesystem.mkdirp(`${avdDirectory}/simlock_one.avd`); - await filesystem.writeFileAtomic(`${avdDirectory}/simlock_one.avd/config.ini`, options.config); - } - return filesystem; -} - -/** why: making a seed from whichever image and device profile the test's request resolves to; `device` pins the profile when the test cares. */ -export function makingASeedFrom( - home = defaultSeedHome, - device: string | RegExp = /.+/, -): ScriptedProcessExpectation[] { - const making = makingTheSeed(home); - making[1] = processResult(binaries.avdmanager, [ - "create", - "avd", - "-n", - "simlock", - "-k", - /.+/, - "-d", - device, - ]); - return making; -} - -/** - * why: the boot of a copy, as the driver asks the emulator for it: `adb devices` for the port, - * the launch, then the readiness polls. `launch` is `-snapshot simlock_boot` for a prepare boot. - */ -function copyBootExpectations(options: { - readonly bootCompleted?: string | undefined; - readonly copyHome: string; - readonly emulatorFlags?: readonly string[] | undefined; - readonly initialAdbFailures?: number | undefined; - readonly launch?: readonly string[]; - readonly unsettledPmAnswers?: readonly ProcessResult[] | undefined; -}): ScriptedProcessExpectation[] { - const bootCompleted = options.bootCompleted ?? "1\n"; - const serial = "emulator-5586"; - const poll = (property: string, answer: string) => - processResult(binaries.adb, ["-s", serial, "shell", "getprop", property], answer); - const path = processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${options.copyHome}/simlock.avd\r\nOK\r\n`, - ); - const pm = (answer: ProcessResult): ScriptedProcessExpectation => ({ - match: { args: ["-s", serial, "shell", "pm", "path", "android"], command: binaries.adb }, - result: answer, - }); - const expectations: ScriptedProcessExpectation[] = [ - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - { - hangs: true, - match: { - args: [ - "-avd", - "simlock", - ...seedEmulatorLine, - ...(options.emulatorFlags ?? []), - ...(options.launch ?? ["-snapshot", "simlock_boot"]), - ], - command: binaries.emulator, - }, - }, - ]; - for (let failure = 0; failure < (options.initialAdbFailures ?? 0); failure += 1) { - expectations.push({ - match: { - args: ["-s", serial, "shell", "getprop", "sys.boot_completed"], - command: binaries.adb, - }, - result: { code: 1, stderr: "connection refused", stdout: "" }, - }); - } - expectations.push(poll("sys.boot_completed", bootCompleted)); - if (bootCompleted.trim() !== "1") return expectations; - for (const answer of options.unsettledPmAnswers ?? []) { - expectations.push( - poll("init.svc.bootanim", ""), - path, - pm(answer), - poll("sys.boot_completed", "1\n"), - ); - } - expectations.push( - poll("init.svc.bootanim", ""), - path, - pm({ code: 0, stderr: "", stdout: "package:/system/framework/framework-res.apk\n" }), - ); - return expectations; -} - -class ListingRunner extends EmulatorExitingRunner { - constructor( - private readonly listing: string, - expectations: readonly ScriptedProcessExpectation[], - ) { - super(expectations); - } - - override spawn( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): ProcessHandle { - if (command === binaries.avdmanager && args[0] === "list") { - return new EmulatorExitingRunner([processResult(command, args, this.listing)]).spawn( - command, - args, - options, - ); - } - return super.spawn(command, args, options); - } -} - -/** why: what the seed and library tests need of whatever makes a seed: a driver at the root, or the part with the catalog beside it. */ -export interface SeedProvisioner { - listCatalog(): Promise; - provision(spec: VirtualSpec): Promise; - resolveSpec(request: ExactDeviceRequest): Promise; -} - -export type BuildProvisioner = ( - filesystem: MemoryFilesystem, - processRunner: ProcessRunner, - options: { - readonly deviceProfileSources?: readonly DeviceProfileSource[]; - readonly extraEnv?: Readonly>; - readonly hostAbi?: string; - readonly ids: readonly string[]; - }, -) => Promise; - -export async function fitHarness( - build: BuildProvisioner, - images: readonly FitImage[], - options: { - readonly devicesXml?: string; - readonly extraEnv?: Readonly>; - readonly hostAbi?: string; - readonly listing?: string; - readonly profiles?: readonly LibraryDeviceProfile[]; - } = {}, -) { - const filesystem = await androidFilesystem({ - images: images.map(([api, tag, abi]) => [api, tag, abi]), - }); - for (const [api, tag, abi, properties] of images) { - if (properties !== undefined) { - await filesystem.writeFileAtomic( - `${ANDROID_SDK_ROOT}/system-images/android-${api}/${tag}/${abi}/source.properties`, - properties, - ); - } - } - if (options.devicesXml !== undefined) await writeDevicesXml(filesystem, options.devicesXml); - await writeSeedFiles(filesystem, defaultSeedHome); - const runner = new ListingRunner(options.listing ?? fitDevices, makingASeedFrom()); - const driver = await build(filesystem, runner, { - hostAbi: options.hostAbi ?? "arm64-v8a", - ids: ["one", "two"], - ...(options.extraEnv === undefined ? {} : { extraEnv: options.extraEnv }), - ...(options.profiles === undefined - ? {} - : { deviceProfileSources: [{ profiles: async () => options.profiles ?? [] }] }), - }); - const resolve = (model: string, osVersion?: string, imageTag?: string) => - driver.resolveSpec({ - model, - platform: "android", - ...(osVersion === undefined ? {} : { osVersion }), - ...(imageTag === undefined ? {} : { imageTag }), - }); - const refusal = (model: string, osVersion?: string, imageTag?: string) => - resolve(model, osVersion, imageTag).then( - () => { - throw new Error(`${model} ${osVersion ?? ""} ${imageTag ?? ""} resolved`); - }, - (error: unknown) => error as Refusal, - ); - const createdPackage = () => { - const create = runner.calls.find((call) => call.args[0] === "create"); - return create?.args[create.args.indexOf("-k") + 1]; - }; - return { createdPackage, driver, filesystem, refusal, resolve, runner }; -} - -const SEED_LINES = [ - "avd.ini.encoding=UTF-8", - "hw.cpu.arch=arm64", - "hw.ramSize=1G", - "hw.lcd.density=1", - "hw.seedOnly=yes", - "image.sysdir.1=system-images/android-35/google_apis/arm64-v8a/", - "tag.id=google_apis", - "abi.type=arm64-v8a", - "target=android-35", - "disk.dataPartition.size=6G", - "showDeviceFrame=yes", - "skin.name=pixel_8", - "skin.path=skins/pixel_8", - "vm.heapSize=64M", - "PlayStore.enabled=no", -]; - -export function seedConfig(extra: readonly string[]): string { - return `${[...SEED_LINES.filter((line) => !extra.some((added) => added.split("=")[0] === line.split("=")[0])), ...extra].join("\n")}\n`; -} - -// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. -export async function libraryHarness( - build: BuildProvisioner, - options: { - readonly devicesXml?: string; - readonly extraImages?: readonly (readonly [string, string, string])[]; - readonly filesystem?: MemoryFilesystem; - readonly imageProperties?: string; - readonly imageTag?: string; - readonly listing?: string; - readonly profiles?: readonly LibraryDeviceProfile[]; - readonly seed?: string; - readonly seedBootProp?: string; - readonly sources?: readonly DeviceProfileSource[]; - } = {}, -) { - const tag = options.imageTag ?? "google_apis"; - const filesystem = options.filesystem ?? new MemoryFilesystem(); - await androidFilesystem( - { images: [["35", tag, "arm64-v8a"], ...(options.extraImages ?? [])] }, - filesystem, - ); - const imageProperties = options.imageProperties ?? `Pkg.Revision=1\nSystemImage.TagId=${tag}\n`; - if (imageProperties !== "") { - await filesystem.writeFileAtomic( - `${ANDROID_SDK_ROOT}/system-images/android-35/${tag}/arm64-v8a/source.properties`, - imageProperties, - ); - } - if (options.devicesXml !== undefined) { - await writeDevicesXml(filesystem, options.devicesXml); - } - // why: the profile is applied to the seed's AVD, which `avdmanager create` writes in the seed's own folder. - const seedFolder = `${avdDirectory}/sources/seed_two`; - const avd = `${seedFolder}/simlock.avd`; - const configPath = `${avd}/config.ini`; - const bootPropPath = `${avd}/boot.prop`; - await writeSeedFiles(filesystem, seedFolder, options.seed ?? seedConfig([])); - if (options.seedBootProp !== undefined) { - await filesystem.writeFileAtomic(bootPropPath, options.seedBootProp); - } - const listing = options.listing ?? pixelDevices; - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], listing), - ...makingASeedFrom(seedFolder), - ]); - const sources = - options.profiles === undefined - ? options.sources - : [{ profiles: async () => options.profiles ?? [] }]; - const driver = await build(filesystem, runner, { - ids: ["one", "two"], - ...(sources === undefined ? {} : { deviceProfileSources: sources }), - }); - const resolve = (model: string) => - driver.resolveSpec({ model, osVersion: "35", platform: "android" }); - const readLines = async () => - (await filesystem.readFile(configPath)).split("\n").filter((line) => line !== ""); - return { - bootPropPath, - config: async () => - new Map( - (await readLines()).map((line) => [ - line.slice(0, line.indexOf("=")), - line.slice(line.indexOf("=") + 1), - ]), - ), - configPath, - createCalls: () => runner.calls.filter((call) => call.args[0] === "create"), - driver, - filesystem, - lines: readLines, - provision: async (model: string) => driver.provision(await resolve(model)), - resolve, - runner, - seedDeviceId: () => { - const create = runner.calls.find((call) => call.args[0] === "create"); - return create?.args[create.args.indexOf("-d") + 1]; - }, - }; -} - -// why: the erasable mark a boot writes into the running device. -export function markWriteExpectation(serial: string, token: string): ScriptedProcessExpectation { - return processResult(binaries.adb, [ - "-s", - serial, - "shell", - `echo '${JSON.stringify({ token })}' > /data/local/tmp/simlock-mark.json`, - ]); -} - -export function createdFromPackage(runner: ScriptedProcessRunner): string | undefined { - const create = runner.calls.find( - (call) => call.command === binaries.avdmanager && call.args[0] === "create", - ); - return create?.args[create.args.indexOf("-k") + 1]; -} - -export async function taggedHarness( - build: BuildProvisioner, - images: readonly (readonly [string, string, string])[], - hostAbi = "arm64-v8a", -) { - const filesystem = await androidFilesystem({ images }); - await writeSeedFiles(filesystem, defaultSeedHome); - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ...makingASeedFrom(defaultSeedHome, "pixel_5"), - ...copyBootExpectations({ copyHome: `${avdDirectory}/simlock_one` }), - markWriteExpectation("emulator-5586", "device-3"), - ]); - const driver = await build(filesystem, runner, { hostAbi, ids: ["one", "two"] }); - return { driver, filesystem, runner }; -} - -export function bootProbes(runner: ScriptedProcessRunner): number { - return runner.calls.filter((call) => call.args.includes("sys.boot_completed")).length; -} - -export const adbServerPort = 5038; - -export const adbServerPid = 4242; - -export const adbRecordPath = `${simlockHome}/adb-server.json`; - -export const scopedEnv = { - ANDROID_ADB_SERVER_PORT: String(adbServerPort), - ANDROID_AVD_HOME: avdDirectory, - ANDROID_HOME: sdk, -} as const; - -export const PHYSICAL_DEVICE = { - address: "R58M123ABC", - deviceId: "R58M123ABC", - driverData: {}, - enrolledApps: ["com.example.kept"], - physical: true, -} as const; - -/** why: the arguments of every emulator start, in order. */ -export function emulatorLaunchArgs(runner: ScriptedProcessRunner): (readonly string[])[] { - return runner.calls - .filter((call) => call.command === binaries.emulator && call.args.includes("-avd")) - .map((call) => call.args); -} - -/** - * why: a driver and the copy it provisioned: the seed is made (scripted) and copied, so what a test - * scripts after `afterwards` is about that copy. The seed's id is the second the generator gives - * and the copy's the first, so a copy's folder is `/simlock_one/` and the mark token of its - * first boot is the third id, `device-3`. - */ -// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. -export async function provisionedHarness( - options: { - readonly bootCompleted?: string; - readonly config?: string; - readonly emulator?: AndroidEmulatorLaunchOptions; - readonly filesystem?: MemoryFilesystem; - readonly emulatorFlags?: readonly string[]; - readonly ids?: readonly string[]; - readonly initialAdbFailures?: number; - readonly tcpListener?: FakeTcpListenerFactory; - readonly unsettledPmAnswers?: readonly ProcessResult[]; - readonly bootTimeoutMs?: number; - readonly firstMarkToken?: string; - readonly afterwards?: readonly ScriptedProcessExpectation[]; - } = {}, -) { - const filesystem = await androidFilesystem({}, options.filesystem); - const ids = options.ids ?? ["one", "two"]; - const seedHome = `${avdDirectory}/sources/seed_${ids[1] ?? "two"}`; - const copyHome = `${avdDirectory}/simlock_${ids[0] ?? "one"}`; - await writeSeedFiles(filesystem, seedHome, options.config); - const tcpProbe = new FakeTcpProbe([adbServerPort]); - const clock = new FakeClock(); - const expectations: ScriptedProcessExpectation[] = [ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ...makingTheSeed(seedHome, "Android emulator version 36.1.9", options.emulatorFlags), - ...copyBootExpectations({ ...options, copyHome }), - ]; - if ((options.bootCompleted ?? "1\n").trim() === "1") { - expectations.push(markWriteExpectation("emulator-5586", options.firstMarkToken ?? "device-3")); - } - expectations.push(...(options.afterwards ?? [])); - const runner = new EmulatorExitingRunner(expectations); - const driver = await createDriver(filesystem, runner, { - clock, - ...(options.emulator === undefined ? {} : { emulator: options.emulator }), - ids, - ...(options.tcpListener === undefined ? {} : { tcpListener: options.tcpListener }), - ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), - tcpProbe, - }); - const spec = await driver.resolveSpec({ - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - const device = await driver.provision(spec); - - return { clock, copyHome, device, driver, filesystem, runner, seedHome, tcpProbe }; -} - -export async function createDriver( - filesystem: Filesystem, - processRunner: ProcessRunner, - options: { - readonly acceptAndroidLicenses?: boolean; - /** why: `ANDROID_AVD_HOME` in the environment the daemon started with; unset by default. */ - readonly avdHome?: string; - readonly clock?: FakeClock; - /** why: a second driver over the same table is the daemon after a restart. */ - readonly database?: Awaited>; - readonly deviceProfileSources?: readonly DeviceProfileSource[]; - readonly driverConfig?: Readonly>; - readonly emulator?: AndroidEmulatorLaunchOptions; - readonly extraEnv?: Readonly>; - /** why: the ABI this host runs natively; `arm64-v8a` unless a test says otherwise. */ - readonly hostAbi?: string; - readonly ids?: readonly string[]; - readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; - readonly bootTimeoutMs?: number; - readonly scrcpyServerPath?: string; - readonly tcpListener?: FakeTcpListenerFactory; - readonly tcpProbe?: FakeTcpProbe; - readonly tcpStreamConnector?: FakeTcpStreamConnector; - } = {}, -) { - let nextId = 0; - const driverConfig = options.driverConfig ?? {}; - const configuredPort = - typeof driverConfig["adbServerPort"] === "number" - ? driverConfig["adbServerPort"] - : adbServerPort; - await recordRunningAdbServer(filesystem, configuredPort); - return AndroidDriver.create({ - ...onlyProvided(options), - clock: options.clock ?? new FakeClock(), - database: options.database ?? (await memoryAndroidSourceDatabase()), - driverConfig, - env: { - ANDROID_HOME: sdk, - ...options.extraEnv, - ...(options.avdHome === undefined ? {} : { ANDROID_AVD_HOME: options.avdHome }), - }, - filesystem, - homeDirectory: home, - hostAbi: options.hostAbi ?? "arm64-v8a", - idGenerator: { - generate: () => options.ids?.[nextId++] ?? `device-${nextId}`, - }, - instanceId, - processRunner, - processSupervisor: new FakeProcessSupervisor([adbServerPid]), - simlockHome, - tcpProbe: options.tcpProbe ?? new FakeTcpProbe([configuredPort]), - }); -} - -/** why: the options this helper only forwards when a test actually set one -- `AndroidDriver.create` - * declares them optional under `exactOptionalPropertyTypes`, so an absent one must be absent - * rather than `undefined`. Split out of `createDriver` so that the list can keep growing - * without the helper itself turning into a pile of conditionals. */ -function onlyProvided(options: { - readonly acceptAndroidLicenses?: boolean; - readonly deviceProfileSources?: readonly DeviceProfileSource[]; - readonly emulator?: AndroidEmulatorLaunchOptions; - readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; - readonly bootTimeoutMs?: number; - readonly scrcpyServerPath?: string; - readonly tcpListener?: FakeTcpListenerFactory; - readonly tcpStreamConnector?: FakeTcpStreamConnector; -}) { - return { - ...(options.tcpListener === undefined ? {} : { tcpListener: options.tcpListener }), - ...(options.scrcpyServerPath === undefined - ? {} - : { scrcpyServerPath: options.scrcpyServerPath }), - ...(options.tcpStreamConnector === undefined - ? {} - : { tcpStreamConnector: options.tcpStreamConnector }), - ...(options.acceptAndroidLicenses === undefined - ? {} - : { acceptAndroidLicenses: options.acceptAndroidLicenses }), - ...(options.deviceProfileSources === undefined - ? {} - : { deviceProfileSources: options.deviceProfileSources }), - config: { - ...(options.emulator === undefined ? {} : { emulator: options.emulator }), - ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), - }, - ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }), - }; -} - -export async function recordRunningAdbServer( - filesystem: Filesystem, - port = adbServerPort, -): Promise { - await filesystem.mkdirp(simlockHome); - await filesystem.writeFileAtomic( - adbRecordPath, - JSON.stringify({ pid: adbServerPid, port, startedAt: 1 }), - ); -} - -/** - * why: The emulators on one machine, which outlive any one daemon: as much of `avdmanager`, - * `emulator` and `adb` as provisioning, booting, saving a snapshot and stopping use. Each - * `runner()` is one daemon process's view of it -- the driver keeps its port reservations per - * runner, as a real daemon keeps them per process. - * - * Two behaviours are the real tools', not Simlock's. An emulator told to take a console port - * another one holds exits at once: `emulator -help` (36.1.9) says that "if any of these ports - * is already used, the emulator will fail to start". And a serial is answered by whichever - * emulator is listening on its port. - */ -export class EmulatorHost { - /** why: the AVD folder each console port's emulator runs, as its `emu avd path` answers it. */ - readonly #listening = new Map(); - readonly #exits = new Map void>(); - #nextPid = 1; - - constructor(private readonly filesystem: MemoryFilesystem) {} - - pathAnswering(serial: string | undefined): string | undefined { - return this.#listening.get(consolePortOf(serial)); - } - - /** why: the id of the device whose AVD answers: a copy's folder name, or an old-layout AVD's name. */ - avdAnswering(serial: string | undefined): string | undefined { - const folder = this.#listening.get(consolePortOf(serial)); - if (folder === undefined) return undefined; - const home = folder.slice(0, folder.lastIndexOf("/")); - return home === avdDirectory ? basename(folder, ".avd") : basename(home); - } - - runner(): ProcessRunner { - return { - run: (command, args, options) => this.#run(command, args, options?.env), - spawn: (command, args, options) => this.#spawn(command, args, options?.env), - spawnStreaming: (command, args) => { - throw new Error(`Unexpected streaming invocation: ${command} ${args.join(" ")}`); - }, - }; - } - - async #run( - command: string, - args: readonly string[], - env: NodeJS.ProcessEnv | undefined, - ): Promise { - const result = - command === binaries.avdmanager - ? await this.#avdmanager(args, env?.["ANDROID_AVD_HOME"] ?? avdDirectory) - : command === binaries.adb - ? this.#adb(args) - : command === binaries.emulator - ? emulatorQuery(args) - : undefined; - if (result === undefined) { - throw new Error(`Unexpected process invocation: ${command} ${args.join(" ")}`); - } - return result; - } - - async #avdmanager(args: readonly string[], home: string): Promise { - if (args[0] === "list") return ok(pixelDevices); - const avdName = args[args.indexOf("-n") + 1] ?? ""; - if (args[0] === "delete") { - await this.filesystem.rm(`${home}/${avdName}.avd`); - return ok(); - } - if (args[0] !== "create") return undefined; - await writeSeedFiles(this.filesystem, home, "hw.ramSize=2048\n", avdName); - return ok(); - } - - // fallow-ignore-next-line complexity -- one fake adb: each branch answers one command of the table. - #adb(args: readonly string[]): ProcessResult | undefined { - if (args[0] === "devices") { - const lines = [...this.#listening.keys()].map((port) => `emulator-${port}\tdevice\n`); - return ok(`List of devices attached\n${lines.join("")}`); - } - if (args[0] !== "-s") return undefined; - const port = consolePortOf(args[1]); - if (!this.#listening.has(port)) { - return { code: 1, stderr: `adb: device '${args[1]}' not found`, stdout: "" }; - } - const rest = args.slice(2).join(" "); - if (rest === "emu kill" || rest === "shell reboot -p") { - this.#exit(port); - return ok("OK: killing emulator, bye bye\n"); - } - if (rest.startsWith("shell echo ")) return ok(); - if (rest.startsWith("shell getprop ro.boot.qemu.avd_name; head -c ")) { - return ok(`${basename(this.#listening.get(port) ?? "", ".avd")}\n`); - } - if (rest === "emu avd path") { - return ok(`${this.#listening.get(port) ?? ""}\r\nOK\r\n`); - } - const answer = ADB_ANSWERS.get(rest); - return answer === undefined ? undefined : ok(answer); - } - - #spawn( - command: string, - args: readonly string[], - env: NodeJS.ProcessEnv | undefined, - ): ProcessHandle { - if (command !== binaries.emulator || args[0] !== "-avd") { - throw new Error(`Unexpected spawn: ${command} ${args.join(" ")}`); - } - const avdName = `${env?.["ANDROID_AVD_HOME"] ?? avdDirectory}/${args[1] ?? ""}.avd`; - const port = Number(args[args.indexOf("-port") + 1]); - let exit!: (result: ProcessResult) => void; - const exited = new Promise((resolve) => { - exit = resolve; - }); - const bound = !this.#listening.has(port); - if (bound) { - this.#listening.set(port, avdName); - this.#exits.set(port, () => exit({ code: 0, stderr: "", stdout: "" })); - } else { - exit({ code: 1, stderr: `console port ${port} is already used`, stdout: "" }); - } - return { - // why: Killing an emulator that never bound must not stop the one that did. - kill: () => { - if (bound && this.#listening.get(port) === avdName) this.#exit(port); - }, - pid: this.#nextPid++, - stderr: emptyLines(), - stdout: emptyLines(), - unref: () => undefined, - wait: () => exited, - }; - } - - #exit(port: number): void { - this.#listening.delete(port); - this.#exits.get(port)?.(); - this.#exits.delete(port); - } -} - -export class RecordingRunner implements ProcessRunner { - readonly calls: { args: readonly string[]; command: string; options: ProcessRunOptions }[] = []; - - constructor(private readonly inner: ProcessRunner) {} - - run(command: string, args: readonly string[], options: ProcessRunOptions = {}) { - this.calls.push({ args, command, options }); - return this.inner.run(command, args, options); - } - - spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}) { - this.calls.push({ args, command, options }); - return this.inner.spawn(command, args, options); - } - - spawnStreaming(...args: Parameters) { - return this.inner.spawnStreaming(...args); - } -} - -const ADB_ANSWERS = new Map([ - ["shell getprop sys.boot_completed", "1\n"], - ["shell getprop init.svc.bootanim", "stopped\n"], - ["shell pm path android", "package:/system/framework/framework-res.apk\n"], - ["shell sync", ""], - ["emu avd snapshot save simlock_boot", "OK\n"], - ["emu avd snapshot list", "simlock_boot\n"], -]); - -export function consolePortOf(serial: string | undefined): number { - return Number(/^emulator-(\d+)$/.exec(serial ?? "")?.[1]); -} - -export function ok(stdout = ""): ProcessResult { - return { code: 0, stderr: "", stdout }; -} - -export async function* emptyLines(): AsyncIterable {} - -/** why: what a running copy answers to `emu avd path`: the AVD inside its own folder. */ -export function copyPathExpectation( - serial: string, - folder = "simlock_one", -): ScriptedProcessExpectation { - return processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${avdDirectory}/${folder}/simlock.avd\r\nOK\r\n`, - ); -} - -export class WorkingDirectoryFilesystem extends MemoryFilesystem { - constructor(private readonly workingDirectory: string) { - super(); - } - - override async realpath(path: string): Promise { - return super.realpath(path.startsWith("/") ? path : `${this.workingDirectory}/${path}`); - } -} - -/** why: the two read-only `emulator` calls the fake answers; any other is unexpected. */ -function emulatorQuery(args: readonly string[]): ProcessResult | undefined { - if (args[0] === "-version") return ok("Android emulator version 36.1.9"); - if (args[0] === "-help") return ok(emulatorHelpListing); - return undefined; -} +export { + ANDROID_SDK_ROOT, + ANDROID_HOME_DIRECTORY, + memoryAndroidSourceDatabase, + processResult, + userAvd, + InstallReflectingProcessRunner, + pixelDevices, + twoPixelDevices, + writeDevicesXml, + customDeviceXml, + fitDevices, + HOST_IMAGES, + testLibraryProfile, + customFitXml, + userListing, + answeringAvdPath, + simlockHome, + instanceId, + avdDirectory, + binaries, + ok, + WorkingDirectoryFilesystem, +} from "./test-sdk-fixtures.js"; + +export type { FitImage, Refusal, CustomFitProfile } from "./test-sdk-fixtures.js"; + +export { + EmulatorExitingRunner, + emulatorHelpListing, + seedEmulatorLine, + bootingAt, + seedStages, + makingTheSeed, + writeSeedFiles, + androidFilesystem, + makingASeedFrom, + fitHarness, + seedConfig, + libraryHarness, + markWriteExpectation, + createdFromPackage, + taggedHarness, +} from "./test-seed-harness.js"; + +export type { SeedProvisioner, BuildProvisioner } from "./test-seed-harness.js"; + +export { + bootProbes, + adbServerPort, + adbServerPid, + adbRecordPath, + scopedEnv, + PHYSICAL_DEVICE, + emulatorLaunchArgs, + provisionedHarness, + createDriver, + recordRunningAdbServer, +} from "./test-driver-harness.js"; + +export { + EmulatorHost, + RecordingRunner, + consolePortOf, + emptyLines, + copyPathExpectation, +} from "./test-emulator-host.js"; From 763b957e2bdae8880a1290f73d28d3fc7eb9c4df Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:02:12 +0100 Subject: [PATCH 03/83] refactor(agents): split spec-lint.mjs into parts Moves the pure constants to spec-lint-patterns.mjs and the pure text and reference helpers to spec-lint-text.mjs. The feature checks read the run's parsed body and issue state, so they stay in the root; the CLI output and exit codes are unchanged. --- .agents/scripts/spec-lint-patterns.mjs | 78 ++++++++++ .agents/scripts/spec-lint-text.mjs | 101 +++++++++++++ .agents/scripts/spec-lint.mjs | 201 ++++--------------------- 3 files changed, 211 insertions(+), 169 deletions(-) create mode 100644 .agents/scripts/spec-lint-patterns.mjs create mode 100644 .agents/scripts/spec-lint-text.mjs diff --git a/.agents/scripts/spec-lint-patterns.mjs b/.agents/scripts/spec-lint-patterns.mjs new file mode 100644 index 000000000..db0b99d55 --- /dev/null +++ b/.agents/scripts/spec-lint-patterns.mjs @@ -0,0 +1,78 @@ +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +export const TASK_H2 = ["Scope", "Technical spec", "Done when", "Out of scope", "Depends on"]; +export const TASK_H3 = [ + "Rows", + "Facts", + "Modules touched", + "Rules in play", + "Retired words", + "Tests", +]; +// why: tasks and features written before the design tables keep their layout until they close. +export const OLD_TASK_H2 = [ + "Scope", + "In short", + "Technical spec", + "Done when", + "Out of scope", + "Depends on", +]; +export const OLD_TASK_H3 = [ + "Facts", + "Modules touched", + "Surface and extension points", + "Contract and event changes", + "Other code on the same state", + "Failure modes", + "Rules in play", + "Retired words", + "Tests", +]; +export const OPTIONAL = new Set(["Surface and extension points", "Retired words"]); +const BUSINESS_H2 = [ + "Problem", + "Who it is for", + "Outcome", + "How it works for the user", + "Examples", + "What could go wrong", + "Words used", + "Non-goals", + "Completion conditions", + "Open questions", + "Decisions", +]; +export const DESIGN = [ + ["Behaviours", "B"], + ["Stored state", "S"], + ["Interfaces", "I"], + ["Existing code", "X"], +]; +export const FEATURE_H2 = [...BUSINESS_H2, ...DESIGN.map(([name]) => name), "Tasks"]; +export const OLD_FEATURE_H2 = [...BUSINESS_H2, "Shared values", "Behaviours", "Tasks"]; +export const SCHEMA_STEP = /^([\w-]+) \+(\d+)$/; +export const REFACTOR_H2 = ["Why", "Target shape", "Out of bounds", "Done when"]; +export const FAILURE_HEADER = ["Call", "Outcome", "What happens", "Limit, counted from"]; +export const CANNOT_SPLIT = + /\b(cannot|can't|can not|does not|doesn't|won't) split\b|\bone task, not\b|\bkept? (it )?whole\b/i; +export const HARDWARE = /slow lane|real (simulator|emulator|device|iPhone|phone)|on a Mac with/i; +export const SLOW_TEST = /e2e\/slow-[\w-]+\.test\.ts/g; +const EXTENSIONS = + "ts|tsx|js|mjs|cjs|json|jsonc|md|mdx|yml|yaml|sh|swift|kt|kts|java|toml|txt|html|css|plist|py|xml|gradle"; +export const PATH_LINE = new RegExp( + `^(v\\d[\\w.-]*:)?([\\w@./-]*[\\w@-]\\.(?:${EXTENSIONS})):(\\d+)(?:-(\\d+))?$`, +); +export const PLAIN_PATH_LINE = new RegExp( + `(? /^\s*```mermaid\s*$/.test(l)); +} + +function ref(path, from, to, tool = false) { + return { tool, path, from: +from, to: +(to ?? from) }; +} + +function backtickedRef(content, last) { + const full = PATH_LINE.exec(content); + if (full !== null) { + const tool = full[1] !== undefined; + return { found: ref(full[2], full[3], full[4], tool), last: tool ? null : full[2] }; + } + const tail = /^:(\d+)(?:-(\d+))?$/.exec(content); + if (tail === null || last === null) return { found: null, last }; + return { found: ref(last, tail[1], tail[2]), last }; +} + +function backtickedRefs(line) { + const refs = []; + let last = null; + for (const m of line.matchAll(/`([^`]+)`/g)) { + const next = backtickedRef(m[1], last); + last = next.last; + if (next.found !== null) refs.push(next.found); + } + return refs; +} + +function plainRefs(line) { + return [...line.replace(/`[^`]*`/g, " ").matchAll(PLAIN_PATH_LINE)].map((m) => + ref(m[1], m[2], m[3]), + ); +} + +export function pathRefs(lines) { + return prose(lines).flatMap((line) => [...backtickedRefs(line), ...plainRefs(line)]); +} + +export function namedPaths(lines) { + return new Set( + [ + ...prose(lines) + .join("\n") + .matchAll(/`([\w@./-]+\/[\w@.-]+)(?::[\d-]+)?`/g), + ].map((m) => m[1]), + ); +} + +export function taskNumbers(cell) { + return [...(cell ?? "").matchAll(/#(\d+)/g)].map((m) => Number(m[1])); +} + +// why: Existing code says `none` under its Commit and Searches lines, so the word is looked for +// on a line of its own, never inside a sentence. +export function isNone(s) { + const lines = prose(s.lines).map((l) => l.replaceAll("`", "").trim()); + return ( + !tables(s.lines).some((t) => t.header[0] === "Id") && lines.some((l) => /^none\.?$/i.test(l)) + ); +} + +export function bare(cell) { + return cell.replaceAll("`", "").trim(); +} + +export function words(s) { + return new Set( + s + .replace(/([a-z])([A-Z])/g, "$1 $2") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((w) => w.length > 2 && !STOP.has(w)) + .map((w) => w.replace(/(ing|ed|es|s)$/, "")), + ); +} + +function shared(a, b) { + return [...a].filter((w) => b.has(w)).length; +} + +// why: a line may spell out the cases its row only names ("object rows of Examples"), so it +// passes when it holds most of the row's Behaviour words, or most of the shorter of the two. +export function overlap(line, row) { + const byBehaviour = + row.behaviour.size === 0 ? 0 : shared(row.behaviour, line) / row.behaviour.size; + const shorter = Math.min(line.size, row.words.size); + return Math.max(byBehaviour, shorter === 0 ? 0 : shared(line, row.words) / shorter); +} diff --git a/.agents/scripts/spec-lint.mjs b/.agents/scripts/spec-lint.mjs index d93f55ec9..7a4d6faa9 100644 --- a/.agents/scripts/spec-lint.mjs +++ b/.agents/scripts/spec-lint.mjs @@ -60,8 +60,7 @@ // directory, which needs the body's commit. The repo is GH_REPO, else the origin remote. import { execFileSync } from "node:child_process"; import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; +import { join } from "node:path"; import { bullets, @@ -78,77 +77,40 @@ import { stripComments, tables, } from "./spec-body.mjs"; +import { + CANNOT_SPLIT, + CODE_PATH, + DESIGN, + DOC_PATH, + FAILURE_HEADER, + FEATURE_H2, + HARDWARE, + OLD_FEATURE_H2, + OLD_TASK_H2, + OLD_TASK_H3, + OPTIONAL, + REFACTOR_H2, + ROOT, + SCHEMA_STEP, + SLOW_TEST, + TASK_H2, + TASK_H3, +} from "./spec-lint-patterns.mjs"; +import { + bare, + isNone, + lacksMermaid, + namedPaths, + overlap, + pathRefs, + squash, + taskNumbers, + text, + words, +} from "./spec-lint-text.mjs"; -const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); -const TASK_H2 = ["Scope", "Technical spec", "Done when", "Out of scope", "Depends on"]; -const TASK_H3 = ["Rows", "Facts", "Modules touched", "Rules in play", "Retired words", "Tests"]; -// why: tasks and features written before the design tables keep their layout until they close. -const OLD_TASK_H2 = [ - "Scope", - "In short", - "Technical spec", - "Done when", - "Out of scope", - "Depends on", -]; -const OLD_TASK_H3 = [ - "Facts", - "Modules touched", - "Surface and extension points", - "Contract and event changes", - "Other code on the same state", - "Failure modes", - "Rules in play", - "Retired words", - "Tests", -]; -const OPTIONAL = new Set(["Surface and extension points", "Retired words"]); -const BUSINESS_H2 = [ - "Problem", - "Who it is for", - "Outcome", - "How it works for the user", - "Examples", - "What could go wrong", - "Words used", - "Non-goals", - "Completion conditions", - "Open questions", - "Decisions", -]; -const DESIGN = [ - ["Behaviours", "B"], - ["Stored state", "S"], - ["Interfaces", "I"], - ["Existing code", "X"], -]; -const FEATURE_H2 = [...BUSINESS_H2, ...DESIGN.map(([name]) => name), "Tasks"]; -const OLD_FEATURE_H2 = [...BUSINESS_H2, "Shared values", "Behaviours", "Tasks"]; -const SCHEMA_STEP = /^([\w-]+) \+(\d+)$/; -const REFACTOR_H2 = ["Why", "Target shape", "Out of bounds", "Done when"]; -const FAILURE_HEADER = ["Call", "Outcome", "What happens", "Limit, counted from"]; -const CANNOT_SPLIT = - /\b(cannot|can't|can not|does not|doesn't|won't) split\b|\bone task, not\b|\bkept? (it )?whole\b/i; -const HARDWARE = /slow lane|real (simulator|emulator|device|iPhone|phone)|on a Mac with/i; -const SLOW_TEST = /e2e\/slow-[\w-]+\.test\.ts/g; -const EXTENSIONS = - "ts|tsx|js|mjs|cjs|json|jsonc|md|mdx|yml|yaml|sh|swift|kt|kts|java|toml|txt|html|css|plist|py|xml|gradle"; -const PATH_LINE = new RegExp( - `^(v\\d[\\w.-]*:)?([\\w@./-]*[\\w@-]\\.(?:${EXTENSIONS})):(\\d+)(?:-(\\d+))?$`, -); -const PLAIN_PATH_LINE = new RegExp( - `(? { @@ -218,10 +180,6 @@ function usage() { process.exit(2); } -function text(s) { - return s === undefined ? "" : prose(s.lines).join("\n").trim(); -} - function h3Within(name) { const parent = section(parsed, name, 2); if (parent === undefined) return []; @@ -256,10 +214,6 @@ function templateComments() { return out; } -function squash(s) { - return s.replace(/\s+/g, " ").trim(); -} - function checkTemplateComments() { const known = templateComments(); for (const m of body.matchAll(//g)) { @@ -287,10 +241,6 @@ function checkFooter() { if (!IS_FOOTER.test(last)) add("B", "body", "the last line is not `*Written by an agent.*`"); } -function lacksMermaid(s) { - return s !== undefined && !s.lines.some((l) => /^\s*```mermaid\s*$/.test(l)); -} - function lintRefactor() { checkOrder( parsed.sections.filter((s) => s.level === 2), @@ -403,42 +353,6 @@ function checkFacts() { } } -function ref(path, from, to, tool = false) { - return { tool, path, from: +from, to: +(to ?? from) }; -} - -function backtickedRef(content, last) { - const full = PATH_LINE.exec(content); - if (full !== null) { - const tool = full[1] !== undefined; - return { found: ref(full[2], full[3], full[4], tool), last: tool ? null : full[2] }; - } - const tail = /^:(\d+)(?:-(\d+))?$/.exec(content); - if (tail === null || last === null) return { found: null, last }; - return { found: ref(last, tail[1], tail[2]), last }; -} - -function backtickedRefs(line) { - const refs = []; - let last = null; - for (const m of line.matchAll(/`([^`]+)`/g)) { - const next = backtickedRef(m[1], last); - last = next.last; - if (next.found !== null) refs.push(next.found); - } - return refs; -} - -function plainRefs(line) { - return [...line.replace(/`[^`]*`/g, " ").matchAll(PLAIN_PATH_LINE)].map((m) => - ref(m[1], m[2], m[3]), - ); -} - -function pathRefs(lines) { - return prose(lines).flatMap((line) => [...backtickedRefs(line), ...plainRefs(line)]); -} - function tree(sha) { if (!trees.has(sha)) { trees.set(sha, new Set(git("ls-tree", "-r", "--name-only", sha).split("\n").filter(Boolean))); @@ -455,16 +369,6 @@ function resolvePath(sha, path, named = new Set()) { return preferred.length === 1 ? preferred : found; } -function namedPaths(lines) { - return new Set( - [ - ...prose(lines) - .join("\n") - .matchAll(/`([\w@./-]+\/[\w@.-]+)(?::[\d-]+)?`/g), - ].map((m) => m[1]), - ); -} - function lineCount(sha, path) { const key = `${sha}:${path}`; if (!counts.has(key)) { @@ -636,10 +540,6 @@ function checkOwnBehaviours() { checkTaskRows({ number, body }, rows); } -function taskNumbers(cell) { - return [...(cell ?? "").matchAll(/#(\d+)/g)].map((m) => Number(m[1])); -} - function designRows(p) { const out = []; for (const [name, prefix] of DESIGN) { @@ -707,19 +607,6 @@ function lintFeature() { checkGraph(subs); } -// why: Existing code says `none` under its Commit and Searches lines, so the word is looked for -// on a line of its own, never inside a sentence. -function isNone(s) { - const lines = prose(s.lines).map((l) => l.replaceAll("`", "").trim()); - return ( - !tables(s.lines).some((t) => t.header[0] === "Id") && lines.some((l) => /^none\.?$/i.test(l)) - ); -} - -function bare(cell) { - return cell.replaceAll("`", "").trim(); -} - function checkIds(s, name, prefix) { const rows = tables(s.lines) .filter((t) => t.header[0] === "Id") @@ -885,30 +772,6 @@ function behaviourRows(p) { })); } -function words(s) { - return new Set( - s - .replace(/([a-z])([A-Z])/g, "$1 $2") - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((w) => w.length > 2 && !STOP.has(w)) - .map((w) => w.replace(/(ing|ed|es|s)$/, "")), - ); -} - -function shared(a, b) { - return [...a].filter((w) => b.has(w)).length; -} - -// why: a line may spell out the cases its row only names ("object rows of Examples"), so it -// passes when it holds most of the row's Behaviour words, or most of the shorter of the two. -function overlap(line, row) { - const byBehaviour = - row.behaviour.size === 0 ? 0 : shared(row.behaviour, line) / row.behaviour.size; - const shorter = Math.min(line.size, row.words.size); - return Math.max(byBehaviour, shorter === 0 ? 0 : shared(line, row.words) / shorter); -} - function checkTestIds(task, own, lines) { const ids = own.map((r) => r.id); const named = lines.map((l) => /^(B\d+):/.exec(l)?.[1] ?? null); From f96200c095b1bc00d5577bf6b72b59cb17809e6b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:03:12 +0100 Subject: [PATCH 04/83] refactor(core): split doctor.ts into parts --- src/core/doctor-drift.ts | 120 +++++++ src/core/doctor-findings.ts | 314 ++++++++++++++++++ src/core/doctor-types.ts | 188 +++++++++++ src/core/doctor.ts | 644 +++--------------------------------- 4 files changed, 661 insertions(+), 605 deletions(-) create mode 100644 src/core/doctor-drift.ts create mode 100644 src/core/doctor-findings.ts create mode 100644 src/core/doctor-types.ts diff --git a/src/core/doctor-drift.ts b/src/core/doctor-drift.ts new file mode 100644 index 000000000..12dbd7153 --- /dev/null +++ b/src/core/doctor-drift.ts @@ -0,0 +1,120 @@ +import type { DeviceOperationClaims } from "./device-operation-claims.js"; +import type { DeviceRecord, DeviceState } from "./domain.js"; +import type { ObservedDevice, ObservedMark } from "./driver.js"; +import { type DoctorFinding, key, type ProvenanceDrift } from "./doctor-types.js"; + +export function registryDriftFindings( + device: DeviceRecord, + realDeviceKeys: ReadonlySet, + observedDevices: ReadonlyMap, + claims?: Pick, +): DoctorFinding[] { + const findings: DoctorFinding[] = []; + const deviceKey = key(device.spec.platform, device.driverDeviceId); + + const missing = missingDeviceFinding(device, deviceKey, realDeviceKeys); + if (missing !== undefined) { + findings.push(missing); + } + + // why: `expected === undefined` means the registry is mid-transition: simlock is acting on the device + // itself, so neither run state nor marks are compared. + // + // A device this daemon holds an operation claim on is excluded too: a lease-path boot from + // `shutdown` runs `makeReady` while the record still says `shutdown`, and a concurrent + // `doctor --fix` would read that as foreign drift, "fix" the record, and make the boot's own + // commit drop its `address`/`driverData`/`mode`. + const expected = expectedRunState(device.state); + const observed = observedDevices.get(deviceKey); + const claimed = claims?.isClaimed(device.id) === true; + if (expected !== undefined && observed !== undefined && !claimed) { + findings.push(...liveDriftFindings(device, expected, observed)); + } + + return findings; +} + +function missingDeviceFinding( + device: DeviceRecord, + deviceKey: string, + realDeviceKeys: ReadonlySet, +): DoctorFinding | undefined { + if (device.state === "deleted" || realDeviceKeys.has(deviceKey)) { + return undefined; + } + return { deviceId: device.id, kind: "registry-device-missing", platform: device.spec.platform }; +} + +function liveDriftFindings( + device: DeviceRecord, + expected: "running" | "stopped", + observed: ObservedDevice, +): DoctorFinding[] { + const findings: DoctorFinding[] = []; + + if (observed.runState !== "transitioning" && observed.runState !== expected) { + findings.push({ + deviceId: device.id, + expected, + kind: "foreign-state-change", + observed: observed.runState, + platform: device.spec.platform, + }); + } + + const detail = observed.mark === undefined ? undefined : provenanceDrift(observed.mark); + if (detail !== undefined) { + findings.push({ + detail, + deviceId: device.id, + kind: "foreign-provenance-change", + platform: device.spec.platform, + }); + } + + return findings; +} + +/** + * why: Report-only by design. Re-marking would overwrite the only evidence that the + * device was tampered with, and a fresh mark cannot be written to a leased + * device without disturbing its holder -- so the core never repairs a mark. + */ +function provenanceDrift(mark: ObservedMark): ProvenanceDrift | undefined { + if (mark.durable === undefined) { + return "durable-mark-missing"; + } + if (!mark.erasableReadable) { + return undefined; + } + if (mark.erasable === undefined) { + return "erased"; + } + return mark.erasable === mark.durable ? undefined : "mark-mismatch"; +} + +export function isDrift(finding: DoctorFinding): boolean { + return ( + finding.kind !== "driver-advisory" && + finding.kind !== "prerequisite-missing" && + finding.kind !== "service-broken" && + finding.kind !== "warm-pool-target-unreachable" && + finding.kind !== "module-unavailable" + ); +} + +function expectedRunState(state: DeviceState): "running" | "stopped" | undefined { + switch (state) { + case "ready": + case "leased": + return "running"; + case "shutdown": + return "stopped"; + case "provisioning": + case "reclaiming": + case "quarantined": + case "absent": + case "deleted": + return undefined; + } +} diff --git a/src/core/doctor-findings.ts b/src/core/doctor-findings.ts new file mode 100644 index 000000000..4fa6a8903 --- /dev/null +++ b/src/core/doctor-findings.ts @@ -0,0 +1,314 @@ +import { boundDriverSummary, DRIVER_DETAIL_MAX, parseOsConstraint } from "../contract/index.js"; +import type { Config } from "./config.js"; +import { + type DeviceRecord, + type DeviceSpec, + type Platform, + transitionEnteredAt, +} from "./domain.js"; +import type { DeviceOperationClaims } from "./device-operation-claims.js"; +import type { + Driver, + DriverCatalogEntry, + DriverDevice, + DriverRejection, + MissingPrerequisite, +} from "./driver.js"; +import { type DoctorFinding, key } from "./doctor-types.js"; +import type { WarmTargetFigures } from "./warm-pool/index.js"; + +/** + * why: A `reclaiming` record does not say which level started it -- every production path + * asks for `standard`, but that is the caller's choice, not an invariant this finding can + * depend on -- and the two errors are not symmetric: an estimate that is too tight turns a + * healthy reclaim into a false stall finding, while one that is too loose only delays a real + * one. So this takes the slower branch rather than assuming. + */ +function slowestReclaimEstimateMs(driver: Pick, spec: DeviceSpec): number { + return Math.max( + driver.estimate({ clean: "standard", operation: "reclaim" }, spec), + driver.estimate({ clean: "full", operation: "reclaim" }, spec), + ); +} + +/** + * why: A `provisioning` / `reclaiming` device is normally in-flight work Simlock itself is + * driving (see `expectedRunState`), not drift -- but only up to a point. Past a + * driver-derived threshold it stops being "still working" and becomes a stall: the + * driver call that was supposed to resolve it never did. This is a documented failure + * mode, not a hypothetical one -- `DeviceProvisioner` deliberately leaves a device + * `provisioning` when a boot-timeout's own cleanup `destroy` also fails ("The + * registered record remains for reconcile when the driver cannot destroy it"). + * + * The threshold is `driver.estimate(...) * thresholdMultiplier`, floored at + * `minimumThresholdMs` (see `stallThresholdMs`): not `estimate` itself, because the + * estimate is tuned for a routine run and real-world variance (a cold Android boot, a + * loaded host) can legitimately run well past it without anything having stalled. No + * driver for the device's platform, or no recorded entry time (defensive; see + * `transitionEnteredAt`), means there is nothing to compare against, so no stall rather + * than a guess. + * + * A device this daemon holds an operation claim on is excluded outright, before any + * of that arithmetic: the claim *is* the statement that work is in progress, so a + * live operation can never be read as a stall no matter how long it legitimately + * runs. This is the same live-versus-orphaned test `StartupConverger + * #recoverInterruptedReclaims` already makes, and it is load-bearing rather than + * belt-and-braces -- every release now backgrounds its reclaim, holding the device in + * `reclaiming` for a full erase, measured at ~34s, and several such erases run at once + * and contend for the same disk. The estimate the threshold is built from now reflects + * that erase rather than the 1s it used to claim (#56), but tuning a threshold against + * contended disk speed would still be guessing; the claim answers it exactly. A reclaim + * orphaned by a crash has no claim in the new process, so the case this test exists to + * catch is untouched. + */ +export function isStalledTransition(input: StallInput): boolean { + const { claims, config, device, driver, now } = input; + if (device.state !== "provisioning" && device.state !== "reclaiming") { + return false; + } + if (claims?.isClaimed(device.id) === true) { + return false; + } + const enteredAt = transitionEnteredAt(device); + if (enteredAt === undefined || driver === undefined) { + return false; + } + return now - enteredAt > stallThresholdMs(device, driver, config); +} + +export interface StallInput { + readonly device: DeviceRecord; + readonly driver: Pick | undefined; + readonly config: Config["stalledTransition"]; + readonly claims: Pick | undefined; + readonly now: number; +} + +function stallThresholdMs( + device: DeviceRecord, + driver: Pick, + config: Config["stalledTransition"], +): number { + const estimateMs = + device.state === "provisioning" + ? driver.estimate({ operation: "provision" }, device.spec) + + driver.estimate({ operation: "boot" }, device.spec) + : slowestReclaimEstimateMs(driver, device.spec); + return Math.max(estimateMs * config.thresholdMultiplier, config.minimumThresholdMs); +} + +export function stalledTransitionFinding( + device: DeviceRecord, + driversByPlatform: ReadonlyMap, + config: Config["stalledTransition"], + now: number, + claims?: Pick, +): DoctorFinding | undefined { + const driver = driversByPlatform.get(device.spec.platform); + if (!isStalledTransition({ claims, config, device, driver, now })) { + return undefined; + } + const enteredAt = transitionEnteredAt(device); + if ( + driver === undefined || + enteredAt === undefined || + (device.state !== "provisioning" && device.state !== "reclaiming") + ) { + return undefined; + } + return { + ageMs: now - enteredAt, + deviceId: device.id, + enteredAt, + kind: "stalled-transition", + platform: device.spec.platform, + state: device.state, + thresholdMs: stallThresholdMs(device, driver, config), + }; +} + +/** + * why: Discovery runs once, at daemon start, so a refusal is a frozen snapshot: repairing the + * root and running `doctor` again reports the identical finding, because nothing has + * looked at the root since. The finding therefore has to name the step that actually + * retries the platform, or it reads as a repair that did not work. + */ +const DRIVER_RETRY_REMEDY = + "Driver discovery runs once, at daemon startup: restart the daemon (`simlock daemon stop`, then any command relaunches it) once this is fixed, or the platform stays unavailable."; + +export function driverUnavailableFindings(rejections: readonly DriverRejection[]): DoctorFinding[] { + return rejections.map((rejection) => ({ + // why: a driver's summary is an error message of any length; the remedy is kept whole. + detail: `${boundDriverSummary(rejection.summary, DRIVER_DETAIL_MAX - DRIVER_RETRY_REMEDY.length - 2)}. ${DRIVER_RETRY_REMEDY}`, + kind: "driver-unavailable" as const, + platform: rejection.platform, + reason: rejection.reason, + })); +} + +const PLATFORM_NAMES: Readonly> = { android: "Android", ios: "iOS" }; + +function isRange(osVersion: string): boolean { + const parsed = parseOsConstraint(osVersion); + return parsed.ok && parsed.constraint.kind === "range"; +} + +// fallow-ignore-next-line complexity -- one finding per short reason, each a fixed message. +export function unreachableTarget( + target: WarmTargetFigures, + catalog: DriverCatalogEntry | undefined, +): DoctorFinding[] { + const { model, osVersion, platform, short } = target; + const named = [model, osVersion, target.mode].filter((part) => part !== undefined).join(" / "); + const name = PLATFORM_NAMES[platform]; + if (short === "runtime-missing") { + if (osVersion !== undefined && isRange(osVersion)) { + return [ + { + kind: "warm-pool-target-unreachable", + message: `no installed ${name} runtime satisfies ${osVersion}`, + platform, + reason: short, + remedy: `run simlock component list --platform ${platform} to see the versions, then simlock component install ${platform} for one inside ${osVersion}`, + target: named, + }, + ]; + } + if ( + model !== undefined && + osVersion !== undefined && + catalog?.runtimes.includes(osVersion) === true && + catalog.modelRuntimes[model] !== undefined && + !catalog.modelRuntimes[model].includes(osVersion) + ) { + return [ + { + kind: "warm-pool-target-unreachable", + message: `${model} does not pair with ${name} ${osVersion}`, + platform, + reason: short, + remedy: `run simlock catalog --platform ${platform}`, + target: named, + }, + ]; + } + return [ + { + kind: "warm-pool-target-unreachable", + message: `${name} ${osVersion ?? "runtime"} is not installed`, + platform, + reason: short, + remedy: + osVersion === undefined + ? `run simlock component list --platform ${platform} to see the versions, then simlock component install ${platform} ` + : `run simlock component install ${platform} ${osVersion}`, + target: named, + }, + ]; + } + if (short === "unknown-model") { + return [ + { + kind: "warm-pool-target-unreachable", + message: `${model} is not a known ${name} model`, + platform, + reason: short, + remedy: `run simlock catalog --platform ${platform} to list the models`, + target: named, + }, + ]; + } + return []; +} + +export function overLimit(input: { + readonly target: string; + readonly sum: number; + readonly limit: number; + readonly reserve: number; + readonly room: number; + readonly scope: string; + readonly platform?: Platform; +}): DoctorFinding { + const { limit, platform, reserve, room, scope, sum, target } = input; + const held = reserve > 0 ? `, of which warmPool.reserveRunning holds ${reserve}` : ""; + return { + kind: "warm-pool-target-unreachable", + message: `the ${scope}targets want ${sum} running devices, and the ${scope}running limit is ${limit}${held}, leaving room for ${room}`, + reason: "over-limit", + remedy: `lower the counts of the ${scope}warmPool.targets, ${reserve > 0 ? "" : "or "}raise the ${scope}running limit${reserve > 0 ? ", or lower warmPool.reserveRunning" : ""}`, + target, + ...(platform === undefined ? {} : { platform }), + }; +} + +export function prerequisiteFindings( + platform: Platform, + missing: readonly MissingPrerequisite[], + running: boolean, + rejected: boolean, +): DoctorFinding[] { + if (missing.length === 0) { + return running || rejected + ? [] + : [ + { + kind: "prerequisite-missing", + message: "Every prerequisite is present, but the daemon started without this platform.", + platform, + prerequisite: "daemon-restart", + remedy: DRIVER_RETRY_REMEDY, + }, + ]; + } + return missing.map((found) => ({ + kind: "prerequisite-missing" as const, + message: found.message, + platform, + prerequisite: found.prerequisite, + remedy: running ? found.remedy : `${found.remedy} ${DRIVER_RETRY_REMEDY}`, + })); +} + +export function orphanFindings( + realities: readonly { + readonly driver: Driver; + readonly reality: { + readonly devices: readonly DriverDevice[]; + readonly processes: readonly DriverDevice[]; + }; + }[], + registryDeviceKeys: ReadonlySet, +): DoctorFinding[] { + const findings: DoctorFinding[] = []; + for (const { driver, reality } of realities) { + for (const device of reality.devices) { + if (!registryDeviceKeys.has(key(driver.platform, device.deviceId))) { + findings.push({ device, kind: "orphan-device", platform: driver.platform }); + } + } + for (const device of reality.processes) { + if (!registryDeviceKeys.has(key(driver.platform, device.deviceId))) { + findings.push({ device, kind: "orphan-process", platform: driver.platform }); + } + } + } + return findings; +} + +export function expiredLeaseFindings( + leases: readonly { + readonly id: string; + readonly deviceId: string; + readonly ttlDeadline: number; + }[], + now: number, +): DoctorFinding[] { + return leases + .filter((lease) => lease.ttlDeadline <= now) + .map((lease) => ({ + deviceId: lease.deviceId, + kind: "expired-live-lease" as const, + leaseId: lease.id, + })); +} diff --git a/src/core/doctor-types.ts b/src/core/doctor-types.ts new file mode 100644 index 000000000..f8bc5bf78 --- /dev/null +++ b/src/core/doctor-types.ts @@ -0,0 +1,188 @@ +import type { EventBus } from "../bus/index.js"; +import type { Clock, Logger } from "../ports/index.js"; +import type { Config } from "./config.js"; +import type { DeviceOperationClaims } from "./device-operation-claims.js"; +import type { DeviceState, Platform } from "./domain.js"; +import type { + Driver, + DriverDevice, + DriverRejection, + DriverRejectionReason, + DriverSource, + PrerequisiteCheck, +} from "./driver.js"; +import type { CapacityReader, LeaseExpirer, WarmPoolReader } from "./core-ports.js"; +import type { Registry } from "./registry.js"; +import type { SerializedDecision } from "./serialized-decision.js"; +import type { StartupRead } from "./startup-read.js"; + +export type DoctorFinding = + | { + readonly kind: "registry-device-missing"; + readonly deviceId: string; + readonly platform: Platform; + } + | { readonly kind: "orphan-device"; readonly device: DriverDevice; readonly platform: Platform } + | { readonly kind: "orphan-process"; readonly device: DriverDevice; readonly platform: Platform } + | { readonly kind: "expired-live-lease"; readonly leaseId: string; readonly deviceId: string } + | { + readonly kind: "foreign-state-change"; + readonly deviceId: string; + readonly platform: Platform; + readonly expected: "running" | "stopped"; + readonly observed: "running" | "stopped"; + } + | { + readonly kind: "foreign-provenance-change"; + readonly deviceId: string; + readonly platform: Platform; + readonly detail: ProvenanceDrift; + } + | { + readonly kind: "driver-unavailable"; + readonly platform: Platform; + readonly reason: DriverRejectionReason; + readonly detail: string; + } + | { + readonly kind: "legacy-device"; + readonly deviceId: string; + readonly platform: Platform; + readonly device: DriverDevice; + readonly path?: string; + } + | { + readonly kind: "stalled-transition"; + readonly deviceId: string; + readonly platform: Platform; + readonly state: "provisioning" | "reclaiming"; + readonly enteredAt: number; + readonly ageMs: number; + readonly thresholdMs: number; + } + | { + readonly kind: "driver-advisory"; + readonly platform: Platform; + readonly code: string; + readonly message: string; + } + | { + readonly kind: "prerequisite-missing"; + readonly platform: Platform; + readonly prerequisite: string; + readonly message: string; + readonly remedy: string; + } + | { + /** + * why: the worker's login service names a Node or `main.js` that is gone. Like + * `driver-advisory` it is not drift: `--fix` never acts on it, and `doctor.reconciled` does not carry it. + */ + readonly kind: "service-broken"; + readonly manager: "launchd" | "systemd"; + readonly name: string; + readonly missing: readonly string[]; + readonly remedy: string; + } + | { + readonly kind: "warm-pool-target-unreachable"; + readonly platform?: Platform; + readonly target: string; + readonly reason: "runtime-missing" | "unknown-model" | "over-limit"; + readonly message: string; + readonly remedy: string; + } + | { + readonly kind: "module-unavailable"; + readonly module: string; + readonly message: string; + readonly remedy: string; + }; + +/** + * why: the service manager's answer for this role's service, injected so the core checks no path + * itself; `check` returns the paths the service names that are gone, none when whole or absent. + */ +export interface ServiceCheck { + readonly manager: "launchd" | "systemd"; + readonly name: string; + readonly check: () => Promise; +} + +export type OptionalModuleFinding = Extract; + +export type ProvenanceDrift = "erased" | "mark-mismatch" | "durable-mark-missing"; + +export type OrphanDeviceFinding = Extract; + +export interface DoctorReport { + readonly findings: readonly DoctorFinding[]; + readonly sources: readonly DoctorSource[]; +} + +export interface DoctorSource extends Omit { + readonly platform: Platform; + readonly baselineName?: string; + readonly baselineVersion?: number; +} + +/** why: the baselines the worker holds name their sources and are checked for staleness before a report. */ +export interface DoctorBaselines { + checkStale(): Promise; + nameSources(sources: readonly DriverSource[]): readonly (DriverSource & { + readonly baselineName?: string; + readonly baselineVersion?: number; + })[]; +} + +export interface DoctorQuarantine { + enterFromStalledTransition(deviceId: string, findingState: DeviceState): Promise; +} + +export type DoctorConfig = Pick; + +export interface DoctorOptions { + readonly clock: Clock; + readonly config: DoctorConfig; + /** + * why: The decision section every registry write of this daemon is made in. A write made outside it + * can land on top of a concurrent one that read the registry before it, and undo it. + */ + readonly baselines?: DoctorBaselines; + readonly decisions: Pick; + readonly drivers: readonly Driver[]; + readonly eventBus: EventBus; + readonly leaseExpirer?: LeaseExpirer; + readonly quarantine?: DoctorQuarantine; + readonly claims?: Pick; + readonly driverRejections?: readonly DriverRejection[]; + readonly logger?: Logger; + readonly optionalModules?: () => readonly OptionalModuleFinding[]; + readonly warmPool?: { + readonly figures: WarmPoolReader["figures"]; + readonly runningCapacity: () => CapacityReader["runningCapacity"]; + }; + readonly prerequisiteChecks?: readonly PrerequisiteCheck[]; + readonly serviceCheck?: ServiceCheck; + readonly runningPlatforms?: () => readonly Platform[]; + readonly registry: Registry; +} + +export interface DoctorReconcileOptions { + readonly fix?: boolean; + readonly read?: StartupRead; + /** + * why: deliberately not part of `fix`: someone already running `doctor --fix` unattended in + * CI must not acquire a destructive behaviour by upgrading (ADR 0001, decision 6). + */ + readonly purgeOrphans?: boolean; + readonly prerequisites?: boolean; +} + +export function key(platform: string, deviceId: string): string { + return `${platform}:${deviceId}`; +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/core/doctor.ts b/src/core/doctor.ts index f57087653..13ae49d51 100644 --- a/src/core/doctor.ts +++ b/src/core/doctor.ts @@ -1,199 +1,53 @@ -import type { EventBus } from "../bus/index.js"; -import { boundDriverSummary, DRIVER_DETAIL_MAX, parseOsConstraint } from "../contract/index.js"; -import { type Clock, type Logger, NoopLogger } from "../ports/index.js"; -import type { Config } from "./config.js"; -import { - type DeviceRecord, - type DeviceSpec, - type DeviceState, - mayBeGranted, - type Platform, - transitionEnteredAt, - type VirtualDevice, -} from "./domain.js"; -import type { DeviceOperationClaims } from "./device-operation-claims.js"; -import type { StartupRead } from "./startup-read.js"; +import { type Logger, NoopLogger } from "../ports/index.js"; +import { mayBeGranted, type DeviceRecord, type Platform, type VirtualDevice } from "./domain.js"; import { type Driver, type DriverCatalogEntry, - type DriverDevice, - type DriverRejection, type DriverReality, type DriverSource, - type DriverRejectionReason, type MissingPrerequisite, type ObservedDevice, - type ObservedMark, - type PrerequisiteCheck, virtualDriverDevice, } from "./driver.js"; -import type { CapacityReader, LeaseExpirer, WarmPoolReader } from "./core-ports.js"; +import { + type DoctorFinding, + type DoctorOptions, + type DoctorReconcileOptions, + type DoctorReport, + type DoctorSource, + errorMessage, + key, + type OptionalModuleFinding, + type OrphanDeviceFinding, +} from "./doctor-types.js"; +import { isDrift, registryDriftFindings } from "./doctor-drift.js"; +import { + driverUnavailableFindings, + expiredLeaseFindings, + orphanFindings, + overLimit, + prerequisiteFindings, + stalledTransitionFinding, + unreachableTarget, +} from "./doctor-findings.js"; import { reservedRunning, type WarmTargetFigures } from "./warm-pool/index.js"; -import type { Registry } from "./registry.js"; -import type { SerializedDecision } from "./serialized-decision.js"; - -export type DoctorFinding = - | { - readonly kind: "registry-device-missing"; - readonly deviceId: string; - readonly platform: Platform; - } - | { readonly kind: "orphan-device"; readonly device: DriverDevice; readonly platform: Platform } - | { readonly kind: "orphan-process"; readonly device: DriverDevice; readonly platform: Platform } - | { readonly kind: "expired-live-lease"; readonly leaseId: string; readonly deviceId: string } - | { - readonly kind: "foreign-state-change"; - readonly deviceId: string; - readonly platform: Platform; - readonly expected: "running" | "stopped"; - readonly observed: "running" | "stopped"; - } - | { - readonly kind: "foreign-provenance-change"; - readonly deviceId: string; - readonly platform: Platform; - readonly detail: ProvenanceDrift; - } - | { - readonly kind: "driver-unavailable"; - readonly platform: Platform; - readonly reason: DriverRejectionReason; - readonly detail: string; - } - | { - readonly kind: "legacy-device"; - readonly deviceId: string; - readonly platform: Platform; - readonly device: DriverDevice; - readonly path?: string; - } - | { - readonly kind: "stalled-transition"; - readonly deviceId: string; - readonly platform: Platform; - readonly state: "provisioning" | "reclaiming"; - readonly enteredAt: number; - readonly ageMs: number; - readonly thresholdMs: number; - } - | { - readonly kind: "driver-advisory"; - readonly platform: Platform; - readonly code: string; - readonly message: string; - } - | { - readonly kind: "prerequisite-missing"; - readonly platform: Platform; - readonly prerequisite: string; - readonly message: string; - readonly remedy: string; - } - | { - /** - * why: the worker's login service names a Node or `main.js` that is gone. Like - * `driver-advisory` it is not drift: `--fix` never acts on it, and `doctor.reconciled` does not carry it. - */ - readonly kind: "service-broken"; - readonly manager: "launchd" | "systemd"; - readonly name: string; - readonly missing: readonly string[]; - readonly remedy: string; - } - | { - readonly kind: "warm-pool-target-unreachable"; - readonly platform?: Platform; - readonly target: string; - readonly reason: "runtime-missing" | "unknown-model" | "over-limit"; - readonly message: string; - readonly remedy: string; - } - | { - readonly kind: "module-unavailable"; - readonly module: string; - readonly message: string; - readonly remedy: string; - }; - -/** - * why: the service manager's answer for this role's service, injected so the core checks no path - * itself; `check` returns the paths the service names that are gone, none when whole or absent. - */ -export interface ServiceCheck { - readonly manager: "launchd" | "systemd"; - readonly name: string; - readonly check: () => Promise; -} - -export type OptionalModuleFinding = Extract; - -export type ProvenanceDrift = "erased" | "mark-mismatch" | "durable-mark-missing"; - -type OrphanDeviceFinding = Extract; - -export interface DoctorReport { - readonly findings: readonly DoctorFinding[]; - readonly sources: readonly DoctorSource[]; -} - -export interface DoctorSource extends Omit { - readonly platform: Platform; - readonly baselineName?: string; - readonly baselineVersion?: number; -} - -/** why: the baselines the worker holds name their sources and are checked for staleness before a report. */ -export interface DoctorBaselines { - checkStale(): Promise; - nameSources(sources: readonly DriverSource[]): readonly (DriverSource & { - readonly baselineName?: string; - readonly baselineVersion?: number; - })[]; -} - -export interface DoctorQuarantine { - enterFromStalledTransition(deviceId: string, findingState: DeviceState): Promise; -} - -export type DoctorConfig = Pick; - -export interface DoctorOptions { - readonly clock: Clock; - readonly config: DoctorConfig; - /** - * why: The decision section every registry write of this daemon is made in. A write made outside it - * can land on top of a concurrent one that read the registry before it, and undo it. - */ - readonly baselines?: DoctorBaselines; - readonly decisions: Pick; - readonly drivers: readonly Driver[]; - readonly eventBus: EventBus; - readonly leaseExpirer?: LeaseExpirer; - readonly quarantine?: DoctorQuarantine; - readonly claims?: Pick; - readonly driverRejections?: readonly DriverRejection[]; - readonly logger?: Logger; - readonly optionalModules?: () => readonly OptionalModuleFinding[]; - readonly warmPool?: { - readonly figures: WarmPoolReader["figures"]; - readonly runningCapacity: () => CapacityReader["runningCapacity"]; - }; - readonly prerequisiteChecks?: readonly PrerequisiteCheck[]; - readonly serviceCheck?: ServiceCheck; - readonly runningPlatforms?: () => readonly Platform[]; - readonly registry: Registry; -} +import type { StartupRead } from "./startup-read.js"; -export interface DoctorReconcileOptions { - readonly fix?: boolean; - readonly read?: StartupRead; - /** - * why: deliberately not part of `fix`: someone already running `doctor --fix` unattended in - * CI must not acquire a destructive behaviour by upgrading (ADR 0001, decision 6). - */ - readonly purgeOrphans?: boolean; - readonly prerequisites?: boolean; -} +export type { + DoctorBaselines, + DoctorConfig, + DoctorFinding, + DoctorOptions, + DoctorQuarantine, + DoctorReconcileOptions, + DoctorReport, + DoctorSource, + OptionalModuleFinding, + ProvenanceDrift, + ServiceCheck, +} from "./doctor-types.js"; +export { isStalledTransition } from "./doctor-findings.js"; +export type { StallInput } from "./doctor-findings.js"; export class Doctor { readonly #logger: Logger; @@ -932,423 +786,3 @@ export class Doctor { await this.options.quarantine.enterFromStalledTransition(finding.deviceId, finding.state); } } - -function registryDriftFindings( - device: DeviceRecord, - realDeviceKeys: ReadonlySet, - observedDevices: ReadonlyMap, - claims?: Pick, -): DoctorFinding[] { - const findings: DoctorFinding[] = []; - const deviceKey = key(device.spec.platform, device.driverDeviceId); - - const missing = missingDeviceFinding(device, deviceKey, realDeviceKeys); - if (missing !== undefined) { - findings.push(missing); - } - - // why: `expected === undefined` means the registry is mid-transition: simlock is acting on the device - // itself, so neither run state nor marks are compared. - // - // A device this daemon holds an operation claim on is excluded too: a lease-path boot from - // `shutdown` runs `makeReady` while the record still says `shutdown`, and a concurrent - // `doctor --fix` would read that as foreign drift, "fix" the record, and make the boot's own - // commit drop its `address`/`driverData`/`mode`. - const expected = expectedRunState(device.state); - const observed = observedDevices.get(deviceKey); - const claimed = claims?.isClaimed(device.id) === true; - if (expected !== undefined && observed !== undefined && !claimed) { - findings.push(...liveDriftFindings(device, expected, observed)); - } - - return findings; -} - -function missingDeviceFinding( - device: DeviceRecord, - deviceKey: string, - realDeviceKeys: ReadonlySet, -): DoctorFinding | undefined { - if (device.state === "deleted" || realDeviceKeys.has(deviceKey)) { - return undefined; - } - return { deviceId: device.id, kind: "registry-device-missing", platform: device.spec.platform }; -} - -function liveDriftFindings( - device: DeviceRecord, - expected: "running" | "stopped", - observed: ObservedDevice, -): DoctorFinding[] { - const findings: DoctorFinding[] = []; - - if (observed.runState !== "transitioning" && observed.runState !== expected) { - findings.push({ - deviceId: device.id, - expected, - kind: "foreign-state-change", - observed: observed.runState, - platform: device.spec.platform, - }); - } - - const detail = observed.mark === undefined ? undefined : provenanceDrift(observed.mark); - if (detail !== undefined) { - findings.push({ - detail, - deviceId: device.id, - kind: "foreign-provenance-change", - platform: device.spec.platform, - }); - } - - return findings; -} - -/** - * why: A `reclaiming` record does not say which level started it -- every production path - * asks for `standard`, but that is the caller's choice, not an invariant this finding can - * depend on -- and the two errors are not symmetric: an estimate that is too tight turns a - * healthy reclaim into a false stall finding, while one that is too loose only delays a real - * one. So this takes the slower branch rather than assuming. - */ -function slowestReclaimEstimateMs(driver: Pick, spec: DeviceSpec): number { - return Math.max( - driver.estimate({ clean: "standard", operation: "reclaim" }, spec), - driver.estimate({ clean: "full", operation: "reclaim" }, spec), - ); -} - -/** - * why: A `provisioning` / `reclaiming` device is normally in-flight work Simlock itself is - * driving (see `expectedRunState`), not drift -- but only up to a point. Past a - * driver-derived threshold it stops being "still working" and becomes a stall: the - * driver call that was supposed to resolve it never did. This is a documented failure - * mode, not a hypothetical one -- `DeviceProvisioner` deliberately leaves a device - * `provisioning` when a boot-timeout's own cleanup `destroy` also fails ("The - * registered record remains for reconcile when the driver cannot destroy it"). - * - * The threshold is `driver.estimate(...) * thresholdMultiplier`, floored at - * `minimumThresholdMs` (see `stallThresholdMs`): not `estimate` itself, because the - * estimate is tuned for a routine run and real-world variance (a cold Android boot, a - * loaded host) can legitimately run well past it without anything having stalled. No - * driver for the device's platform, or no recorded entry time (defensive; see - * `transitionEnteredAt`), means there is nothing to compare against, so no stall rather - * than a guess. - * - * A device this daemon holds an operation claim on is excluded outright, before any - * of that arithmetic: the claim *is* the statement that work is in progress, so a - * live operation can never be read as a stall no matter how long it legitimately - * runs. This is the same live-versus-orphaned test `StartupConverger - * #recoverInterruptedReclaims` already makes, and it is load-bearing rather than - * belt-and-braces -- every release now backgrounds its reclaim, holding the device in - * `reclaiming` for a full erase, measured at ~34s, and several such erases run at once - * and contend for the same disk. The estimate the threshold is built from now reflects - * that erase rather than the 1s it used to claim (#56), but tuning a threshold against - * contended disk speed would still be guessing; the claim answers it exactly. A reclaim - * orphaned by a crash has no claim in the new process, so the case this test exists to - * catch is untouched. - */ -export function isStalledTransition(input: StallInput): boolean { - const { claims, config, device, driver, now } = input; - if (device.state !== "provisioning" && device.state !== "reclaiming") { - return false; - } - if (claims?.isClaimed(device.id) === true) { - return false; - } - const enteredAt = transitionEnteredAt(device); - if (enteredAt === undefined || driver === undefined) { - return false; - } - return now - enteredAt > stallThresholdMs(device, driver, config); -} - -export interface StallInput { - readonly device: DeviceRecord; - readonly driver: Pick | undefined; - readonly config: Config["stalledTransition"]; - readonly claims: Pick | undefined; - readonly now: number; -} - -function stallThresholdMs( - device: DeviceRecord, - driver: Pick, - config: Config["stalledTransition"], -): number { - const estimateMs = - device.state === "provisioning" - ? driver.estimate({ operation: "provision" }, device.spec) + - driver.estimate({ operation: "boot" }, device.spec) - : slowestReclaimEstimateMs(driver, device.spec); - return Math.max(estimateMs * config.thresholdMultiplier, config.minimumThresholdMs); -} - -function stalledTransitionFinding( - device: DeviceRecord, - driversByPlatform: ReadonlyMap, - config: Config["stalledTransition"], - now: number, - claims?: Pick, -): DoctorFinding | undefined { - const driver = driversByPlatform.get(device.spec.platform); - if (!isStalledTransition({ claims, config, device, driver, now })) { - return undefined; - } - const enteredAt = transitionEnteredAt(device); - if ( - driver === undefined || - enteredAt === undefined || - (device.state !== "provisioning" && device.state !== "reclaiming") - ) { - return undefined; - } - return { - ageMs: now - enteredAt, - deviceId: device.id, - enteredAt, - kind: "stalled-transition", - platform: device.spec.platform, - state: device.state, - thresholdMs: stallThresholdMs(device, driver, config), - }; -} - -/** - * why: Report-only by design. Re-marking would overwrite the only evidence that the - * device was tampered with, and a fresh mark cannot be written to a leased - * device without disturbing its holder -- so the core never repairs a mark. - */ -function provenanceDrift(mark: ObservedMark): ProvenanceDrift | undefined { - if (mark.durable === undefined) { - return "durable-mark-missing"; - } - if (!mark.erasableReadable) { - return undefined; - } - if (mark.erasable === undefined) { - return "erased"; - } - return mark.erasable === mark.durable ? undefined : "mark-mismatch"; -} - -/** - * why: Discovery runs once, at daemon start, so a refusal is a frozen snapshot: repairing the - * root and running `doctor` again reports the identical finding, because nothing has - * looked at the root since. The finding therefore has to name the step that actually - * retries the platform, or it reads as a repair that did not work. - */ -const DRIVER_RETRY_REMEDY = - "Driver discovery runs once, at daemon startup: restart the daemon (`simlock daemon stop`, then any command relaunches it) once this is fixed, or the platform stays unavailable."; - -function driverUnavailableFindings(rejections: readonly DriverRejection[]): DoctorFinding[] { - return rejections.map((rejection) => ({ - // why: a driver's summary is an error message of any length; the remedy is kept whole. - detail: `${boundDriverSummary(rejection.summary, DRIVER_DETAIL_MAX - DRIVER_RETRY_REMEDY.length - 2)}. ${DRIVER_RETRY_REMEDY}`, - kind: "driver-unavailable" as const, - platform: rejection.platform, - reason: rejection.reason, - })); -} - -const PLATFORM_NAMES: Readonly> = { android: "Android", ios: "iOS" }; - -function isRange(osVersion: string): boolean { - const parsed = parseOsConstraint(osVersion); - return parsed.ok && parsed.constraint.kind === "range"; -} - -// fallow-ignore-next-line complexity -- one finding per short reason, each a fixed message. -function unreachableTarget( - target: WarmTargetFigures, - catalog: DriverCatalogEntry | undefined, -): DoctorFinding[] { - const { model, osVersion, platform, short } = target; - const named = [model, osVersion, target.mode].filter((part) => part !== undefined).join(" / "); - const name = PLATFORM_NAMES[platform]; - if (short === "runtime-missing") { - if (osVersion !== undefined && isRange(osVersion)) { - return [ - { - kind: "warm-pool-target-unreachable", - message: `no installed ${name} runtime satisfies ${osVersion}`, - platform, - reason: short, - remedy: `run simlock component list --platform ${platform} to see the versions, then simlock component install ${platform} for one inside ${osVersion}`, - target: named, - }, - ]; - } - if ( - model !== undefined && - osVersion !== undefined && - catalog?.runtimes.includes(osVersion) === true && - catalog.modelRuntimes[model] !== undefined && - !catalog.modelRuntimes[model].includes(osVersion) - ) { - return [ - { - kind: "warm-pool-target-unreachable", - message: `${model} does not pair with ${name} ${osVersion}`, - platform, - reason: short, - remedy: `run simlock catalog --platform ${platform}`, - target: named, - }, - ]; - } - return [ - { - kind: "warm-pool-target-unreachable", - message: `${name} ${osVersion ?? "runtime"} is not installed`, - platform, - reason: short, - remedy: - osVersion === undefined - ? `run simlock component list --platform ${platform} to see the versions, then simlock component install ${platform} ` - : `run simlock component install ${platform} ${osVersion}`, - target: named, - }, - ]; - } - if (short === "unknown-model") { - return [ - { - kind: "warm-pool-target-unreachable", - message: `${model} is not a known ${name} model`, - platform, - reason: short, - remedy: `run simlock catalog --platform ${platform} to list the models`, - target: named, - }, - ]; - } - return []; -} - -function overLimit(input: { - readonly target: string; - readonly sum: number; - readonly limit: number; - readonly reserve: number; - readonly room: number; - readonly scope: string; - readonly platform?: Platform; -}): DoctorFinding { - const { limit, platform, reserve, room, scope, sum, target } = input; - const held = reserve > 0 ? `, of which warmPool.reserveRunning holds ${reserve}` : ""; - return { - kind: "warm-pool-target-unreachable", - message: `the ${scope}targets want ${sum} running devices, and the ${scope}running limit is ${limit}${held}, leaving room for ${room}`, - reason: "over-limit", - remedy: `lower the counts of the ${scope}warmPool.targets, ${reserve > 0 ? "" : "or "}raise the ${scope}running limit${reserve > 0 ? ", or lower warmPool.reserveRunning" : ""}`, - target, - ...(platform === undefined ? {} : { platform }), - }; -} - -function isDrift(finding: DoctorFinding): boolean { - return ( - finding.kind !== "driver-advisory" && - finding.kind !== "prerequisite-missing" && - finding.kind !== "service-broken" && - finding.kind !== "warm-pool-target-unreachable" && - finding.kind !== "module-unavailable" - ); -} - -function prerequisiteFindings( - platform: Platform, - missing: readonly MissingPrerequisite[], - running: boolean, - rejected: boolean, -): DoctorFinding[] { - if (missing.length === 0) { - return running || rejected - ? [] - : [ - { - kind: "prerequisite-missing", - message: "Every prerequisite is present, but the daemon started without this platform.", - platform, - prerequisite: "daemon-restart", - remedy: DRIVER_RETRY_REMEDY, - }, - ]; - } - return missing.map((found) => ({ - kind: "prerequisite-missing" as const, - message: found.message, - platform, - prerequisite: found.prerequisite, - remedy: running ? found.remedy : `${found.remedy} ${DRIVER_RETRY_REMEDY}`, - })); -} - -function orphanFindings( - realities: readonly { - readonly driver: Driver; - readonly reality: { - readonly devices: readonly DriverDevice[]; - readonly processes: readonly DriverDevice[]; - }; - }[], - registryDeviceKeys: ReadonlySet, -): DoctorFinding[] { - const findings: DoctorFinding[] = []; - for (const { driver, reality } of realities) { - for (const device of reality.devices) { - if (!registryDeviceKeys.has(key(driver.platform, device.deviceId))) { - findings.push({ device, kind: "orphan-device", platform: driver.platform }); - } - } - for (const device of reality.processes) { - if (!registryDeviceKeys.has(key(driver.platform, device.deviceId))) { - findings.push({ device, kind: "orphan-process", platform: driver.platform }); - } - } - } - return findings; -} - -function expiredLeaseFindings( - leases: readonly { - readonly id: string; - readonly deviceId: string; - readonly ttlDeadline: number; - }[], - now: number, -): DoctorFinding[] { - return leases - .filter((lease) => lease.ttlDeadline <= now) - .map((lease) => ({ - deviceId: lease.deviceId, - kind: "expired-live-lease" as const, - leaseId: lease.id, - })); -} - -function expectedRunState(state: DeviceState): "running" | "stopped" | undefined { - switch (state) { - case "ready": - case "leased": - return "running"; - case "shutdown": - return "stopped"; - case "provisioning": - case "reclaiming": - case "quarantined": - case "absent": - case "deleted": - return undefined; - } -} - -function key(platform: string, deviceId: string): string { - return `${platform}:${deviceId}`; -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} From 49ab97f8c85a8a88ad78436241cbec6217c35793 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:06:50 +0100 Subject: [PATCH 05/83] chore(agents): the comment lint excuses a block moved unchanged from another file --- .agents/scripts/comments.mjs | 27 ++++++++++++++++++++++++++- src/comment-lint.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/.agents/scripts/comments.mjs b/.agents/scripts/comments.mjs index 05c148efe..b2f034ee7 100755 --- a/.agents/scripts/comments.mjs +++ b/.agents/scripts/comments.mjs @@ -29,7 +29,32 @@ export function listComments({ base, head, all = false, files = null }) { const kinds = SHELL.test(path) ? shellKinds(lines) : scriptKinds(lines); blocks.push(...blocksIn(path, lines, kinds, added, all)); } - return blocks; + if (all) return blocks; + const removed = removedLines(base, head); + return blocks.filter((block) => !movedVerbatim(block, removed)); +} + +// why: a split moves code between files unchanged, so a block the diff adds whole while it +// removes the same lines elsewhere is not new; one removed line excuses one added line. +function movedVerbatim(block, removed) { + if (block.state !== "added") return false; + const wanted = new Map(); + for (const { text } of block.lines) wanted.set(text.trim(), (wanted.get(text.trim()) ?? 0) + 1); + if ([...wanted].some(([text, n]) => (removed.get(text) ?? 0) < n)) return false; + for (const [text, n] of wanted) removed.set(text, removed.get(text) - n); + return true; +} + +function removedLines(base, head) { + const from = git("merge-base", base, head ?? "HEAD").trim(); + const range = head === undefined ? [from] : [from, head]; + const removed = new Map(); + for (const line of git("diff", "-U0", "--no-color", ...range).split("\n")) { + if (!line.startsWith("-") || line.startsWith("--- ")) continue; + const text = line.slice(1).trim(); + removed.set(text, (removed.get(text) ?? 0) + 1); + } + return removed; } export function changedFiles({ base, head, files = null }) { diff --git a/src/comment-lint.test.ts b/src/comment-lint.test.ts index f9b10c466..a414161fc 100644 --- a/src/comment-lint.test.ts +++ b/src/comment-lint.test.ts @@ -89,6 +89,30 @@ describe("comment-lint.mjs", () => { expect(result.lines).toEqual([`scripts/run.mjs:4: ${REASON}`, `src/a.ts:5: ${REASON}`]); }); + it("accepts a block moved unchanged from another file, and rejects one whose moved text was edited", () => { + const dir = repo( + { + "src/big.ts": [ + "// Says what a is.", + "const a = 1;", + "// Says what b is.", + "const b = 2;", + "", + ].join("\n"), + }, + { + "src/big.ts": "", + "src/a.ts": " // Says what a is.\nconst a = 1;\n", + "src/b.ts": "// Says what b was.\nconst b = 2;\n", + }, + ); + + const result = lint(dir); + + expect(result.status).toBe(1); + expect(result.lines).toEqual([`src/b.ts:1: ${REASON}`]); + }); + it("accepts a doc comment on an export the client or admin entry point re-exports, and rejects one elsewhere", () => { const dir = repo( {}, From 28bf90526986217285c85e7a8a154e534b1d4ecc Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:05:01 +0100 Subject: [PATCH 06/83] refactor(core): split config.ts into parts Pure structure split: types, defaults, role checks, validators and generic helpers move to sibling files; config.ts keeps the entry point and every previous export. --- src/core/config-defaults.ts | 177 +++++ src/core/config-helpers.ts | 31 + src/core/config-role-checks.ts | 280 ++++++++ src/core/config-types.ts | 310 ++++++++ src/core/config-validators.ts | 401 +++++++++++ src/core/config.ts | 1225 ++------------------------------ 6 files changed, 1248 insertions(+), 1176 deletions(-) create mode 100644 src/core/config-defaults.ts create mode 100644 src/core/config-helpers.ts create mode 100644 src/core/config-role-checks.ts create mode 100644 src/core/config-types.ts create mode 100644 src/core/config-validators.ts diff --git a/src/core/config-defaults.ts b/src/core/config-defaults.ts new file mode 100644 index 000000000..b2fa3bf9b --- /dev/null +++ b/src/core/config-defaults.ts @@ -0,0 +1,177 @@ +import type { AuthStrategyDefinition } from "../auth/index.js"; +import type { SystemStats } from "../ports/index.js"; +import { + defaultCapacityOptions, + type CapacityConfig, + type CapacityStrategyName, +} from "./capacity/index.js"; +import type { Platform } from "./domain.js"; +import type { Config, DaemonMode, DriverConfigEntry, ModuleConfigEntry } from "./config-types.js"; +import { defaultWarmPoolConfig } from "./warm-pool/index.js"; + +export const DEFAULT_CONFIG_PATH = "~/.simlock/config.json"; + +const DEFAULT_LEASE_TTL_MS = 15 * 60_000; + +const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; + +/** why: `devices.deletedRetentionMs`'s default: how long a `deleted` device record is kept, counted from its deletion. */ +export const DEFAULT_DEVICES_DELETED_RETENTION_MS = 60 * 60_000; + +export const DEFAULT_LEASE_REQUEST_RETENTION_MS = 10 * 60_000; + +/** why: `lease.endedIdRetentionMs`'s default: how long a chosen lease id is refused after its lease ended (ADR 0074 §4). */ +export const DEFAULT_LEASE_ENDED_ID_RETENTION_MS = 24 * 60 * 60_000; + +/** why: `gateway.admission.timeoutMs`'s default (ADR 0074 §1): a policy script answers in well under this, and a stuck one refuses requests for no longer. */ +const DEFAULT_ADMISSION_TIMEOUT_MS = 5_000; + +/** why: the bounds on `gateway.admission`, checked when the config is read. */ +export const ADMISSION_TIMEOUT_MAX_MS = 60_000; +export const ADMISSION_COMMAND_MAX_ENTRIES = 64; +export const ADMISSION_ARGUMENT_MAX_LENGTH = 4096; + +const DEFAULT_LEASE_MAX_PER_REQUESTER = 1; + +export const DEFAULT_LEASE_MAX_REQUEST_RECORDS = 10_000; + +const DEFAULT_DISCONNECTED_RETENTION_MS = 24 * 60 * 60_000; + +/** why: `gateway.execTimeoutMs`'s default (ADR 0005 §19e): eleven minutes, one more than the + * worker's authoritative ten, so the worker's timeout is the one that fires. */ +const DEFAULT_GATEWAY_EXEC_TIMEOUT_MS = 11 * 60_000; + +/** why: `gateway.leaseRequestTimeoutMs`'s default (P2, round 2 review): five minutes -- generous + * against a cold device provision-plus-boot, and well short of `execTimeoutMs`, since granting a + * lease should never take as long as a command a caller runs against the device afterward. */ +const DEFAULT_LEASE_REQUEST_TIMEOUT_MS = 5 * 60_000; + +const DEFAULT_ROUTING_POLICY = "warm-then-free"; + +/** + * why: `exec.timeoutMs`'s default (ADR 0005 §19e): how long a single `device.exec` command may run + * on this worker before it is killed and the operation fails with `EXEC_TIMEOUT`. Ten minutes, + * because the commands this wraps are installs and boots-worth of `adb wait-for-device`, not + * sub-second reads -- and because a stuck command holds no lease of its own, so the cost of a + * generous bound is one process, not a device. + */ +const DEFAULT_EXEC_TIMEOUT_MS = 10 * 60_000; + +export const DEFAULT_METRICS_RETENTION_MS = 90 * 24 * 60 * 60 * 1000; + +export const MIN_EVENT_LOG_MAX_BYTES = 16 * 1024 * 1024; + +export function defaultConfig( + systemStats: SystemStats, + strategy: CapacityStrategyName, + role: DaemonMode, + modules: readonly ModuleConfigEntry[], + authStrategies: readonly AuthStrategyDefinition[], + standalone: boolean, + defaulted: ReadonlySet, + driverSections: readonly DriverConfigEntry[], +): Omit { + return { + mode: role, + capabilities: Object.fromEntries( + modules.map((entry) => [ + entry.name, + { ...(entry.defaults as Record), enabled: defaulted.has(entry.name) }, + ]), + ) as Config["capabilities"], + capacity: { + strategy, + config: defaultCapacityOptions(strategy, systemStats), + } as CapacityConfig, + drivers: {}, + downloads: { + policy: "on-request", + acceptAndroidLicenses: false, + timeoutMs: 1_200_000, + }, + idle: { + shutdownAfterMs: 10 * 60_000, + deleteAfterMs: 60 * 60_000, + }, + update: { check: true }, + warmPool: { + ...defaultWarmPoolConfig(systemStats), + quarantine: { + maxRetries: 3, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + maxRetryBackoffMs: 5 * 60_000, + }, + }, + devices: { autoEnroll: false, deletedRetentionMs: DEFAULT_DEVICES_DELETED_RETENTION_MS }, + lease: { + defaultTtlMs: DEFAULT_LEASE_TTL_MS, + maxTtlMs: DEFAULT_LEASE_MAX_TTL_MS, + identity: { ios: "reusable", android: "reusable" }, + requestRetentionMs: DEFAULT_LEASE_REQUEST_RETENTION_MS, + endedIdRetentionMs: DEFAULT_LEASE_ENDED_ID_RETENTION_MS, + maxRequestRecords: DEFAULT_LEASE_MAX_REQUEST_RECORDS, + maxPerRequester: DEFAULT_LEASE_MAX_PER_REQUESTER, + endWaitMs: DEFAULT_EXEC_TIMEOUT_MS, + }, + exec: { timeoutMs: DEFAULT_EXEC_TIMEOUT_MS }, + diskPressure: { freeBytesThreshold: 10 * 1024 ** 3 }, + memoryPressure: { + enabled: true, + pauseWarming: { level: "warn", minAvailablePercent: 20 }, + pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, + clearAfterMs: 60_000, + }, + eventBuffer: { capacity: 1_000 }, + log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, + eventLog: { + retention: 7 * 24 * 60 * 60 * 1000, + maxBytes: 256 * 1024 * 1024, + }, + // why: ADR 0005 §2 and ADR 0046 §1: a gateway and a standalone worker listen on HTTP by default; a joined worker is reached through its gateway, so its default is off. + http: { + enabled: role === "gateway" || standalone, + host: "127.0.0.1", + port: role === "gateway" ? 4700 : 4701, + ...Object.fromEntries( + authStrategies.flatMap((definition) => + definition.config === undefined + ? [] + : [[definition.config.key, definition.config.defaults]], + ), + ), + }, + health: { + enabled: true, + probeIntervalMs: 30_000, + stableObservations: 2, + maxRecoveryAttempts: 3, + recoveryBackoffMs: 5_000, + maxConcurrentRecoveries: 1, + }, + ios: { ...driverDefaults(driverSections, "ios"), defaultMode: "full", defaultModels: {} }, + android: { ...driverDefaults(driverSections, "android"), defaultModels: {} }, + gateway: { + disconnectedRetentionMs: DEFAULT_DISCONNECTED_RETENTION_MS, + execTimeoutMs: DEFAULT_GATEWAY_EXEC_TIMEOUT_MS, + leaseRequestTimeoutMs: DEFAULT_LEASE_REQUEST_TIMEOUT_MS, + routing: DEFAULT_ROUTING_POLICY, + alerts: { destinations: [] }, + admission: { timeoutMs: DEFAULT_ADMISSION_TIMEOUT_MS }, + }, + stalledTransition: { + thresholdMultiplier: 3, + minimumThresholdMs: 60_000, + }, + }; +} + +function driverDefaults( + driverSections: readonly DriverConfigEntry[], + platform: Platform, +): Record { + return Object.assign( + {}, + ...driverSections.filter((entry) => entry.platform === platform).map((entry) => entry.defaults), + ) as Record; +} diff --git a/src/core/config-helpers.ts b/src/core/config-helpers.ts new file mode 100644 index 000000000..b5c01cbbe --- /dev/null +++ b/src/core/config-helpers.ts @@ -0,0 +1,31 @@ +import type { Layer } from "./config-types.js"; + +export function pick(layer: Layer, keys: readonly string[]): Layer { + return Object.fromEntries( + keys.filter((key) => layer[key] !== undefined).map((key) => [key, layer[key]]), + ); +} + +export function omit(layer: Layer, keys: readonly string[]): Layer { + return Object.fromEntries(Object.entries(layer).filter(([key]) => !keys.includes(key))); +} + +export function asObject(value: unknown): Layer { + return isObject(value) ? value : {}; +} + +export function mergeConfig(base: Base, overrides: object): Base { + const merged: Record = { ...(base as Record) }; + + for (const [key, override] of Object.entries(overrides)) { + const current = merged[key]; + merged[key] = + isObject(current) && isObject(override) ? mergeConfig(current, override) : override; + } + + return merged as Base; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/src/core/config-role-checks.ts b/src/core/config-role-checks.ts new file mode 100644 index 000000000..8330f82f2 --- /dev/null +++ b/src/core/config-role-checks.ts @@ -0,0 +1,280 @@ +import { + DEFAULT_CAPACITY_STRATEGY, + capacityStrategyNames, + isCapacityStrategyName, + type CapacityStrategyName, +} from "./capacity/index.js"; +import type { Config, DownloadPolicy, Layer, ModuleConfigEntry } from "./config-types.js"; +import { MIN_EVENT_LOG_MAX_BYTES } from "./config-defaults.js"; +import { asObject, mergeConfig, omit, pick } from "./config-helpers.js"; +import { invalidValue, requireObject, type Warn } from "./validation.js"; + +/** + * why: The one consent function (ADR 0010 §4): reduces `downloads.policy` and what the caller asked + * for to whether a download may start. A lease request passes its own `--allow-download` / + * `allowDownload` flag; `component.install` passes `true`, because the command itself is the + * consent. `never` overrides an explicit `true` -- the whole point of the policy is that no + * request and no operation can opt back into it -- and `always` grants permission the request + * never had to ask for. Only `on-request` defers to what the caller asked for. + */ +export function effectiveAllowDownload(policy: DownloadPolicy, requested: boolean): boolean { + if (policy === "always") return true; + if (policy === "never") return false; + return requested; +} + +/** + * why: Cross-field check (ADR 0004's Consequences): the default TTL a request falls back to cannot + * exceed the cap every request is measured against, or every defaulted request would be born + * asking for more than the daemon will grant. A violating config fails the daemon start naming + * the offending key rather than being clamped to a value the operator never wrote -- the + * opposite treatment from a retired key, which is a leftover with a safe reading ("ignore it") + * where a self-contradicting TTL pair has none. + */ +export function validateLeaseTtls(config: Config): void { + if (config.lease.defaultTtlMs > config.lease.maxTtlMs) { + throw invalidValue("lease.defaultTtlMs", "at most lease.maxTtlMs"); + } +} + +export function validateDeletedRetention(config: Config): void { + if (config.devices.deletedRetentionMs < config.lease.requestRetentionMs) { + throw invalidValue("devices.deletedRetentionMs", "at least lease.requestRetentionMs"); + } +} + +export function validateEventLogBounds(config: Config): void { + if (config.eventLog.maxBytes < MIN_EVENT_LOG_MAX_BYTES) { + throw invalidValue("eventLog.maxBytes", "at least 16 MiB"); + } +} + +export function validateMetricsRetention(config: Config): void { + if (config.metrics.retention < config.eventLog.retention) { + throw invalidValue("metrics.retention", "at least eventLog.retention"); + } +} + +/** + * why: The config keys a gateway reads (ADR 0005 §2). Everything else in a gateway's config is a + * worker-only key: it configures drivers, devices, capacity, or cleanup, none of which a + * gateway has. Listed here rather than derived from `Config`'s shape because the *set* is the + * decision -- adding a key to `Config` should not silently make it gateway-relevant. + */ +const GATEWAY_CONFIG_KEYS: readonly string[] = [ + "http", + "log", + "lease", + "eventBuffer", + "eventLog", + "metrics", + "gateway", + "capabilities", + "update", +]; + +const WORKER_ONLY_GATEWAY_KEYS: readonly string[] = ["url", "token", "label", "headers"]; + +/** + * why: ADR 0005 §2: "worker-only keys in a gateway's config are ignored with a warning, as unknown + * keys are today". Warned per key the operator actually wrote (the file and override layers), + * never per defaulted key -- a gateway's own defaults contain the whole worker-side `Config` + * shape, and warning about those would fire on every start with nothing to fix. + * + * Ignoring rather than failing is the deliberate choice the ADR makes, and it is what lets one + * config file be copied between a worker and a gateway while only `mode` differs. + */ +export function warnWorkerOnlyKeys( + layers: readonly unknown[], + warn: Warn, + modules: readonly ModuleConfigEntry[], +): void { + for (const layer of layers) { + const object = requireObject(layer, "config"); + for (const key of Object.keys(object)) { + if (!GATEWAY_CONFIG_KEYS.includes(key)) { + warn(`Ignoring "${key}": it configures a worker, and this daemon runs in gateway mode.`); + } + } + warnWorkerOnlySubKeys( + object, + "gateway", + WORKER_ONLY_GATEWAY_KEYS, + warn, + (key) => + `Ignoring "gateway.${key}": it points a worker at its gateway, ` + + `and this daemon runs in gateway mode.`, + ); + warnWorkerOnlyModuleKeys(object, modules, warn); + } +} + +function warnWorkerOnlyModuleKeys( + layer: Record, + modules: readonly ModuleConfigEntry[], + warn: Warn, +): void { + const sections = layer["capabilities"]; + if (sections === undefined) return; + for (const [name, section] of Object.entries(requireObject(sections, "capabilities"))) { + const entry = modules.find((candidate) => candidate.name === name); + if (entry === undefined) continue; + for (const key of Object.keys(requireObject(section, `capabilities.${name}`))) { + if (key !== "enabled" && !entry.gatewayKeys.includes(key)) { + warn( + `Ignoring "capabilities.${name}.${key}": it configures a worker, ` + + `and this daemon runs in gateway mode.`, + ); + } + } + } +} + +export function runModuleChecks(config: Config, modules: readonly ModuleConfigEntry[]): void { + for (const entry of modules) { + entry.check?.(config.capabilities[entry.name] as never, config); + } +} + +function warnWorkerOnlySubKeys( + layer: Record, + section: string, + workerOnly: readonly string[], + warn: Warn, + message: (key: string) => string, +): void { + const value = layer[section]; + if (value === undefined) return; + for (const key of Object.keys(requireObject(value, section))) { + if (workerOnly.includes(key)) warn(message(key)); + } +} + +/** + * why: ADR 0005 §2: "a gateway always listens on HTTP (it is the fleet's contact point) and on its + * unix socket". HTTP is not optional for a gateway in a way it is for a worker: it is how + * every agent reaches the fleet *and* the transport the worker uplink upgrades from + * (`/v1/uplink`), so a gateway with `http.enabled: false` is a process nothing can reach and + * no worker can join. + * + * The default is flipped to `true` for a gateway (see `defaultConfig`), so this only ever + * fires for a config that says `false` out loud. That is refused rather than silently + * overridden, on the same reasoning as `lease.defaultTtlMs > lease.maxTtlMs`: a value the + * operator wrote and the daemon quietly inverted is worse than a start that says why. + */ +export function requireGatewayHttp(config: Config): void { + if (!config.http.enabled) { + throw invalidValue("http.enabled", "true in gateway mode (a gateway is reached over HTTP)"); + } +} + +/** + * why: A worker's fleet membership, checked at load rather than at dial time so a typo fails the + * daemon start naming the key instead of surfacing as an endless reconnect loop hours later. + * + * `gateway.url` is the gateway's **base** URL (`ws://host:port`); the worker derives the + * uplink endpoint (`/v1/uplink`) from it, so nothing here inspects the path -- a URL that + * already carries one is accepted and the endpoint is resolved against it, which is what makes + * a reverse proxy on a sub-path work. + * + * The two keys are required together: a URL with no token cannot authenticate and a token with + * no URL has nowhere to go, and either alone is an operator half-finishing a join. Failing the + * start is right where ignoring would be wrong -- a worker that silently never joined would + * look identical to one whose gateway is down. + */ +export function validateGatewayMembership(config: Config): void { + const { token, url } = config.gateway; + if (url !== undefined) requireWebSocketUrl(url); + if (url !== undefined && token === undefined) { + throw invalidValue("gateway.token", "a join token, since gateway.url is set"); + } + if (token !== undefined && url === undefined) { + throw invalidValue("gateway.url", "a gateway URL, since gateway.token is set"); + } + const { headers } = config.gateway; + if (url === undefined && headers !== undefined && Object.keys(headers).length > 0) { + throw invalidValue( + "gateway.headers", + "gateway.url to be set too, since gateway.headers is set", + ); + } + requireJoinedWorkerHttpOff(config); +} + +function requireWebSocketUrl(url: string): void { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw invalidValue("gateway.url", "a ws:// or wss:// URL"); + } + if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") { + throw invalidValue("gateway.url", "a ws:// or wss:// URL"); + } +} + +// why: a joined worker is reached through the gateway; its own listener would be a second entrance. +function requireJoinedWorkerHttpOff(config: Config): void { + if (config.gateway.url !== undefined && config.http.enabled) { + throw invalidValue( + "http.enabled", + "false when gateway.url is set (joined workers are reached through the gateway)", + ); + } +} + +export function resolveStrategyName(layers: readonly unknown[]): CapacityStrategyName { + let resolved: CapacityStrategyName = DEFAULT_CAPACITY_STRATEGY; + + for (const layer of layers) { + const capacity = requireObject(layer, "config")["capacity"]; + if (capacity === undefined) continue; + + const candidate = requireObject(capacity, "capacity")["strategy"]; + if (candidate === undefined) continue; + + if (!isCapacityStrategyName(candidate)) { + throw invalidValue( + "capacity.strategy", + `one of ${capacityStrategyNames.map((name) => `"${name}"`).join(", ")}`, + ); + } + + resolved = candidate; + } + + return resolved; +} + +const LEGACY_RESOURCE_KEYS = ["limits", "ramBudget"] as const; + +/** + * why: Folds the legacy top-level keys into `capacity.config`. Runs per layer, before + * merging, so precedence between layers is unaffected by which spelling each one + * happens to use. Within a layer `capacity.config` wins. + */ +export function normalizeLayer(layer: Layer, strategy: CapacityStrategyName, warn: Warn): Layer { + const legacy = pick(layer, LEGACY_RESOURCE_KEYS); + const present = Object.keys(legacy); + if (present.length === 0) return layer; + + const rest = omit(layer, LEGACY_RESOURCE_KEYS); + if (strategy !== "resource") { + warn(legacyIgnoredMessage(present, strategy)); + return rest; + } + + const capacity = asObject(rest["capacity"]); + return { + ...rest, + capacity: { ...capacity, config: mergeConfig(legacy, asObject(capacity["config"])) }, + }; +} + +function legacyIgnoredMessage(keys: readonly string[], strategy: CapacityStrategyName): string { + const subject = keys.length > 1 ? "these keys configure" : "this key configures"; + return ( + `Ignoring ${keys.join(" and ")}: ${subject} the "resource" capacity strategy, ` + + `but "${strategy}" is selected. Move the settings under "capacity.config".` + ); +} diff --git a/src/core/config-types.ts b/src/core/config-types.ts new file mode 100644 index 000000000..9c9d06951 --- /dev/null +++ b/src/core/config-types.ts @@ -0,0 +1,310 @@ +import type { AuthStrategyDefinition } from "../auth/index.js"; +import type { Filesystem, LogLevel, SystemStats } from "../ports/index.js"; +import type { CapacityConfig, CapacityLimits, ResourceStrategyOptions } from "./capacity/index.js"; +import type { DeviceClass, DeviceMode, LeaseIdentity, Platform } from "./domain.js"; +import type { AlertDestination } from "./config-alerts.js"; +import type { Validator, Warn } from "./validation.js"; +import type { WarmPoolConfig } from "./warm-pool/index.js"; + +/** + * why: `gateway.routing`'s valid names (ADR 0005 §13/Decision 7) and its default. Declared here + * rather than imported from `src/gateway/routing.ts`'s own registry: `core` stays + * platform-agnostic and gateway-unaware (`AGENTS.md`/`architecture.md`), so this list is + * re-declared the same way `src/contract/schemas.ts` re-declares it independently rather than + * importing across its own boundary -- three places name `"warm-then-free"`, the same + * redundancy `capacity.strategy`'s "fixed"/"resource" already carries between `core` and the + * contract. Adding a second policy means adding it here, in the gateway's own registry, and in + * the contract schema -- one line each, never a shared import. + */ +export const ROUTING_POLICIES = ["warm-then-free"] as const; + +/** why: The literal union `ROUTING_POLICIES` names -- `Config["gateway"]["routing"]`'s type, so it + * agrees with the contract's own `z.enum(["warm-then-free"])` (also re-declared, never + * imported) instead of widening to a bare `string` the boundary between the two would otherwise + * force. */ +type RoutingPolicyName = (typeof ROUTING_POLICIES)[number]; + +export type DownloadPolicy = "never" | "on-request" | "always"; + +/** + * why: one number counts a requester's leases across all platforms; the object counts per + * platform, and a platform it leaves out is 1. + */ +export type MaxPerRequester = number | { readonly ios?: number; readonly android?: number }; + +/** + * why: which shape a daemon runs as (ADR 0005 §1): a `worker` owns the devices on its machine and a + * `gateway` owns none and fronts the workers that joined it. One daemon runs exactly one mode, + * and the folder it lives in decides it (ADR 0034): the daemon is told its role at start, and + * `loadConfig` takes it as an argument -- no config key names it. + */ +export type DaemonMode = "worker" | "gateway"; + +interface MemoryPressureRule { + readonly level: "normal" | "warn" | "critical" | "off"; + readonly minAvailablePercent: number; +} + +export interface Config { + readonly capabilities: Readonly< + Record>> + >; + readonly mode: DaemonMode; + readonly capacity: CapacityConfig; + /** + * why: Per-driver settings, opaque to the core: stored, merged, and handed to the driver + * that owns the key without a single key being interpreted here. What + * `drivers.ios.deviceRoot` means is the iOS driver's business, and knowing it here + * would be exactly the leak architecture rule 2 forbids. + */ + readonly drivers: Readonly>>>; + readonly downloads: { + readonly policy: DownloadPolicy; + /** why: Explicit legal consent for Android SDK licenses, independent of `policy`. */ + readonly acceptAndroidLicenses: boolean; + /** + * why: One budget per install request, from its arrival, covering the wait behind another + * download and the download itself (ADR 0010 §3); downloads run minutes, not seconds. + */ + readonly timeoutMs: number; + }; + readonly idle: { + readonly shutdownAfterMs: number; + readonly deleteAfterMs: number; + }; + readonly update: { + readonly check: boolean; + }; + readonly warmPool: WarmPoolConfig & { + readonly quarantine: { + readonly maxRetries: number; + readonly retryBackoffMs: number; + readonly retryBackoffMultiplier: number; + readonly maxRetryBackoffMs: number; + }; + }; + readonly devices: { + /** + * why: how long a `deleted` device record stays in `state.db` and in `status.get`, counted + * from its deletion. At least `lease.requestRetentionMs`: a granted request names its device. + */ + readonly deletedRetentionMs: number; + /** + * why: a worker enrols the phones plugged into it by USB, as `device add` would; off by default + * because a personal phone on the worker would be wiped of apps at every lease's end. + */ + readonly autoEnroll: boolean; + }; + readonly lease: { + /** + * why: ADR 0004 §4: the TTL applied to a lease whose `lease.request` carried no `ttlMs` -- that + * request only. It is deliberately *not* the renew fallback: a `lease.renew` given no + * explicit TTL re-applies the lease's own stored width, so a lease granted for longer + * keeps it. + */ + readonly defaultTtlMs: number; + /** why: The largest TTL a request or a renew may ask for; more is `BAD_REQUEST`, never a + * silent clamp, so a caller is never left believing it has more time than it does. */ + readonly maxTtlMs: number; + /** + * why: The lease-identity policy per platform (see `LeaseIdentity`) no longer changes anything: + * every virtual device serves one lease and is deleted when it ends (ADR 0061 §2). It stays + * accepted and stored on each device record. + */ + readonly identity: { + readonly ios: LeaseIdentity; + readonly android: LeaseIdentity; + }; + readonly requestRetentionMs: number; + /** why: ADR 0074 §4: how long a chosen lease id is refused after its lease ended, counted from the end; read by a worker and a gateway. */ + readonly endedIdRetentionMs: number; + readonly maxRequestRecords: number; + readonly maxPerRequester: MaxPerRequester; + readonly endWaitMs: number; + }; + /** + * why: ADR 0005 §19e. Platform-agnostic on purpose: it bounds the *daemon's* willingness to wait + * on a child, not anything either driver knows about, so it sits at the top level rather + * than under `drivers.*`. + */ + readonly exec: { readonly timeoutMs: number }; + readonly diskPressure: { readonly freeBytesThreshold: number }; + /** why: ADR 0068 §1: when host memory pauses a worker's warming and its new devices; worker only. */ + readonly memoryPressure: { + readonly enabled: boolean; + readonly pauseWarming: MemoryPressureRule; + readonly pauseNewDevices: MemoryPressureRule; + readonly clearAfterMs: number; + }; + readonly eventBuffer: { readonly capacity: number }; + readonly log: { readonly level: LogLevel; readonly rotateBytes: number }; + readonly eventLog: { + readonly retention: number; + readonly maxBytes: number; + }; + readonly metrics: { readonly retention: number }; + readonly http: { + readonly enabled: boolean; + readonly host: string; + readonly port: number; + readonly proxyAuth?: { + readonly header: string; + readonly jwksUrl: string; + readonly issuer: string; + readonly audience: string; + readonly signOutUrl?: string; + }; + }; + readonly health: { + readonly enabled: boolean; + readonly probeIntervalMs: number; + readonly stableObservations: number; + readonly maxRecoveryAttempts: number; + readonly recoveryBackoffMs: number; + readonly maxConcurrentRecoveries: number; + }; + readonly ios: { + readonly defaultMode: DeviceMode; + readonly defaultModels: DefaultModels; + } & DriverConfigSection; + readonly android: { + readonly defaultModels: DefaultModels; + } & DriverConfigSection; + /** + * why: ADR 0005 §3/§6. Both sides of the fleet live under one key, and which half is read depends + * on `mode`: + * + * - a **worker** reads `url`, `token`, and `label` -- the two keys that join a fleet, plus a + * display name. Absent `url`/`token` simply means "this worker has no gateway", which is + * every worker today. + * - a **gateway** reads `disconnectedRetentionMs` and ignores the other three with a + * warning (`loadConfig` reports every worker-only key it finds in a gateway's config). + * + * Kept as one block rather than `gateway.*` / `worker.*` blocks because both halves are + * facts about the same relationship, and a `worker.gateway.url` spelling reads worse than + * the thing it configures. + */ + readonly gateway: { + readonly url?: string; + /** why: A join token minted on the gateway with `simlock token create --role worker`. + * Worker-side. It is a secret, and it sits in the daemon's config file, which `simlock + * config set` writes owner-only (0600) every time; a file written by hand, or before + * that, keeps whatever mode it has until the next `config set`. Everywhere else the + * config is written out -- the startup log line, `daemon.started`, and `config.get` + * unless it is asked with `showSecrets: true` -- it goes through `redactConfig`. */ + readonly token?: string; + readonly label?: string; + readonly headers?: Readonly>; + /** + * why: How long a gateway keeps a disconnected worker's view before forgetting it (§6). + * Gateway-side. Default 24 hours: long enough that a machine rebooting overnight is still + * the same machine in the morning, short enough that a decommissioned one does not haunt + * `simlock status` forever. A view whose leases have not all expired is never dropped on + * this timer, however long it has been gone. + */ + readonly disconnectedRetentionMs: number; + /** + * why: Gateway-side backstop on a proxied `device.exec` (ADR 0005 §19e). Deliberately longer + * than the worker's own `exec.timeoutMs` (ten minutes): the worker's is authoritative and + * should be what kills a runaway command, so a gateway that timed out first would report + * a failure for a command still running on the machine that owns the device. Read by #118, + * which is what proxies `device.exec`; declared now so the key does not appear mid-series. + */ + readonly execTimeoutMs: number; + /** + * why: P2 (round 2 review): bounds a forwarded `lease.request` -- the one uplink call that used + * to have no timeout of its own, parking a waiter `processing` where neither `timeoutMs`'s + * deadline check nor `lease.cancel` could reach it (`WaitQueue#armTimeout` declines to + * reject a `processing` waiter, and `cancelPending` answers `not-cancellable`) if the + * worker's own `lease.request` handler ever wedged. Generous relative to how long granting a + * lease can plausibly take -- provisioning plus a cold device boot -- and bounded well below + * `execTimeoutMs`, since granting a lease should never take as long as an arbitrary command + * a caller runs afterward might. Expiry maps to `WORKER_UNREACHABLE`, returning the waiter + * to a state `enqueue`'s deadline check and `cancelPending` can both act on again. + */ + readonly leaseRequestTimeoutMs: number; + readonly routing: RoutingPolicyName; + readonly alerts: { readonly destinations: readonly AlertDestination[] }; + /** + * why: ADR 0074 §1: the program a gateway runs for each new lease request, and how long it may + * take; a worker ignores both, as it does `execTimeoutMs`. + */ + readonly admission: { readonly command?: readonly string[]; readonly timeoutMs: number }; + }; + readonly stalledTransition: { + /** + * why: Applied to the driver's own estimate (`provision + boot` for `provisioning`, + * `reclaim` for `reclaiming`) to get the stall threshold. Deliberately generous: + * the estimate itself is already tuned for a routine run, so the threshold has to + * clear real-world variance on top of it, not just match it -- a cold Android + * provision-plus-boot can legitimately run well past the raw estimate. + */ + readonly thresholdMultiplier: number; + /** why: Floor under the multiplied estimate, for a driver whose estimate is near zero. */ + readonly minimumThresholdMs: number; + }; +} + +/** + * why: The pre-`capacity` spelling of the resource strategy's options. Still accepted + * everywhere the new shape is, and folded into `capacity.config` before anything + * downstream sees it, so existing config files keep working untouched. + */ +export interface LegacyCapacityOverrides { + readonly limits?: DeepPartial; + readonly ramBudget?: DeepPartial; +} + +export type ConfigOverrides = DeepPartial & LegacyCapacityOverrides; + +// why: a platform's section also holds its driver's own keys, which the loader carries without reading one. +type DriverConfigSection = Readonly>; + +// why: a driver declares the keys it reads under its platform's section, so a new driver setting changes no file of the core; `defaults` holds every key that has one, typed as the driver reads the section. +export interface DriverConfigEntry
{ + readonly platform: Platform; + readonly validators: Readonly>; + readonly defaults: Section; + // why: the values the driver uses for the `drivers..*` keys config.json leaves unset, which depend on the data directory and on the keys that are set; `config.get` shows them, so `config get` reads a key at its default. + readonly settingDefaults?: ( + simlockHome: string, + settings: Readonly>, + ) => Readonly>; +} + +export interface ModuleConfigEntry { + readonly name: string; + readonly validator: Validator; + readonly defaults: unknown; + readonly gatewayKeys: readonly string[]; + readonly enabledByDefault?: "standalone-worker"; + readonly check?: (section: never, config: Config) => void; +} + +export interface ConfigLayerOptions { + readonly authStrategies?: readonly AuthStrategyDefinition[]; + readonly driverSections?: readonly DriverConfigEntry[]; + readonly modules?: readonly ModuleConfigEntry[]; + readonly systemStats: SystemStats; + readonly overrides?: ConfigOverrides; + readonly role?: DaemonMode; + readonly warn?: Warn; +} + +export interface LoadConfigOptions extends ConfigLayerOptions { + readonly filesystem: Filesystem; + readonly configPath?: string; +} + +export type Layer = Record; + +export interface LoadedConfig { + readonly config: Config; + readonly defaulted: ReadonlySet; +} + +export type DefaultModels = Partial>; + +export type DeepPartial = Value extends object + ? { readonly [Key in keyof Value]?: DeepPartial } + : Value; diff --git a/src/core/config-validators.ts b/src/core/config-validators.ts new file mode 100644 index 000000000..20cdb70b0 --- /dev/null +++ b/src/core/config-validators.ts @@ -0,0 +1,401 @@ +import type { ZodType } from "zod"; + +import type { LogLevel } from "../ports/index.js"; +import type { AuthStrategyDefinition } from "../auth/index.js"; +import { + capacityStrategyNames, + capacityStrategyValidator, + resourceOptionValidators, + type CapacityStrategyName, +} from "./capacity/index.js"; +import { DEVICE_CLASSES, type LeaseIdentity, type Platform } from "./domain.js"; +import { alertDestinationsValidator } from "./config-alerts.js"; +import { + ADMISSION_ARGUMENT_MAX_LENGTH, + ADMISSION_COMMAND_MAX_ENTRIES, + ADMISSION_TIMEOUT_MAX_MS, +} from "./config-defaults.js"; +import { + ROUTING_POLICIES, + type DownloadPolicy, + type DriverConfigEntry, + type ModuleConfigEntry, +} from "./config-types.js"; +import { + booleanValue, + integerInRange, + invalidValue, + nonNegativeNumber, + numberAtLeast, + numberInRange, + objectValidator, + positiveInteger, + positiveNumber, + requireObject, + stringUnion, + stringValue, + type Validator, + type Warn, +} from "./validation.js"; +import { warmPoolConfigValidator } from "./warm-pool/index.js"; + +const LOG_LEVELS: readonly LogLevel[] = ["debug", "info", "warn", "error"]; +const DOWNLOAD_POLICIES: readonly DownloadPolicy[] = ["never", "on-request", "always"]; +const LEASE_IDENTITIES: readonly LeaseIdentity[] = ["reusable", "fresh"]; + +const HEADER_NAME_PATTERN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]{1,100}$/; +const HEADER_VALUE_PATTERN = /^[\t -~]*$/; +const MAX_HEADERS = 16; +const MAX_HEADER_VALUE_LENGTH = 4096; +const MAX_HEADER_CHARACTERS = 8192; +const SIMLOCK_HEADER_NAMES: ReadonlySet = new Set([ + "authorization", + "x-simlock-worker-id", + "x-simlock-worker-label", + "host", + "connection", + "upgrade", + "origin", +]); + +type HeaderEntry = [name: string, text: string]; + +/** + * why: Node throws on a header name or value it cannot send, so every such entry is refused + * here at load, where the operator sees the key, rather than on each dial. + */ +function headersValidator(value: unknown, path: string): Readonly> { + const shape = "an object of header names to text values"; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw invalidValue(path, shape); + } + const entries = Object.entries(value); + if (!entries.every((entry): entry is HeaderEntry => typeof entry[1] === "string")) { + throw invalidValue(path, shape); + } + if (entries.length > MAX_HEADERS) throw invalidValue(path, "at most 16 headers"); + checkHeaderNames(entries, path); + checkHeaderValues(entries, path); + for (const [name] of entries) { + const lower = name.toLowerCase(); + if (SIMLOCK_HEADER_NAMES.has(lower) || lower.startsWith("sec-websocket-")) { + throw invalidValue(path, `no header Simlock sets itself (${name})`); + } + } + return Object.fromEntries(entries); +} + +function admissionCommandValidator(value: unknown, path: string): readonly string[] { + const shape = `a list of 1 to ${String(ADMISSION_COMMAND_MAX_ENTRIES)} strings of 1 to ${String(ADMISSION_ARGUMENT_MAX_LENGTH)} characters`; + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > ADMISSION_COMMAND_MAX_ENTRIES || + !value.every( + (entry): entry is string => + typeof entry === "string" && + entry.length >= 1 && + entry.length <= ADMISSION_ARGUMENT_MAX_LENGTH, + ) + ) { + throw invalidValue(path, shape); + } + return value; +} + +function admissionTimeoutValidator(value: unknown, path: string): number { + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value < 1 || + value > ADMISSION_TIMEOUT_MAX_MS + ) { + throw invalidValue(path, `an integer from 1 to ${String(ADMISSION_TIMEOUT_MAX_MS)}`); + } + return value; +} + +function checkHeaderNames(entries: readonly HeaderEntry[], path: string): void { + const seen = new Set(); + for (const [name] of entries) { + if (!HEADER_NAME_PATTERN.test(name)) { + throw invalidValue( + path, + `a header name of 1 to 100 letters, digits and HTTP name symbols (got ${name})`, + ); + } + } + for (const [name] of entries) { + const lower = name.toLowerCase(); + if (seen.has(lower)) throw invalidValue(path, `each header name once (got ${name} twice)`); + seen.add(lower); + } +} + +function checkHeaderValues(entries: readonly HeaderEntry[], path: string): void { + let total = 0; + for (const [name, text] of entries) { + if (text.length > MAX_HEADER_VALUE_LENGTH || !HEADER_VALUE_PATTERN.test(text)) { + throw invalidValue( + path, + `a header value of at most 4096 plain ASCII characters (in ${name})`, + ); + } + total += name.length + text.length; + } + if (total > MAX_HEADER_CHARACTERS) { + throw invalidValue(path, "at most 8192 characters of header names and values in all"); + } +} + +/** + * why: The `capacity.config` validator is the selected strategy's own, so a strategy + * owns its options end to end and nothing here has to know their shape. + */ +export function configValidators( + strategy: CapacityStrategyName, + modules: readonly ModuleConfigEntry[], + authStrategies: readonly AuthStrategyDefinition[], + driverSections: readonly DriverConfigEntry[] = [], +): Record { + return { + capabilities: capabilitiesValidator(modules), + gateway: objectValidator({ + url: stringValue, + token: stringValue, + label: stringValue, + headers: headersValidator, + disconnectedRetentionMs: positiveNumber, + execTimeoutMs: positiveNumber, + leaseRequestTimeoutMs: positiveNumber, + routing: stringUnion(ROUTING_POLICIES), + alerts: objectValidator({ destinations: alertDestinationsValidator }), + admission: objectValidator({ + command: admissionCommandValidator, + timeoutMs: admissionTimeoutValidator, + }), + }), + capacity: objectValidator({ + strategy: stringUnion(capacityStrategyNames), + config: capacityStrategyValidator(strategy), + }), + // why: Legacy spelling of the resource options; still type-checked here so a typo + // inside them is still reported rather than silently dropped. + ...resourceOptionValidators, + drivers: driversValidator, + downloads: objectValidator({ + policy: stringUnion(DOWNLOAD_POLICIES), + acceptAndroidLicenses: booleanValue, + timeoutMs: positiveNumber, + }), + idle: objectValidator({ shutdownAfterMs: nonNegativeNumber, deleteAfterMs: nonNegativeNumber }), + update: objectValidator({ check: booleanValue }), + warmPool: objectValidator({ + ...warmPoolConfigValidator, + quarantine: objectValidator({ + maxRetries: positiveInteger, + retryBackoffMs: nonNegativeNumber, + retryBackoffMultiplier: numberAtLeast(1), + maxRetryBackoffMs: nonNegativeNumber, + }), + }), + // why: The three pre-ADR-0004 keys (`lease.detachedTtlMs`, `lease.heldTtlBackstopMs`, + // `lease.heartbeatIntervalMs`) are deliberately absent rather than aliased: `objectValidator` + // warns about each as an unrecognized key and ignores it, so an old config still boots and + // gets the new key's default rather than the value it wrote. + devices: objectValidator({ autoEnroll: booleanValue, deletedRetentionMs: positiveNumber }), + lease: objectValidator({ + defaultTtlMs: positiveNumber, + maxTtlMs: positiveNumber, + identity: objectValidator({ + ios: stringUnion(LEASE_IDENTITIES), + android: stringUnion(LEASE_IDENTITIES), + }), + requestRetentionMs: positiveNumber, + endedIdRetentionMs: positiveNumber, + maxRequestRecords: positiveInteger, + maxPerRequester: maxPerRequesterValidator, + endWaitMs: endWaitValidator, + }), + exec: objectValidator({ timeoutMs: positiveNumber }), + diskPressure: objectValidator({ freeBytesThreshold: nonNegativeNumber }), + memoryPressure: objectValidator({ + enabled: booleanValue, + pauseWarming: memoryPressureRuleValidator, + pauseNewDevices: memoryPressureRuleValidator, + clearAfterMs: nonNegativeNumber, + }), + eventBuffer: objectValidator({ capacity: positiveInteger }), + log: objectValidator({ level: stringUnion(LOG_LEVELS), rotateBytes: positiveInteger }), + eventLog: objectValidator({ + retention: positiveInteger, + maxBytes: positiveInteger, + }), + metrics: objectValidator({ retention: positiveInteger }), + http: objectValidator({ + enabled: booleanValue, + host: stringValue, + port: integerInRange(1, 65535), + ...Object.fromEntries( + authStrategies.flatMap((definition) => + definition.config === undefined + ? [] + : [[definition.config.key, schemaValidator(definition.config.schema)]], + ), + ), + }), + health: objectValidator({ + enabled: booleanValue, + probeIntervalMs: positiveNumber, + stableObservations: positiveInteger, + maxRecoveryAttempts: positiveInteger, + recoveryBackoffMs: positiveNumber, + maxConcurrentRecoveries: positiveInteger, + }), + ios: objectValidator({ + ...driverValidators(driverSections, "ios"), + defaultMode: stringUnion(["slim", "full"]), + defaultModels: defaultModelsValidator, + }), + android: objectValidator({ + ...driverValidators(driverSections, "android"), + defaultModels: defaultModelsValidator, + }), + stalledTransition: objectValidator({ + thresholdMultiplier: numberAtLeast(1), + minimumThresholdMs: nonNegativeNumber, + }), + }; +} + +const memoryPressureRuleValidator = objectValidator({ + level: stringUnion(["normal", "warn", "critical", "off"]), + minAvailablePercent: numberInRange(0, 100), +}); + +function driverValidators( + driverSections: readonly DriverConfigEntry[], + platform: Platform, +): Record { + return Object.assign( + {}, + ...driverSections + .filter((entry) => entry.platform === platform) + .map((entry) => entry.validators), + ) as Record; +} + +function schemaValidator(schema: ZodType): Validator { + return (value, path) => { + const result = schema.safeParse(value); + if (result.success) return result.data; + const issue = result.error.issues[0]; + if (issue === undefined) throw invalidValue(path, "a valid value"); + const issuePath = [path, ...issue.path.map(String)].join("."); + // why: a strategy's own messages are lower-case phrases that finish "expected ..."; zod's built-in ones start with a capital. + const expected = /^[A-Z]/.test(issue.message) + ? `a valid value (${issue.message})` + : issue.message; + throw invalidValue(issuePath, expected); + }; +} + +function capabilitiesValidator(modules: readonly ModuleConfigEntry[]): Validator { + return (value, path, warn) => { + const result: Record = {}; + for (const [name, section] of Object.entries(requireObject(value, path))) { + const sectionPath = `${path}.${name}`; + const entry = modules.find((candidate) => candidate.name === name); + if (entry === undefined) { + warn(`Unknown config key: "${sectionPath}"`); + continue; + } + const { enabled, ...rest } = requireObject(section, sectionPath); + result[name] = { + ...(entry.validator(rest, sectionPath, warn) as Record), + ...(enabled === undefined + ? {} + : { enabled: booleanValue(enabled, `${sectionPath}.enabled`) }), + }; + } + return result; + }; +} + +/** why: `setTimeout` fires after about 1 ms above this delay, which would end the wait at once. */ +const MAXIMUM_TIMER_DELAY_MS = 2_147_483_647; + +const endWaitValidator: Validator = (value, path) => { + const wait = positiveNumber(value, path); + if (wait > MAXIMUM_TIMER_DELAY_MS) { + throw invalidValue(path, `a positive number no larger than ${MAXIMUM_TIMER_DELAY_MS}`); + } + return wait; +}; + +/** + * why: Each key is a device class and each value one non-empty model name or a non-empty list of + * them, stored as a list. Any other key is refused rather than warned about: a misspelled class + * would otherwise leave the operator's preference silently unused. + */ +const defaultModelsValidator: Validator = (value, path) => { + const object = requireObject(value, path); + const result: Record = {}; + + for (const [key, names] of Object.entries(object)) { + const keyPath = `${path}.${key}`; + if (!(DEVICE_CLASSES as readonly string[]).includes(key)) { + throw invalidValue(keyPath, `a device class (${DEVICE_CLASSES.join(", ")})`); + } + const list = typeof names === "string" ? [names] : names; + if (!Array.isArray(list) || list.length === 0 || !list.every(isNonEmptyString)) { + throw invalidValue(keyPath, "a model name or a non-empty list of model names"); + } + result[key] = list; + } + + return result; +}; + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +/** + * why: Checks the shape of `drivers` and nothing else. Every leaf is type-checked, because a + * config file is hand-edited and a nested object or array there is a mistake worth + * naming -- but an unrecognised key inside a driver block is deliberately *not* a + * warning. The core does not know which keys a driver has, and warning about the ones it + * has not heard of would be the same leak as interpreting them. + */ +const driversValidator: Validator = (value, path) => { + const drivers = requireObject(value, path); + + return Object.fromEntries( + Object.entries(drivers).map(([name, block]) => [ + name, + driverSettings(block, `${path}.${name}`), + ]), + ); +}; + +function driverSettings(value: unknown, path: string): Record { + const settings = requireObject(value, path); + const result: Record = {}; + + for (const [key, leaf] of Object.entries(settings)) { + if (typeof leaf !== "string" && typeof leaf !== "number" && typeof leaf !== "boolean") { + throw invalidValue(`${path}.${key}`, "a string, number, or boolean"); + } + + result[key] = leaf; + } + + return result; +} + +function maxPerRequesterValidator(value: unknown, path: string, warn: Warn): unknown { + return typeof value === "object" && value !== null + ? objectValidator({ ios: positiveInteger, android: positiveInteger })(value, path, warn) + : positiveInteger(value, path); +} diff --git a/src/core/config.ts b/src/core/config.ts index bc644b7c7..4757e6383 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -1,108 +1,55 @@ -import type { ZodType } from "zod"; - -import type { Filesystem, LogLevel, SystemStats } from "../ports/index.js"; -import { - authStrategies as registeredAuthStrategies, - type AuthStrategyDefinition, -} from "../auth/index.js"; -import { - capacityStrategyNames, - capacityStrategyValidator, - DEFAULT_CAPACITY_STRATEGY, - defaultCapacityOptions, - isCapacityStrategyName, - resourceOptionValidators, - type CapacityConfig, - type CapacityLimits, - type CapacityStrategyName, - type ResourceStrategyOptions, -} from "./capacity/index.js"; -import { - DEVICE_CLASSES, - type DeviceClass, - type DeviceMode, - type LeaseIdentity, - type Platform, -} from "./domain.js"; -import { alertDestinationsValidator, type AlertDestination } from "./config-alerts.js"; +import type { Filesystem } from "../ports/index.js"; +import { authStrategies as registeredAuthStrategies } from "../auth/index.js"; +import type { Platform } from "./domain.js"; +import { ConfigError, requireObject, type Validator, type Warn } from "./validation.js"; import { - booleanValue, - ConfigError, - integerInRange, - numberInRange, - invalidValue, - nonNegativeNumber, - numberAtLeast, - objectValidator, - positiveInteger, - positiveNumber, - requireObject, - stringUnion, - stringValue, - type Validator, - type Warn, -} from "./validation.js"; + DEFAULT_CONFIG_PATH, + DEFAULT_METRICS_RETENTION_MS, + defaultConfig, +} from "./config-defaults.js"; +import { asObject, mergeConfig } from "./config-helpers.js"; import { - defaultWarmPoolConfig, - warmPoolConfigValidator, - type WarmPoolConfig, -} from "./warm-pool/index.js"; - -const DEFAULT_CONFIG_PATH = "~/.simlock/config.json"; - -const DEFAULT_LEASE_TTL_MS = 15 * 60_000; - -const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; - -/** why: `devices.deletedRetentionMs`'s default: how long a `deleted` device record is kept, counted from its deletion. */ -export const DEFAULT_DEVICES_DELETED_RETENTION_MS = 60 * 60_000; - -export const DEFAULT_LEASE_REQUEST_RETENTION_MS = 10 * 60_000; - -/** why: `lease.endedIdRetentionMs`'s default: how long a chosen lease id is refused after its lease ended (ADR 0074 §4). */ -export const DEFAULT_LEASE_ENDED_ID_RETENTION_MS = 24 * 60 * 60_000; - -/** why: `gateway.admission.timeoutMs`'s default (ADR 0074 §1): a policy script answers in well under this, and a stuck one refuses requests for no longer. */ -const DEFAULT_ADMISSION_TIMEOUT_MS = 5_000; - -/** why: the bounds on `gateway.admission`, checked when the config is read. */ -const ADMISSION_TIMEOUT_MAX_MS = 60_000; -const ADMISSION_COMMAND_MAX_ENTRIES = 64; -const ADMISSION_ARGUMENT_MAX_LENGTH = 4096; - -const DEFAULT_LEASE_MAX_PER_REQUESTER = 1; - -export const DEFAULT_LEASE_MAX_REQUEST_RECORDS = 10_000; - -const DEFAULT_DISCONNECTED_RETENTION_MS = 24 * 60 * 60_000; + normalizeLayer, + requireGatewayHttp, + resolveStrategyName, + runModuleChecks, + validateDeletedRetention, + validateEventLogBounds, + validateGatewayMembership, + validateLeaseTtls, + validateMetricsRetention, + warnWorkerOnlyKeys, +} from "./config-role-checks.js"; +import { configValidators } from "./config-validators.js"; +import type { + Config, + ConfigLayerOptions, + DriverConfigEntry, + Layer, + LoadConfigOptions, + LoadedConfig, +} from "./config-types.js"; -/** why: `gateway.execTimeoutMs`'s default (ADR 0005 §19e): eleven minutes, one more than the - * worker's authoritative ten, so the worker's timeout is the one that fires. */ -const DEFAULT_GATEWAY_EXEC_TIMEOUT_MS = 11 * 60_000; - -/** why: `gateway.leaseRequestTimeoutMs`'s default (P2, round 2 review): five minutes -- generous - * against a cold device provision-plus-boot, and well short of `execTimeoutMs`, since granting a - * lease should never take as long as a command a caller runs against the device afterward. */ -const DEFAULT_LEASE_REQUEST_TIMEOUT_MS = 5 * 60_000; - -/** - * why: `gateway.routing`'s valid names (ADR 0005 §13/Decision 7) and its default. Declared here - * rather than imported from `src/gateway/routing.ts`'s own registry: `core` stays - * platform-agnostic and gateway-unaware (`AGENTS.md`/`architecture.md`), so this list is - * re-declared the same way `src/contract/schemas.ts` re-declares it independently rather than - * importing across its own boundary -- three places name `"warm-then-free"`, the same - * redundancy `capacity.strategy`'s "fixed"/"resource" already carries between `core` and the - * contract. Adding a second policy means adding it here, in the gateway's own registry, and in - * the contract schema -- one line each, never a shared import. - */ -const ROUTING_POLICIES = ["warm-then-free"] as const; -const DEFAULT_ROUTING_POLICY = "warm-then-free"; - -/** why: The literal union `ROUTING_POLICIES` names -- `Config["gateway"]["routing"]`'s type, so it - * agrees with the contract's own `z.enum(["warm-then-free"])` (also re-declared, never - * imported) instead of widening to a bare `string` the boundary between the two would otherwise - * force. */ -type RoutingPolicyName = (typeof ROUTING_POLICIES)[number]; +export { + DEFAULT_DEVICES_DELETED_RETENTION_MS, + DEFAULT_LEASE_ENDED_ID_RETENTION_MS, + DEFAULT_LEASE_MAX_REQUEST_RECORDS, + DEFAULT_LEASE_REQUEST_RETENTION_MS, +} from "./config-defaults.js"; +export { configValidators } from "./config-validators.js"; +export { effectiveAllowDownload } from "./config-role-checks.js"; +export type { + Config, + ConfigOverrides, + DaemonMode, + DefaultModels, + DownloadPolicy, + DriverConfigEntry, + LegacyCapacityOverrides, + LoadConfigOptions, + MaxPerRequester, + ModuleConfigEntry, +} from "./config-types.js"; export { ALERT_EVENTS, @@ -112,236 +59,6 @@ export { type AlertFormatName, } from "./config-alerts.js"; -/** - * why: `exec.timeoutMs`'s default (ADR 0005 §19e): how long a single `device.exec` command may run - * on this worker before it is killed and the operation fails with `EXEC_TIMEOUT`. Ten minutes, - * because the commands this wraps are installs and boots-worth of `adb wait-for-device`, not - * sub-second reads -- and because a stuck command holds no lease of its own, so the cost of a - * generous bound is one process, not a device. - */ -const DEFAULT_EXEC_TIMEOUT_MS = 10 * 60_000; - -export type DownloadPolicy = "never" | "on-request" | "always"; - -/** - * why: one number counts a requester's leases across all platforms; the object counts per - * platform, and a platform it leaves out is 1. - */ -export type MaxPerRequester = number | { readonly ios?: number; readonly android?: number }; - -/** - * why: which shape a daemon runs as (ADR 0005 §1): a `worker` owns the devices on its machine and a - * `gateway` owns none and fronts the workers that joined it. One daemon runs exactly one mode, - * and the folder it lives in decides it (ADR 0034): the daemon is told its role at start, and - * `loadConfig` takes it as an argument -- no config key names it. - */ -export type DaemonMode = "worker" | "gateway"; - -interface MemoryPressureRule { - readonly level: "normal" | "warn" | "critical" | "off"; - readonly minAvailablePercent: number; -} - -export interface Config { - readonly capabilities: Readonly< - Record>> - >; - readonly mode: DaemonMode; - readonly capacity: CapacityConfig; - /** - * why: Per-driver settings, opaque to the core: stored, merged, and handed to the driver - * that owns the key without a single key being interpreted here. What - * `drivers.ios.deviceRoot` means is the iOS driver's business, and knowing it here - * would be exactly the leak architecture rule 2 forbids. - */ - readonly drivers: Readonly>>>; - readonly downloads: { - readonly policy: DownloadPolicy; - /** why: Explicit legal consent for Android SDK licenses, independent of `policy`. */ - readonly acceptAndroidLicenses: boolean; - /** - * why: One budget per install request, from its arrival, covering the wait behind another - * download and the download itself (ADR 0010 §3); downloads run minutes, not seconds. - */ - readonly timeoutMs: number; - }; - readonly idle: { - readonly shutdownAfterMs: number; - readonly deleteAfterMs: number; - }; - readonly update: { - readonly check: boolean; - }; - readonly warmPool: WarmPoolConfig & { - readonly quarantine: { - readonly maxRetries: number; - readonly retryBackoffMs: number; - readonly retryBackoffMultiplier: number; - readonly maxRetryBackoffMs: number; - }; - }; - readonly devices: { - /** - * why: how long a `deleted` device record stays in `state.db` and in `status.get`, counted - * from its deletion. At least `lease.requestRetentionMs`: a granted request names its device. - */ - readonly deletedRetentionMs: number; - /** - * why: a worker enrols the phones plugged into it by USB, as `device add` would; off by default - * because a personal phone on the worker would be wiped of apps at every lease's end. - */ - readonly autoEnroll: boolean; - }; - readonly lease: { - /** - * why: ADR 0004 §4: the TTL applied to a lease whose `lease.request` carried no `ttlMs` -- that - * request only. It is deliberately *not* the renew fallback: a `lease.renew` given no - * explicit TTL re-applies the lease's own stored width, so a lease granted for longer - * keeps it. - */ - readonly defaultTtlMs: number; - /** why: The largest TTL a request or a renew may ask for; more is `BAD_REQUEST`, never a - * silent clamp, so a caller is never left believing it has more time than it does. */ - readonly maxTtlMs: number; - /** - * why: The lease-identity policy per platform (see `LeaseIdentity`) no longer changes anything: - * every virtual device serves one lease and is deleted when it ends (ADR 0061 §2). It stays - * accepted and stored on each device record. - */ - readonly identity: { - readonly ios: LeaseIdentity; - readonly android: LeaseIdentity; - }; - readonly requestRetentionMs: number; - /** why: ADR 0074 §4: how long a chosen lease id is refused after its lease ended, counted from the end; read by a worker and a gateway. */ - readonly endedIdRetentionMs: number; - readonly maxRequestRecords: number; - readonly maxPerRequester: MaxPerRequester; - readonly endWaitMs: number; - }; - /** - * why: ADR 0005 §19e. Platform-agnostic on purpose: it bounds the *daemon's* willingness to wait - * on a child, not anything either driver knows about, so it sits at the top level rather - * than under `drivers.*`. - */ - readonly exec: { readonly timeoutMs: number }; - readonly diskPressure: { readonly freeBytesThreshold: number }; - /** why: ADR 0068 §1: when host memory pauses a worker's warming and its new devices; worker only. */ - readonly memoryPressure: { - readonly enabled: boolean; - readonly pauseWarming: MemoryPressureRule; - readonly pauseNewDevices: MemoryPressureRule; - readonly clearAfterMs: number; - }; - readonly eventBuffer: { readonly capacity: number }; - readonly log: { readonly level: LogLevel; readonly rotateBytes: number }; - readonly eventLog: { - readonly retention: number; - readonly maxBytes: number; - }; - readonly metrics: { readonly retention: number }; - readonly http: { - readonly enabled: boolean; - readonly host: string; - readonly port: number; - readonly proxyAuth?: { - readonly header: string; - readonly jwksUrl: string; - readonly issuer: string; - readonly audience: string; - readonly signOutUrl?: string; - }; - }; - readonly health: { - readonly enabled: boolean; - readonly probeIntervalMs: number; - readonly stableObservations: number; - readonly maxRecoveryAttempts: number; - readonly recoveryBackoffMs: number; - readonly maxConcurrentRecoveries: number; - }; - readonly ios: { - readonly defaultMode: DeviceMode; - readonly defaultModels: DefaultModels; - } & DriverConfigSection; - readonly android: { - readonly defaultModels: DefaultModels; - } & DriverConfigSection; - /** - * why: ADR 0005 §3/§6. Both sides of the fleet live under one key, and which half is read depends - * on `mode`: - * - * - a **worker** reads `url`, `token`, and `label` -- the two keys that join a fleet, plus a - * display name. Absent `url`/`token` simply means "this worker has no gateway", which is - * every worker today. - * - a **gateway** reads `disconnectedRetentionMs` and ignores the other three with a - * warning (`loadConfig` reports every worker-only key it finds in a gateway's config). - * - * Kept as one block rather than `gateway.*` / `worker.*` blocks because both halves are - * facts about the same relationship, and a `worker.gateway.url` spelling reads worse than - * the thing it configures. - */ - readonly gateway: { - readonly url?: string; - /** why: A join token minted on the gateway with `simlock token create --role worker`. - * Worker-side. It is a secret, and it sits in the daemon's config file, which `simlock - * config set` writes owner-only (0600) every time; a file written by hand, or before - * that, keeps whatever mode it has until the next `config set`. Everywhere else the - * config is written out -- the startup log line, `daemon.started`, and `config.get` - * unless it is asked with `showSecrets: true` -- it goes through `redactConfig`. */ - readonly token?: string; - readonly label?: string; - readonly headers?: Readonly>; - /** - * why: How long a gateway keeps a disconnected worker's view before forgetting it (§6). - * Gateway-side. Default 24 hours: long enough that a machine rebooting overnight is still - * the same machine in the morning, short enough that a decommissioned one does not haunt - * `simlock status` forever. A view whose leases have not all expired is never dropped on - * this timer, however long it has been gone. - */ - readonly disconnectedRetentionMs: number; - /** - * why: Gateway-side backstop on a proxied `device.exec` (ADR 0005 §19e). Deliberately longer - * than the worker's own `exec.timeoutMs` (ten minutes): the worker's is authoritative and - * should be what kills a runaway command, so a gateway that timed out first would report - * a failure for a command still running on the machine that owns the device. Read by #118, - * which is what proxies `device.exec`; declared now so the key does not appear mid-series. - */ - readonly execTimeoutMs: number; - /** - * why: P2 (round 2 review): bounds a forwarded `lease.request` -- the one uplink call that used - * to have no timeout of its own, parking a waiter `processing` where neither `timeoutMs`'s - * deadline check nor `lease.cancel` could reach it (`WaitQueue#armTimeout` declines to - * reject a `processing` waiter, and `cancelPending` answers `not-cancellable`) if the - * worker's own `lease.request` handler ever wedged. Generous relative to how long granting a - * lease can plausibly take -- provisioning plus a cold device boot -- and bounded well below - * `execTimeoutMs`, since granting a lease should never take as long as an arbitrary command - * a caller runs afterward might. Expiry maps to `WORKER_UNREACHABLE`, returning the waiter - * to a state `enqueue`'s deadline check and `cancelPending` can both act on again. - */ - readonly leaseRequestTimeoutMs: number; - readonly routing: RoutingPolicyName; - readonly alerts: { readonly destinations: readonly AlertDestination[] }; - /** - * why: ADR 0074 §1: the program a gateway runs for each new lease request, and how long it may - * take; a worker ignores both, as it does `execTimeoutMs`. - */ - readonly admission: { readonly command?: readonly string[]; readonly timeoutMs: number }; - }; - readonly stalledTransition: { - /** - * why: Applied to the driver's own estimate (`provision + boot` for `provisioning`, - * `reclaim` for `reclaiming`) to get the stall threshold. Deliberately generous: - * the estimate itself is already tuned for a routine run, so the threshold has to - * clear real-world variance on top of it, not just match it -- a cold Android - * provision-plus-boot can legitimately run well past the raw estimate. - */ - readonly thresholdMultiplier: number; - /** why: Floor under the multiplied estimate, for a driver whose estimate is near zero. */ - readonly minimumThresholdMs: number; - }; -} - export const REDACTED_VALUE = "[redacted]"; // why: masked rather than deleted, so the copy keeps `Config`'s keys and a reader can tell "set" from "unset"; a copy, because the live config is what `config.get` answers and the uplink presents. @@ -385,33 +102,6 @@ export function redactConfig< } as Shape; } -/** - * why: The pre-`capacity` spelling of the resource strategy's options. Still accepted - * everywhere the new shape is, and folded into `capacity.config` before anything - * downstream sees it, so existing config files keep working untouched. - */ -export interface LegacyCapacityOverrides { - readonly limits?: DeepPartial; - readonly ramBudget?: DeepPartial; -} - -export type ConfigOverrides = DeepPartial & LegacyCapacityOverrides; - -// why: a platform's section also holds its driver's own keys, which the loader carries without reading one. -type DriverConfigSection = Readonly>; - -// why: a driver declares the keys it reads under its platform's section, so a new driver setting changes no file of the core; `defaults` holds every key that has one, typed as the driver reads the section. -export interface DriverConfigEntry
{ - readonly platform: Platform; - readonly validators: Readonly>; - readonly defaults: Section; - // why: the values the driver uses for the `drivers..*` keys config.json leaves unset, which depend on the data directory and on the keys that are set; `config.get` shows them, so `config get` reads a key at its default. - readonly settingDefaults?: ( - simlockHome: string, - settings: Readonly>, - ) => Readonly>; -} - // why: typed by the entry the loader validated the section with; a config loaded without that entry holds none of its keys. export function driverConfig
( config: Pick, @@ -420,44 +110,13 @@ export function driverConfig
( return config[entry.platform] as Section; } -export interface ModuleConfigEntry { - readonly name: string; - readonly validator: Validator; - readonly defaults: unknown; - readonly gatewayKeys: readonly string[]; - readonly enabledByDefault?: "standalone-worker"; - readonly check?: (section: never, config: Config) => void; -} - const MODE_KEY_REMOVED = '"mode" is no longer a config key. A daemon started with --gateway is a gateway; any other is a worker. Remove "mode" from config.json.'; -interface ConfigLayerOptions { - readonly authStrategies?: readonly AuthStrategyDefinition[]; - readonly driverSections?: readonly DriverConfigEntry[]; - readonly modules?: readonly ModuleConfigEntry[]; - readonly systemStats: SystemStats; - readonly overrides?: ConfigOverrides; - readonly role?: DaemonMode; - readonly warn?: Warn; -} - -export interface LoadConfigOptions extends ConfigLayerOptions { - readonly filesystem: Filesystem; - readonly configPath?: string; -} - -type Layer = Record; - export async function loadConfig(options: LoadConfigOptions): Promise { return (await loadConfigWithDefaults(options)).config; } -interface LoadedConfig { - readonly config: Config; - readonly defaulted: ReadonlySet; -} - export async function loadConfigWithDefaults({ configPath = DEFAULT_CONFIG_PATH, filesystem, @@ -561,399 +220,6 @@ export function configFromLayers( return { config: deepFreeze(merged), defaulted }; } -/** - * why: The one consent function (ADR 0010 §4): reduces `downloads.policy` and what the caller asked - * for to whether a download may start. A lease request passes its own `--allow-download` / - * `allowDownload` flag; `component.install` passes `true`, because the command itself is the - * consent. `never` overrides an explicit `true` -- the whole point of the policy is that no - * request and no operation can opt back into it -- and `always` grants permission the request - * never had to ask for. Only `on-request` defers to what the caller asked for. - */ -export function effectiveAllowDownload(policy: DownloadPolicy, requested: boolean): boolean { - if (policy === "always") return true; - if (policy === "never") return false; - return requested; -} - -/** - * why: Cross-field check (ADR 0004's Consequences): the default TTL a request falls back to cannot - * exceed the cap every request is measured against, or every defaulted request would be born - * asking for more than the daemon will grant. A violating config fails the daemon start naming - * the offending key rather than being clamped to a value the operator never wrote -- the - * opposite treatment from a retired key, which is a leftover with a safe reading ("ignore it") - * where a self-contradicting TTL pair has none. - */ -function validateLeaseTtls(config: Config): void { - if (config.lease.defaultTtlMs > config.lease.maxTtlMs) { - throw invalidValue("lease.defaultTtlMs", "at most lease.maxTtlMs"); - } -} - -function validateDeletedRetention(config: Config): void { - if (config.devices.deletedRetentionMs < config.lease.requestRetentionMs) { - throw invalidValue("devices.deletedRetentionMs", "at least lease.requestRetentionMs"); - } -} - -const DEFAULT_METRICS_RETENTION_MS = 90 * 24 * 60 * 60 * 1000; - -const MIN_EVENT_LOG_MAX_BYTES = 16 * 1024 * 1024; - -function validateEventLogBounds(config: Config): void { - if (config.eventLog.maxBytes < MIN_EVENT_LOG_MAX_BYTES) { - throw invalidValue("eventLog.maxBytes", "at least 16 MiB"); - } -} - -function validateMetricsRetention(config: Config): void { - if (config.metrics.retention < config.eventLog.retention) { - throw invalidValue("metrics.retention", "at least eventLog.retention"); - } -} - -/** - * why: The config keys a gateway reads (ADR 0005 §2). Everything else in a gateway's config is a - * worker-only key: it configures drivers, devices, capacity, or cleanup, none of which a - * gateway has. Listed here rather than derived from `Config`'s shape because the *set* is the - * decision -- adding a key to `Config` should not silently make it gateway-relevant. - */ -const GATEWAY_CONFIG_KEYS: readonly string[] = [ - "http", - "log", - "lease", - "eventBuffer", - "eventLog", - "metrics", - "gateway", - "capabilities", - "update", -]; - -const WORKER_ONLY_GATEWAY_KEYS: readonly string[] = ["url", "token", "label", "headers"]; - -/** - * why: ADR 0005 §2: "worker-only keys in a gateway's config are ignored with a warning, as unknown - * keys are today". Warned per key the operator actually wrote (the file and override layers), - * never per defaulted key -- a gateway's own defaults contain the whole worker-side `Config` - * shape, and warning about those would fire on every start with nothing to fix. - * - * Ignoring rather than failing is the deliberate choice the ADR makes, and it is what lets one - * config file be copied between a worker and a gateway while only `mode` differs. - */ -function warnWorkerOnlyKeys( - layers: readonly unknown[], - warn: Warn, - modules: readonly ModuleConfigEntry[], -): void { - for (const layer of layers) { - const object = requireObject(layer, "config"); - for (const key of Object.keys(object)) { - if (!GATEWAY_CONFIG_KEYS.includes(key)) { - warn(`Ignoring "${key}": it configures a worker, and this daemon runs in gateway mode.`); - } - } - warnWorkerOnlySubKeys( - object, - "gateway", - WORKER_ONLY_GATEWAY_KEYS, - warn, - (key) => - `Ignoring "gateway.${key}": it points a worker at its gateway, ` + - `and this daemon runs in gateway mode.`, - ); - warnWorkerOnlyModuleKeys(object, modules, warn); - } -} - -function warnWorkerOnlyModuleKeys( - layer: Record, - modules: readonly ModuleConfigEntry[], - warn: Warn, -): void { - const sections = layer["capabilities"]; - if (sections === undefined) return; - for (const [name, section] of Object.entries(requireObject(sections, "capabilities"))) { - const entry = modules.find((candidate) => candidate.name === name); - if (entry === undefined) continue; - for (const key of Object.keys(requireObject(section, `capabilities.${name}`))) { - if (key !== "enabled" && !entry.gatewayKeys.includes(key)) { - warn( - `Ignoring "capabilities.${name}.${key}": it configures a worker, ` + - `and this daemon runs in gateway mode.`, - ); - } - } - } -} - -function runModuleChecks(config: Config, modules: readonly ModuleConfigEntry[]): void { - for (const entry of modules) { - entry.check?.(config.capabilities[entry.name] as never, config); - } -} - -function warnWorkerOnlySubKeys( - layer: Record, - section: string, - workerOnly: readonly string[], - warn: Warn, - message: (key: string) => string, -): void { - const value = layer[section]; - if (value === undefined) return; - for (const key of Object.keys(requireObject(value, section))) { - if (workerOnly.includes(key)) warn(message(key)); - } -} - -/** - * why: ADR 0005 §2: "a gateway always listens on HTTP (it is the fleet's contact point) and on its - * unix socket". HTTP is not optional for a gateway in a way it is for a worker: it is how - * every agent reaches the fleet *and* the transport the worker uplink upgrades from - * (`/v1/uplink`), so a gateway with `http.enabled: false` is a process nothing can reach and - * no worker can join. - * - * The default is flipped to `true` for a gateway (see `defaultConfig`), so this only ever - * fires for a config that says `false` out loud. That is refused rather than silently - * overridden, on the same reasoning as `lease.defaultTtlMs > lease.maxTtlMs`: a value the - * operator wrote and the daemon quietly inverted is worse than a start that says why. - */ -function requireGatewayHttp(config: Config): void { - if (!config.http.enabled) { - throw invalidValue("http.enabled", "true in gateway mode (a gateway is reached over HTTP)"); - } -} - -/** - * why: A worker's fleet membership, checked at load rather than at dial time so a typo fails the - * daemon start naming the key instead of surfacing as an endless reconnect loop hours later. - * - * `gateway.url` is the gateway's **base** URL (`ws://host:port`); the worker derives the - * uplink endpoint (`/v1/uplink`) from it, so nothing here inspects the path -- a URL that - * already carries one is accepted and the endpoint is resolved against it, which is what makes - * a reverse proxy on a sub-path work. - * - * The two keys are required together: a URL with no token cannot authenticate and a token with - * no URL has nowhere to go, and either alone is an operator half-finishing a join. Failing the - * start is right where ignoring would be wrong -- a worker that silently never joined would - * look identical to one whose gateway is down. - */ -function validateGatewayMembership(config: Config): void { - const { token, url } = config.gateway; - if (url !== undefined) requireWebSocketUrl(url); - if (url !== undefined && token === undefined) { - throw invalidValue("gateway.token", "a join token, since gateway.url is set"); - } - if (token !== undefined && url === undefined) { - throw invalidValue("gateway.url", "a gateway URL, since gateway.token is set"); - } - const { headers } = config.gateway; - if (url === undefined && headers !== undefined && Object.keys(headers).length > 0) { - throw invalidValue( - "gateway.headers", - "gateway.url to be set too, since gateway.headers is set", - ); - } - requireJoinedWorkerHttpOff(config); -} - -function requireWebSocketUrl(url: string): void { - let parsed: URL; - try { - parsed = new URL(url); - } catch { - throw invalidValue("gateway.url", "a ws:// or wss:// URL"); - } - if (parsed.protocol !== "ws:" && parsed.protocol !== "wss:") { - throw invalidValue("gateway.url", "a ws:// or wss:// URL"); - } -} - -// why: a joined worker is reached through the gateway; its own listener would be a second entrance. -function requireJoinedWorkerHttpOff(config: Config): void { - if (config.gateway.url !== undefined && config.http.enabled) { - throw invalidValue( - "http.enabled", - "false when gateway.url is set (joined workers are reached through the gateway)", - ); - } -} - -function resolveStrategyName(layers: readonly unknown[]): CapacityStrategyName { - let resolved: CapacityStrategyName = DEFAULT_CAPACITY_STRATEGY; - - for (const layer of layers) { - const capacity = requireObject(layer, "config")["capacity"]; - if (capacity === undefined) continue; - - const candidate = requireObject(capacity, "capacity")["strategy"]; - if (candidate === undefined) continue; - - if (!isCapacityStrategyName(candidate)) { - throw invalidValue( - "capacity.strategy", - `one of ${capacityStrategyNames.map((name) => `"${name}"`).join(", ")}`, - ); - } - - resolved = candidate; - } - - return resolved; -} - -const LEGACY_RESOURCE_KEYS = ["limits", "ramBudget"] as const; - -/** - * why: Folds the legacy top-level keys into `capacity.config`. Runs per layer, before - * merging, so precedence between layers is unaffected by which spelling each one - * happens to use. Within a layer `capacity.config` wins. - */ -function normalizeLayer(layer: Layer, strategy: CapacityStrategyName, warn: Warn): Layer { - const legacy = pick(layer, LEGACY_RESOURCE_KEYS); - const present = Object.keys(legacy); - if (present.length === 0) return layer; - - const rest = omit(layer, LEGACY_RESOURCE_KEYS); - if (strategy !== "resource") { - warn(legacyIgnoredMessage(present, strategy)); - return rest; - } - - const capacity = asObject(rest["capacity"]); - return { - ...rest, - capacity: { ...capacity, config: mergeConfig(legacy, asObject(capacity["config"])) }, - }; -} - -function legacyIgnoredMessage(keys: readonly string[], strategy: CapacityStrategyName): string { - const subject = keys.length > 1 ? "these keys configure" : "this key configures"; - return ( - `Ignoring ${keys.join(" and ")}: ${subject} the "resource" capacity strategy, ` + - `but "${strategy}" is selected. Move the settings under "capacity.config".` - ); -} - -function pick(layer: Layer, keys: readonly string[]): Layer { - return Object.fromEntries( - keys.filter((key) => layer[key] !== undefined).map((key) => [key, layer[key]]), - ); -} - -function omit(layer: Layer, keys: readonly string[]): Layer { - return Object.fromEntries(Object.entries(layer).filter(([key]) => !keys.includes(key))); -} - -function asObject(value: unknown): Layer { - return isObject(value) ? value : {}; -} - -function defaultConfig( - systemStats: SystemStats, - strategy: CapacityStrategyName, - role: DaemonMode, - modules: readonly ModuleConfigEntry[], - authStrategies: readonly AuthStrategyDefinition[], - standalone: boolean, - defaulted: ReadonlySet, - driverSections: readonly DriverConfigEntry[], -): Omit { - return { - mode: role, - capabilities: Object.fromEntries( - modules.map((entry) => [ - entry.name, - { ...(entry.defaults as Record), enabled: defaulted.has(entry.name) }, - ]), - ) as Config["capabilities"], - capacity: { - strategy, - config: defaultCapacityOptions(strategy, systemStats), - } as CapacityConfig, - drivers: {}, - downloads: { - policy: "on-request", - acceptAndroidLicenses: false, - timeoutMs: 1_200_000, - }, - idle: { - shutdownAfterMs: 10 * 60_000, - deleteAfterMs: 60 * 60_000, - }, - update: { check: true }, - warmPool: { - ...defaultWarmPoolConfig(systemStats), - quarantine: { - maxRetries: 3, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - maxRetryBackoffMs: 5 * 60_000, - }, - }, - devices: { autoEnroll: false, deletedRetentionMs: DEFAULT_DEVICES_DELETED_RETENTION_MS }, - lease: { - defaultTtlMs: DEFAULT_LEASE_TTL_MS, - maxTtlMs: DEFAULT_LEASE_MAX_TTL_MS, - identity: { ios: "reusable", android: "reusable" }, - requestRetentionMs: DEFAULT_LEASE_REQUEST_RETENTION_MS, - endedIdRetentionMs: DEFAULT_LEASE_ENDED_ID_RETENTION_MS, - maxRequestRecords: DEFAULT_LEASE_MAX_REQUEST_RECORDS, - maxPerRequester: DEFAULT_LEASE_MAX_PER_REQUESTER, - endWaitMs: DEFAULT_EXEC_TIMEOUT_MS, - }, - exec: { timeoutMs: DEFAULT_EXEC_TIMEOUT_MS }, - diskPressure: { freeBytesThreshold: 10 * 1024 ** 3 }, - memoryPressure: { - enabled: true, - pauseWarming: { level: "warn", minAvailablePercent: 20 }, - pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, - clearAfterMs: 60_000, - }, - eventBuffer: { capacity: 1_000 }, - log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, - eventLog: { - retention: 7 * 24 * 60 * 60 * 1000, - maxBytes: 256 * 1024 * 1024, - }, - // why: ADR 0005 §2 and ADR 0046 §1: a gateway and a standalone worker listen on HTTP by default; a joined worker is reached through its gateway, so its default is off. - http: { - enabled: role === "gateway" || standalone, - host: "127.0.0.1", - port: role === "gateway" ? 4700 : 4701, - ...Object.fromEntries( - authStrategies.flatMap((definition) => - definition.config === undefined - ? [] - : [[definition.config.key, definition.config.defaults]], - ), - ), - }, - health: { - enabled: true, - probeIntervalMs: 30_000, - stableObservations: 2, - maxRecoveryAttempts: 3, - recoveryBackoffMs: 5_000, - maxConcurrentRecoveries: 1, - }, - ios: { ...driverDefaults(driverSections, "ios"), defaultMode: "full", defaultModels: {} }, - android: { ...driverDefaults(driverSections, "android"), defaultModels: {} }, - gateway: { - disconnectedRetentionMs: DEFAULT_DISCONNECTED_RETENTION_MS, - execTimeoutMs: DEFAULT_GATEWAY_EXEC_TIMEOUT_MS, - leaseRequestTimeoutMs: DEFAULT_LEASE_REQUEST_TIMEOUT_MS, - routing: DEFAULT_ROUTING_POLICY, - alerts: { destinations: [] }, - admission: { timeoutMs: DEFAULT_ADMISSION_TIMEOUT_MS }, - }, - stalledTransition: { - thresholdMultiplier: 3, - minimumThresholdMs: 60_000, - }, - }; -} - async function readConfigFile(filesystem: Filesystem, configPath: string): Promise { if (!(await filesystem.exists(configPath))) { return {}; @@ -994,395 +260,6 @@ function validateConfigLayer( return result; } -const LOG_LEVELS: readonly LogLevel[] = ["debug", "info", "warn", "error"]; -const DOWNLOAD_POLICIES: readonly DownloadPolicy[] = ["never", "on-request", "always"]; -const LEASE_IDENTITIES: readonly LeaseIdentity[] = ["reusable", "fresh"]; - -const HEADER_NAME_PATTERN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]{1,100}$/; -const HEADER_VALUE_PATTERN = /^[\t -~]*$/; -const MAX_HEADERS = 16; -const MAX_HEADER_VALUE_LENGTH = 4096; -const MAX_HEADER_CHARACTERS = 8192; -const SIMLOCK_HEADER_NAMES: ReadonlySet = new Set([ - "authorization", - "x-simlock-worker-id", - "x-simlock-worker-label", - "host", - "connection", - "upgrade", - "origin", -]); - -type HeaderEntry = [name: string, text: string]; - -/** - * why: Node throws on a header name or value it cannot send, so every such entry is refused - * here at load, where the operator sees the key, rather than on each dial. - */ -function headersValidator(value: unknown, path: string): Readonly> { - const shape = "an object of header names to text values"; - if (typeof value !== "object" || value === null || Array.isArray(value)) { - throw invalidValue(path, shape); - } - const entries = Object.entries(value); - if (!entries.every((entry): entry is HeaderEntry => typeof entry[1] === "string")) { - throw invalidValue(path, shape); - } - if (entries.length > MAX_HEADERS) throw invalidValue(path, "at most 16 headers"); - checkHeaderNames(entries, path); - checkHeaderValues(entries, path); - for (const [name] of entries) { - const lower = name.toLowerCase(); - if (SIMLOCK_HEADER_NAMES.has(lower) || lower.startsWith("sec-websocket-")) { - throw invalidValue(path, `no header Simlock sets itself (${name})`); - } - } - return Object.fromEntries(entries); -} - -function admissionCommandValidator(value: unknown, path: string): readonly string[] { - const shape = `a list of 1 to ${String(ADMISSION_COMMAND_MAX_ENTRIES)} strings of 1 to ${String(ADMISSION_ARGUMENT_MAX_LENGTH)} characters`; - if ( - !Array.isArray(value) || - value.length < 1 || - value.length > ADMISSION_COMMAND_MAX_ENTRIES || - !value.every( - (entry): entry is string => - typeof entry === "string" && - entry.length >= 1 && - entry.length <= ADMISSION_ARGUMENT_MAX_LENGTH, - ) - ) { - throw invalidValue(path, shape); - } - return value; -} - -function admissionTimeoutValidator(value: unknown, path: string): number { - if ( - typeof value !== "number" || - !Number.isInteger(value) || - value < 1 || - value > ADMISSION_TIMEOUT_MAX_MS - ) { - throw invalidValue(path, `an integer from 1 to ${String(ADMISSION_TIMEOUT_MAX_MS)}`); - } - return value; -} - -function checkHeaderNames(entries: readonly HeaderEntry[], path: string): void { - const seen = new Set(); - for (const [name] of entries) { - if (!HEADER_NAME_PATTERN.test(name)) { - throw invalidValue( - path, - `a header name of 1 to 100 letters, digits and HTTP name symbols (got ${name})`, - ); - } - } - for (const [name] of entries) { - const lower = name.toLowerCase(); - if (seen.has(lower)) throw invalidValue(path, `each header name once (got ${name} twice)`); - seen.add(lower); - } -} - -function checkHeaderValues(entries: readonly HeaderEntry[], path: string): void { - let total = 0; - for (const [name, text] of entries) { - if (text.length > MAX_HEADER_VALUE_LENGTH || !HEADER_VALUE_PATTERN.test(text)) { - throw invalidValue( - path, - `a header value of at most 4096 plain ASCII characters (in ${name})`, - ); - } - total += name.length + text.length; - } - if (total > MAX_HEADER_CHARACTERS) { - throw invalidValue(path, "at most 8192 characters of header names and values in all"); - } -} - -/** - * why: The `capacity.config` validator is the selected strategy's own, so a strategy - * owns its options end to end and nothing here has to know their shape. - */ -export function configValidators( - strategy: CapacityStrategyName, - modules: readonly ModuleConfigEntry[], - authStrategies: readonly AuthStrategyDefinition[], - driverSections: readonly DriverConfigEntry[] = [], -): Record { - return { - capabilities: capabilitiesValidator(modules), - gateway: objectValidator({ - url: stringValue, - token: stringValue, - label: stringValue, - headers: headersValidator, - disconnectedRetentionMs: positiveNumber, - execTimeoutMs: positiveNumber, - leaseRequestTimeoutMs: positiveNumber, - routing: stringUnion(ROUTING_POLICIES), - alerts: objectValidator({ destinations: alertDestinationsValidator }), - admission: objectValidator({ - command: admissionCommandValidator, - timeoutMs: admissionTimeoutValidator, - }), - }), - capacity: objectValidator({ - strategy: stringUnion(capacityStrategyNames), - config: capacityStrategyValidator(strategy), - }), - // why: Legacy spelling of the resource options; still type-checked here so a typo - // inside them is still reported rather than silently dropped. - ...resourceOptionValidators, - drivers: driversValidator, - downloads: objectValidator({ - policy: stringUnion(DOWNLOAD_POLICIES), - acceptAndroidLicenses: booleanValue, - timeoutMs: positiveNumber, - }), - idle: objectValidator({ shutdownAfterMs: nonNegativeNumber, deleteAfterMs: nonNegativeNumber }), - update: objectValidator({ check: booleanValue }), - warmPool: objectValidator({ - ...warmPoolConfigValidator, - quarantine: objectValidator({ - maxRetries: positiveInteger, - retryBackoffMs: nonNegativeNumber, - retryBackoffMultiplier: numberAtLeast(1), - maxRetryBackoffMs: nonNegativeNumber, - }), - }), - // why: The three pre-ADR-0004 keys (`lease.detachedTtlMs`, `lease.heldTtlBackstopMs`, - // `lease.heartbeatIntervalMs`) are deliberately absent rather than aliased: `objectValidator` - // warns about each as an unrecognized key and ignores it, so an old config still boots and - // gets the new key's default rather than the value it wrote. - devices: objectValidator({ autoEnroll: booleanValue, deletedRetentionMs: positiveNumber }), - lease: objectValidator({ - defaultTtlMs: positiveNumber, - maxTtlMs: positiveNumber, - identity: objectValidator({ - ios: stringUnion(LEASE_IDENTITIES), - android: stringUnion(LEASE_IDENTITIES), - }), - requestRetentionMs: positiveNumber, - endedIdRetentionMs: positiveNumber, - maxRequestRecords: positiveInteger, - maxPerRequester: maxPerRequesterValidator, - endWaitMs: endWaitValidator, - }), - exec: objectValidator({ timeoutMs: positiveNumber }), - diskPressure: objectValidator({ freeBytesThreshold: nonNegativeNumber }), - memoryPressure: objectValidator({ - enabled: booleanValue, - pauseWarming: memoryPressureRuleValidator, - pauseNewDevices: memoryPressureRuleValidator, - clearAfterMs: nonNegativeNumber, - }), - eventBuffer: objectValidator({ capacity: positiveInteger }), - log: objectValidator({ level: stringUnion(LOG_LEVELS), rotateBytes: positiveInteger }), - eventLog: objectValidator({ - retention: positiveInteger, - maxBytes: positiveInteger, - }), - metrics: objectValidator({ retention: positiveInteger }), - http: objectValidator({ - enabled: booleanValue, - host: stringValue, - port: integerInRange(1, 65535), - ...Object.fromEntries( - authStrategies.flatMap((definition) => - definition.config === undefined - ? [] - : [[definition.config.key, schemaValidator(definition.config.schema)]], - ), - ), - }), - health: objectValidator({ - enabled: booleanValue, - probeIntervalMs: positiveNumber, - stableObservations: positiveInteger, - maxRecoveryAttempts: positiveInteger, - recoveryBackoffMs: positiveNumber, - maxConcurrentRecoveries: positiveInteger, - }), - ios: objectValidator({ - ...driverValidators(driverSections, "ios"), - defaultMode: stringUnion(["slim", "full"]), - defaultModels: defaultModelsValidator, - }), - android: objectValidator({ - ...driverValidators(driverSections, "android"), - defaultModels: defaultModelsValidator, - }), - stalledTransition: objectValidator({ - thresholdMultiplier: numberAtLeast(1), - minimumThresholdMs: nonNegativeNumber, - }), - }; -} - -const memoryPressureRuleValidator = objectValidator({ - level: stringUnion(["normal", "warn", "critical", "off"]), - minAvailablePercent: numberInRange(0, 100), -}); - -function driverValidators( - driverSections: readonly DriverConfigEntry[], - platform: Platform, -): Record { - return Object.assign( - {}, - ...driverSections - .filter((entry) => entry.platform === platform) - .map((entry) => entry.validators), - ) as Record; -} - -function driverDefaults( - driverSections: readonly DriverConfigEntry[], - platform: Platform, -): Record { - return Object.assign( - {}, - ...driverSections.filter((entry) => entry.platform === platform).map((entry) => entry.defaults), - ) as Record; -} - -function schemaValidator(schema: ZodType): Validator { - return (value, path) => { - const result = schema.safeParse(value); - if (result.success) return result.data; - const issue = result.error.issues[0]; - if (issue === undefined) throw invalidValue(path, "a valid value"); - const issuePath = [path, ...issue.path.map(String)].join("."); - // why: a strategy's own messages are lower-case phrases that finish "expected ..."; zod's built-in ones start with a capital. - const expected = /^[A-Z]/.test(issue.message) - ? `a valid value (${issue.message})` - : issue.message; - throw invalidValue(issuePath, expected); - }; -} - -function capabilitiesValidator(modules: readonly ModuleConfigEntry[]): Validator { - return (value, path, warn) => { - const result: Record = {}; - for (const [name, section] of Object.entries(requireObject(value, path))) { - const sectionPath = `${path}.${name}`; - const entry = modules.find((candidate) => candidate.name === name); - if (entry === undefined) { - warn(`Unknown config key: "${sectionPath}"`); - continue; - } - const { enabled, ...rest } = requireObject(section, sectionPath); - result[name] = { - ...(entry.validator(rest, sectionPath, warn) as Record), - ...(enabled === undefined - ? {} - : { enabled: booleanValue(enabled, `${sectionPath}.enabled`) }), - }; - } - return result; - }; -} - -export type DefaultModels = Partial>; - -/** why: `setTimeout` fires after about 1 ms above this delay, which would end the wait at once. */ -const MAXIMUM_TIMER_DELAY_MS = 2_147_483_647; - -const endWaitValidator: Validator = (value, path) => { - const wait = positiveNumber(value, path); - if (wait > MAXIMUM_TIMER_DELAY_MS) { - throw invalidValue(path, `a positive number no larger than ${MAXIMUM_TIMER_DELAY_MS}`); - } - return wait; -}; - -/** - * why: Each key is a device class and each value one non-empty model name or a non-empty list of - * them, stored as a list. Any other key is refused rather than warned about: a misspelled class - * would otherwise leave the operator's preference silently unused. - */ -const defaultModelsValidator: Validator = (value, path) => { - const object = requireObject(value, path); - const result: Record = {}; - - for (const [key, names] of Object.entries(object)) { - const keyPath = `${path}.${key}`; - if (!(DEVICE_CLASSES as readonly string[]).includes(key)) { - throw invalidValue(keyPath, `a device class (${DEVICE_CLASSES.join(", ")})`); - } - const list = typeof names === "string" ? [names] : names; - if (!Array.isArray(list) || list.length === 0 || !list.every(isNonEmptyString)) { - throw invalidValue(keyPath, "a model name or a non-empty list of model names"); - } - result[key] = list; - } - - return result; -}; - -function isNonEmptyString(value: unknown): value is string { - return typeof value === "string" && value.length > 0; -} - -/** - * why: Checks the shape of `drivers` and nothing else. Every leaf is type-checked, because a - * config file is hand-edited and a nested object or array there is a mistake worth - * naming -- but an unrecognised key inside a driver block is deliberately *not* a - * warning. The core does not know which keys a driver has, and warning about the ones it - * has not heard of would be the same leak as interpreting them. - */ -const driversValidator: Validator = (value, path) => { - const drivers = requireObject(value, path); - - return Object.fromEntries( - Object.entries(drivers).map(([name, block]) => [ - name, - driverSettings(block, `${path}.${name}`), - ]), - ); -}; - -function driverSettings(value: unknown, path: string): Record { - const settings = requireObject(value, path); - const result: Record = {}; - - for (const [key, leaf] of Object.entries(settings)) { - if (typeof leaf !== "string" && typeof leaf !== "number" && typeof leaf !== "boolean") { - throw invalidValue(`${path}.${key}`, "a string, number, or boolean"); - } - - result[key] = leaf; - } - - return result; -} - -function maxPerRequesterValidator(value: unknown, path: string, warn: Warn): unknown { - return typeof value === "object" && value !== null - ? objectValidator({ ios: positiveInteger, android: positiveInteger })(value, path, warn) - : positiveInteger(value, path); -} - -function mergeConfig(base: Base, overrides: object): Base { - const merged: Record = { ...(base as Record) }; - - for (const [key, override] of Object.entries(overrides)) { - const current = merged[key]; - merged[key] = - isObject(current) && isObject(override) ? mergeConfig(current, override) : override; - } - - return merged as Base; -} - -function isObject(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - function deepFreeze(value: Value): Value { if (typeof value === "object" && value !== null) { for (const child of Object.values(value)) { @@ -1393,7 +270,3 @@ function deepFreeze(value: Value): Value { return value; } - -type DeepPartial = Value extends object - ? { readonly [Key in keyof Value]?: DeepPartial } - : Value; From dd79de1b9153e12513a7577e79b75e68ab629c8d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:06:25 +0100 Subject: [PATCH 07/83] refactor(contract): split operations.ts into parts Protocol-bump: none (refactor: operations.ts split into files, no wire shape changes) --- src/contract/operation-definition.ts | 44 + src/contract/operations-admin.ts | 294 +++++ src/contract/operations-fleet.ts | 344 ++++++ src/contract/operations-lease-request.ts | 319 ++++++ src/contract/operations-leases.ts | 255 +++++ src/contract/operations.ts | 1331 ++-------------------- 6 files changed, 1369 insertions(+), 1218 deletions(-) create mode 100644 src/contract/operation-definition.ts create mode 100644 src/contract/operations-admin.ts create mode 100644 src/contract/operations-fleet.ts create mode 100644 src/contract/operations-lease-request.ts create mode 100644 src/contract/operations-leases.ts diff --git a/src/contract/operation-definition.ts b/src/contract/operation-definition.ts new file mode 100644 index 000000000..302a9b84a --- /dev/null +++ b/src/contract/operation-definition.ts @@ -0,0 +1,44 @@ +import { z } from "zod"; +import { type AuthorizeContext, type Role } from "./roles.js"; + +export type Effect = "read" | "write"; + +export interface OperationDefinition< + Name extends string = string, + InputSchema extends z.ZodType = z.ZodType, + OutputSchema extends z.ZodType = z.ZodType, +> { + readonly name: Name; + readonly role: Role | ((input: z.infer) => Role); + readonly effect: Effect | ((input: z.infer) => Effect); + readonly input: InputSchema; + readonly output: OutputSchema; + readonly authorize?: (input: z.infer, context: AuthorizeContext) => boolean; +} + +export function defineOperation< + Name extends string, + InputSchema extends z.ZodType, + OutputSchema extends z.ZodType, +>( + definition: OperationDefinition, +): OperationDefinition { + return definition; +} + +/** + * why: `device.exec`'s and `device.rotate`'s one rule, so a gateway's admin session is held to the lease's requester on both. + */ +export function authorizeOneLeaseCall( + input: { readonly leaseId: string; readonly requesterId?: string | undefined }, + context: AuthorizeContext, +): boolean { + if (context.role !== "admin") { + if (input.requesterId !== undefined) return false; + const ownerId = context.ownerId(input.leaseId); + return ownerId === undefined || ownerId === context.principal; + } + const requesterId = input.requesterId ?? context.principal; + const leaseRequesterId = context.leaseRequesterId(input.leaseId); + return leaseRequesterId === undefined || leaseRequesterId === requesterId; +} diff --git a/src/contract/operations-admin.ts b/src/contract/operations-admin.ts new file mode 100644 index 000000000..3b722a53c --- /dev/null +++ b/src/contract/operations-admin.ts @@ -0,0 +1,294 @@ +import { z } from "zod"; +import { + EVENTS_PAGE_MAX, + EVENTS_PAGE_WITH_SINCE_MESSAGE, + EVENTS_WORKER_MAX, + eventsPageWithoutSince, +} from "./events-page.js"; +import { defineOperation, type Effect } from "./operation-definition.js"; +import { type Role } from "./roles.js"; +import { + baselineEntrySchema, + baselineNameSchema, + baselineNoteSchema, + cleanupRuleSummarySchema, + configSchema, + daemonHealthSchema, + daemonUpdateFields, + deviceRecordSchema, + doctorReportSchema, + EVENT_ID_PATTERN, + eventEnvelopeSchema, + imageTagSchema, + nukeReportSchema, + platformSchema, + proposalSchema, + statusDeviceSchema, + statusLeaseSchema, + tokenRecordSchema, + tokenRoleSchema, + updateRequestedVersionSchema, + usageOutputSchema, + usageWindowSchema, + waitingRequestSchema, + WARM_POOL_LEARNED_KINDS_MAX, + WARM_POOL_NAME_MAX, +} from "./schemas.js"; + +const doctorRunInputSchema = z.object({ + fix: z.boolean().optional(), + /** + * why: Destroys every orphan the run finds (ADR 0001, decision 5) -- the single opt-in exception + * to registry-only destruction (safety rule 1). Its own flag rather than part of `fix`, so a + * `doctor --fix` already running unattended in CI does not acquire a destructive behaviour on + * upgrade. + */ + purgeOrphans: z.boolean().optional(), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const doctorRun = defineOperation({ + name: "doctor.run", + /** + * why: The role genuinely depends on the input: `fix: false` (read-only, shells out per device) + * is agent-visible; `fix: true` (applies safe fixes to registry-owned devices) is admin + * only -- see ADR §3's matrix, which lists `doctor.run` twice for exactly this reason. A + * function-of-input role, rather than an `authorize` hook, is the right tool here: `authorize` + * (ADR §1's `ownsLease` example) is for *ownership* checks against a specific resource once a + * role has already cleared the gate; this is a *role* decision, and it is entirely computable + * from the already-validated input with no session/resource context needed. Keeping it as + * `role` also means the dispatcher's role-check step (PR 2) stays uniform -- "compute the + * required role from the input, compare to the session role" -- for every operation, instead + * of `doctor.run` needing a second, bespoke gating path. + */ + role: (input: z.infer): Role => + input.fix || input.purgeOrphans ? "admin" : "agent", + effect: (input: z.infer): Effect => + input.fix || input.purgeOrphans ? "write" : "read", + input: doctorRunInputSchema, + output: doctorReportSchema, +}); + +/** Arm order is load-bearing: a full record matches the first arm, so a worker's answer is unchanged. */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const listGet = defineOperation({ + name: "list.get", + role: "admin", + effect: "read", + input: z.object({ kind: z.enum(["devices", "leases", "rules", "requests"]).optional() }), + output: z.union([ + z.array(deviceRecordSchema), + z.array(statusDeviceSchema), + z.array(statusLeaseSchema), + z.array(cleanupRuleSummarySchema), + z.array(waitingRequestSchema), + ]), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const cleanupRun = defineOperation({ + name: "cleanup.run", + role: "admin", + effect: (input: { dryRun?: boolean | undefined }): Effect => (input.dryRun ? "read" : "write"), + input: z.object({ dryRun: z.boolean().optional(), rule: z.string().optional() }), + output: z.array(proposalSchema), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const nukeRun = defineOperation({ + name: "nuke.run", + role: "admin", + effect: "write", + input: z.object({ deleteDevices: z.boolean().optional() }), + output: nukeReportSchema, +}); + +const warmKindSchema = z.object({ + platform: platformSchema, + model: z.string().max(WARM_POOL_NAME_MAX), + osVersion: z.string().max(WARM_POOL_NAME_MAX), + mode: z.enum(["slim", "full"]), + imageTag: imageTagSchema.optional(), + baseline: baselineNameSchema.optional(), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const warmForget = defineOperation({ + name: "warm.forget", + role: "admin", + effect: "write", + input: z.object({ + platform: platformSchema, + model: z.string().min(1).max(WARM_POOL_NAME_MAX), + osVersion: z.string().min(1).max(WARM_POOL_NAME_MAX).optional(), + mode: z.enum(["slim", "full"]).optional(), + imageTag: imageTagSchema.optional(), + }), + output: z.object({ forgotten: z.array(warmKindSchema).max(WARM_POOL_LEARNED_KINDS_MAX) }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const baselineSave = defineOperation({ + name: "baseline.save", + role: "admin", + effect: "write", + input: z + .object({ + leaseId: z.string().min(1).max(256), + name: baselineNameSchema, + note: baselineNoteSchema.optional(), + }) + .strict(), + output: z.object({ baseline: baselineEntrySchema, leaseId: z.string().max(256) }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const baselineList = defineOperation({ + name: "baseline.list", + role: "agent", + effect: "read", + input: z.object({}).strict(), + output: z.object({ baselines: z.array(baselineEntrySchema).max(1000) }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const baselineRemove = defineOperation({ + name: "baseline.remove", + role: "admin", + effect: "write", + // why: `workerId` names the worker a gateway forwards the remove to; a gateway requires it and strips it, a worker refuses it. + input: z + .object({ + name: baselineNameSchema, + workerId: z.string().min(1).max(EVENTS_WORKER_MAX).optional(), + }) + .strict(), + output: z.object({ + name: baselineNameSchema, + versions: z.array(z.number().int().min(1)).max(1000), + deletedDevices: z.array(z.string().max(256)).max(1000), + }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const configGet = defineOperation({ + name: "config.get", + role: "admin", + effect: "read", + input: z.object({ showSecrets: z.boolean().optional() }), + output: configSchema, +}); + +// fallow-ignore-next-line unused-export -- see the comment above. +export const daemonStop = defineOperation({ + name: "daemon.stop", + role: "admin", + effect: "write", + input: z.object({}), + output: z.object({ stopping: z.literal(true) }), +}); + +// why: `DaemonServer` dispatches it on a connection whose protocol range does not overlap (like `daemon.stop`), so its input and answer never change shape. +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const daemonUpdate = defineOperation({ + name: "daemon.update", + role: "admin", + effect: "write", + input: z + .object({ + version: updateRequestedVersionSchema.optional(), + force: z.boolean().optional(), + query: z.boolean().optional(), + }) + .refine( + (input) => input.query !== true || (input.version === undefined && input.force === undefined), + { + message: "query takes no other field", + }, + ), + output: z.object({ + started: z.boolean(), + upToDate: z.boolean(), + resolved: updateRequestedVersionSchema.nullable(), + version: daemonUpdateFields.version, + health: daemonHealthSchema, + update: daemonUpdateFields.update, + lastUpdateFailure: daemonUpdateFields.lastUpdateFailure, + latestVersion: daemonUpdateFields.latestVersion, + }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const eventsReplay = defineOperation({ + name: "events.replay", + role: "admin", + effect: "read", + input: z + .object({ + beforeId: z.string().regex(EVENT_ID_PATTERN).optional(), + limit: z.number().int().min(1).max(EVENTS_PAGE_MAX).optional(), + sinceTs: z.number().optional(), + workerId: z.string().min(1).max(EVENTS_WORKER_MAX).optional(), + }) + .refine(eventsPageWithoutSince, { message: EVENTS_PAGE_WITH_SINCE_MESSAGE }), + output: z.array(eventEnvelopeSchema), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const usageGet = defineOperation({ + name: "usage.get", + role: "admin", + effect: "read", + input: usageWindowSchema, + output: usageOutputSchema, +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const eventsSubscribe = defineOperation({ + name: "events.subscribe", + role: "admin", + effect: "read", + input: z.object({}), + output: z.object({ subscribed: z.literal(true), subscriptionId: z.string() }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const eventsUnsubscribe = defineOperation({ + name: "events.unsubscribe", + role: "admin", + effect: "read", + input: z.object({}), + output: z.object({ subscribed: z.literal(false) }), +}); + +// why: ADR §11 makes the daemon the only owner of the tokens, and that move has landed: both +// dispatchers handle these three, the CLI reaches them as admin round-trips from +// `src/cli/commands/token.ts`, and no CLI code reads the token store itself. The declarations live +// here because the `OPERATIONS` map every dispatcher validates against is built from this file. + +// fallow-ignore-next-line unused-export -- reachable from outside this file only through OPERATIONS. +export const tokenCreate = defineOperation({ + name: "token.create", + role: "admin", + effect: "write", + input: z.object({ role: tokenRoleSchema, label: z.string().optional() }), + output: z.object({ secret: z.string(), token: tokenRecordSchema }), +}); + +// fallow-ignore-next-line unused-export -- same as tokenCreate above. +export const tokenList = defineOperation({ + name: "token.list", + role: "admin", + effect: "read", + input: z.object({}), + output: z.object({ tokens: z.array(tokenRecordSchema) }), +}); + +// fallow-ignore-next-line unused-export -- same as tokenCreate above. +export const tokenRevoke = defineOperation({ + name: "token.revoke", + role: "admin", + effect: "write", + input: z.object({ id: z.string() }), + output: z.object({ revoked: z.boolean() }), +}); diff --git a/src/contract/operations-fleet.ts b/src/contract/operations-fleet.ts new file mode 100644 index 000000000..ab40c6351 --- /dev/null +++ b/src/contract/operations-fleet.ts @@ -0,0 +1,344 @@ +import { z } from "zod"; +import { defineOperation } from "./operation-definition.js"; +import { daemonUpdate } from "./operations-admin.js"; +import { componentVersionSchema, leaseIdSchema } from "./operations-lease-request.js"; +import { relayChannelSchema, relayHeadersSchema, relayIdSchema } from "./relay.js"; +import { + capabilityNameSchema, + inboxItemSchema, + platformSchema, + updateRequestedVersionSchema, + workerViewSchema, +} from "./schemas.js"; + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerList = defineOperation({ + name: "worker.list", + role: "admin", + effect: "read", + input: z.object({}), + output: z.object({ workers: z.array(workerViewSchema) }), +}); + +/** + * ADR 0005 §9: a drained worker keeps its existing leases and receives no new dispatches -- + * the operator's tool for taking a machine down without killing anyone's device. This PR + * records the flag (persisted, so it survives a worker reconnect and a gateway restart) and + * reports it; #118's dispatcher is what honours it, because this PR has no dispatch to skip. + * + * Idempotent: draining an already-drained worker succeeds and changes nothing. An id the + * gateway has no view of is `UNKNOWN_WORKER` -- draining a machine that is not in the fleet is + * a typo, and answering "done" would leave an operator believing a machine was taken out of + * service when nothing was. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerDrain = defineOperation({ + name: "worker.drain", + role: "admin", + effect: "write", + input: z.object({ workerId: z.string().min(1) }), + output: z.object({ workerId: z.string(), drained: z.literal(true) }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerUndrain = defineOperation({ + name: "worker.undrain", + role: "admin", + effect: "write", + input: z.object({ workerId: z.string().min(1) }), + output: z.object({ workerId: z.string(), drained: z.literal(false) }), +}); + +/** + * ADR 0005 §8: forgets a *disconnected* worker's view. A connected one is refused with + * `WORKER_CONNECTED` -- removing it would only make the gateway forget a machine that is about + * to announce itself again on its very next refresh, which is a confusing no-op rather than an + * operator action. + * + * `removed: false` is the answer for an id with no view: unlike drain, "forget this" is + * already true of something the gateway has never heard of or has already forgotten, so it is + * an outcome to report rather than an error to raise (the same shape `token.revoke` uses). + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerRemove = defineOperation({ + name: "worker.remove", + role: "admin", + effect: "write", + input: z.object({ workerId: z.string().min(1) }), + output: z.object({ workerId: z.string(), removed: z.boolean() }), +}); + +// why: ADR 0042 §2: the gateway sends `daemon.update` over that worker's uplink and answers with the worker's own answer plus the id. +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerUpdate = defineOperation({ + name: "worker.update", + role: "admin", + effect: "write", + input: z.object({ + workerId: z.string().min(1), + version: updateRequestedVersionSchema.optional(), + force: z.boolean().optional(), + }), + output: daemonUpdate.output.extend({ workerId: z.string() }), +}); + +/** + * Installs one iOS simulator runtime or Android system image without leasing a device (ADR 0010 + * §6). Admin-only: the command is an operator's consent to a multi-GB download, which + * `downloads.policy: "never"` still refuses (§4). Not durable: a daemon stop ends the install, + * and running it again is the recovery. `component` echoes the version asked for; `version` is + * the exact one installed or found. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const componentInstall = defineOperation({ + name: "component.install", + role: "admin", + effect: "write", + input: z.object({ platform: platformSchema, version: componentVersionSchema }), + output: z.object({ + platform: platformSchema, + component: z.string(), + outcome: z.enum(["installed", "already-installed"]), + version: z.string(), + }), +}); + +/** + * Every installed iOS simulator runtime and Android system image on this machine, ordered by + * platform, then version, then variant. A read for any session. `installedBySimlock` is true only + * when Simlock's record of an install names this exact component; `installedAt` is that + * record's time. `devices` counts Simlock's own devices of this platform and version in any + * state but deleted, so two variants of one version show the same count; `foreignDevices` + * counts the devices outside Simlock that use it. `variant` tells two components of one + * version apart, in the platform's own words; `sizeBytes` is absent when it could not be read. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const componentList = defineOperation({ + name: "component.list", + role: "agent", + effect: "read", + input: z.object({ platform: platformSchema.optional() }), + output: z.object({ + components: z.array( + z.object({ + platform: platformSchema, + version: z.string(), + variant: z.string().optional(), + sizeBytes: z.number().int().nonnegative().optional(), + installedBySimlock: z.boolean(), + installedAt: z.number().int().optional(), + devices: z.number().int().nonnegative(), + foreignDevices: z.number().int().nonnegative(), + }), + ), + }), +}); + +const WORKER_ID_MAX_LENGTH = 128; + +/** The longest error message a relayed result carries. A worker's message is a claim, cut to + * this before it is stored in a result (safety rule 10). */ +export const WORKER_RESULT_MESSAGE_MAX_LENGTH = 1024; + +const workerInstallTargetsSchema = z.union([ + z.literal("all"), + z + .array(z.string().min(1).max(WORKER_ID_MAX_LENGTH)) + .min(1) + .max(64) + .refine((ids) => new Set(ids).size === ids.length, "worker ids must be distinct"), +]); + +const workerInstallResultSchema = z.object({ + workerId: z.string(), + label: z.string().optional(), + outcome: z.enum(["installed", "already-installed", "refused", "failed", "skipped", "unknown"]), + version: z.string().max(64).optional(), + error: z + .object({ + code: z.string().max(64), + message: z.string().max(WORKER_RESULT_MESSAGE_MAX_LENGTH), + }) + .optional(), +}); + +/** + * ADR 0010 §7: installs one component on named workers, or on every connected one, through a + * gateway. The gateway asks each target at the same time with `component.install`, and each + * worker answers for itself under its own `downloads.policy`. One result per worker, in + * ascending worker id. Progress arrives as `component-progress` pushes carrying `workerId`. + * A worker answers `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012 §2). + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerInstallComponent = defineOperation({ + name: "worker.install-component", + role: "admin", + effect: "write", + input: z.object({ + platform: platformSchema, + version: componentVersionSchema, + workers: workerInstallTargetsSchema, + }), + output: z.object({ results: z.array(workerInstallResultSchema) }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const componentRemove = defineOperation({ + name: "component.remove", + role: "admin", + effect: "write", + input: z.object({ + platform: platformSchema, + version: componentVersionSchema, + dryRun: z.boolean().optional(), + }), + output: z.object({ + platform: platformSchema, + version: z.string(), + outcome: z.enum(["removed", "would-remove"]), + sizeBytes: z.number().int().nonnegative().optional(), + residue: z.string().optional(), + }), +}); + +const ENROLLED_DEVICE_ID_MAX_LENGTH = 200; + +const enrollmentDeviceIdSchema = z + .string() + .min(1) + .max(ENROLLED_DEVICE_ID_MAX_LENGTH) + .refine( + (id) => [...id].every((character) => !/\s/u.test(character) && !isControlCharacter(character)), + "must not contain whitespace or control characters", + ) + .refine((id) => !id.startsWith("-"), "must not start with -"); + +function isControlCharacter(character: string): boolean { + const code = character.codePointAt(0) ?? 0; + return code <= 0x1f || (code >= 0x7f && code <= 0x9f); +} + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const deviceAdd = defineOperation({ + name: "device.add", + role: "admin", + effect: "write", + input: z.object({ platform: platformSchema, id: enrollmentDeviceIdSchema }).strict(), + output: z.object({ + id: z.string(), + driverDeviceId: z.string(), + platform: platformSchema, + model: z.string(), + osVersion: z.string(), + class: z.enum(["phone", "tablet"]), + appCount: z.number().int().nonnegative(), + reenrolled: z.boolean(), + }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const deviceRemove = defineOperation({ + name: "device.remove", + role: "admin", + effect: "write", + input: z.object({ id: enrollmentDeviceIdSchema }).strict(), + output: z.object({ id: z.string(), driverDeviceId: z.string() }), +}); + +const RELAY_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"] as const; + +const relayUrlSchema = z + .string() + .max(4096) + .refine((value) => { + try { + const { protocol } = new URL(value); + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } + }); + +/** + * One HTTP exchange a gateway hands to a module on a worker, over the gateway's uplink. The + * request body arrives, and the response body leaves, on the byte channel `channel` names (ADR 0028); + * the head leaves as a `relay-head` push keyed by this call's frame id. The + * answer comes once the response body has ended, or `{ cancelled: true }` once a `relay.cancel` took + * effect. With `twoWay` (GET, no body) both directions of the channel stay open: the answer comes once + * both have ended. Admin only, and on a worker only the gateway's own uplink session may call it. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const relayRequest = defineOperation({ + name: "relay.request", + role: "admin", + effect: "write", + input: z + .object({ + relayId: relayIdSchema, + target: capabilityNameSchema, + method: z.enum(RELAY_METHODS), + url: relayUrlSchema, + headers: relayHeadersSchema, + tenant: z.string().min(1).max(300), + leaseId: leaseIdSchema.optional(), + hasBody: z.boolean(), + twoWay: z.literal(true).optional(), + channel: relayChannelSchema, + }) + .strict() + .refine((input) => !input.hasBody || (input.method !== "GET" && input.method !== "HEAD"), { + message: "hasBody is not allowed with GET or HEAD", + }) + .refine((input) => input.twoWay !== true || (input.method === "GET" && !input.hasBody), { + message: "twoWay needs GET and no body", + }), + output: z.object({ cancelled: z.literal(true).optional() }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const relayCancel = defineOperation({ + name: "relay.cancel", + role: "admin", + effect: "write", + input: z.object({ relayId: relayIdSchema }).strict(), + output: z.object({}), +}); + +/** + * why: ADR 0035 §1: asks the gateway to post one sample message to one of its alert destinations, + * once, outside the destination's queue. `destination` is only looked up by name among the + * configured ones, never used in a URL. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const alertsTest = defineOperation({ + name: "alerts.test", + role: "admin", + effect: "write", + input: z.object({ destination: z.string().min(1).max(32) }), + output: z.object({ destination: z.string(), status: z.number() }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const inboxGet = defineOperation({ + name: "inbox.get", + role: "admin", + effect: "read", + input: z.object({}), + output: z.object({ items: z.array(inboxItemSchema) }), +}); + +const identityGet = defineOperation({ + name: "identity.get", + role: "agent", + effect: "read", + input: z.object({}), + output: z.object({ + principal: z.string(), + name: z.string(), + role: z.enum(["admin", "agent"]), + via: z.enum(["token", "proxy", "socket", "uplink"]), + signOutUrl: z.string().optional(), + }), +}); + +export { identityGet }; diff --git a/src/contract/operations-lease-request.ts b/src/contract/operations-lease-request.ts new file mode 100644 index 000000000..a79feb151 --- /dev/null +++ b/src/contract/operations-lease-request.ts @@ -0,0 +1,319 @@ +import { z } from "zod"; +import { defineOperation } from "./operation-definition.js"; +import { parseOsConstraint } from "./os-range.js"; +import { + baselineNameSchema, + capabilityNameSchema, + type DeviceClass, + deviceClassSchema, + EXCEPT_MAX_NAMES, + EXCEPT_NAME_PATTERN, + imageTagSchema, + leaseGrantSchema, + platformSchema, + requesterIdSchema, + WARM_POOL_NAME_MAX, +} from "./schemas.js"; + +/** + * The lease ID a requester may choose (ADR 0020 §1): ASCII, 1 to 64 characters, starting with a + * letter or digit, then letters, digits, `-` and `_`, case-sensitive. It has no `.`, so it can + * never look like a gateway's `.`. The one definition: a gateway reads it too, + * to decide which reported lease it names bare. + */ +export const LEASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; + +export const leaseIdSchema = z.string().regex(LEASE_ID_PATTERN); + +/** + * why: a component's version (`26.4`, `35`) reaches the driver unread and becomes an installer + * argument, so its shape is bounded here (safety rule 10): no whitespace or control character, no + * `;` (sdkmanager's package path separator), no leading `-` (an option). An exact `osVersion` in + * `lease.request` is held to it too, because a downloadable missing runtime installs that string. + */ +export const componentVersionSchema = z + .string() + .min(1) + .max(64) + .regex(/^[^\s\p{Cc};]+$/u, "must contain no whitespace, control character or ';'") + .refine((version) => !version.startsWith("-"), "must not start with '-'"); + +const EXACT_OS_VERSION_BOUND = + "an exact OS version is 1 to 64 characters, none of them whitespace, a control character or `;`, and does not start with `-`"; + +const leaseRequestBaseSchema = z + .object({ + model: z.string().min(1).max(WARM_POOL_NAME_MAX).optional(), + class: deviceClassSchema.optional(), + platform: platformSchema, + // why: A plain string, checked by `refuseBadOsVersion`: a refined field would be checked by the MCP SDK + // before the handler, and its error would stand in for `BAD_REQUEST`. + osVersion: z.string().optional(), + mode: z.enum(["slim", "full"]).optional(), + // why: A plain array of plain strings, bounded by `refuseBadExcept` as the one object-level check, like `osVersion`: a bound on the field itself would be checked by the MCP SDK before the handler. + except: z.array(z.string()).optional(), + imageTag: imageTagSchema.optional(), + baseline: baselineNameSchema.optional(), + physical: z.boolean().optional(), + requesterId: requesterIdSchema.optional(), + allowDownload: z.boolean().optional(), + noWait: z.boolean().optional(), + timeoutMs: z.number().finite().positive().optional(), + /** why: Capped by the dispatcher's handler, not here: the cap is daemon config this module cannot see. */ + ttlMs: z.number().finite().positive().optional(), + owner: z.string().optional(), + idempotencyKey: z.string().min(1).max(200).optional(), + leaseId: leaseIdSchema.optional(), + fleetRequestId: z.string().min(1).max(200).optional(), + // why: A plain record of plain strings, bounded by `refuseBadMetadata` as one object-level check, like `except`: a bound on the field itself would be checked by the MCP SDK before the handler. + metadata: z.record(z.string(), z.string()).optional(), + requires: z + .array(capabilityNameSchema) + .max(8) + .refine((names) => new Set(names).size === names.length, "a capability name appears twice") + .optional(), + }) + .strict(); + +export function refuseModelWithClass( + modelField = "model", +): ( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.model !== undefined && input.class !== undefined) { + context.addIssue({ + code: "custom", + message: `a request names a model or a class, not both: send only one of \`${modelField}\` and \`class\``, + path: ["class"], + }); + } + }; +} + +function refuseBaselineWithClass(): < + T extends { readonly baseline?: unknown; readonly class?: unknown }, +>( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.baseline !== undefined && input.class !== undefined) { + context.addIssue({ + code: "custom", + message: + "a request names a baseline or a class, not both: send only one of `baseline` and `class`", + path: ["class"], + }); + } + }; +} + +/** why: `allowDownload: false` passes because the CLI always sends it. */ +function refusePhysicalWithImageFields(): < + T extends { + readonly physical?: boolean | undefined; + readonly mode?: unknown; + readonly imageTag?: unknown; + readonly baseline?: unknown; + readonly allowDownload?: boolean | undefined; + }, +>( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.physical !== true) return; + const refused = [ + input.mode === undefined ? undefined : "mode", + input.imageTag === undefined ? undefined : "imageTag", + input.baseline === undefined ? undefined : "baseline", + input.allowDownload === true ? "allowDownload" : undefined, + ]; + for (const field of refused) { + if (field === undefined) continue; + context.addIssue({ + code: "custom", + message: `\`${field}\` has no meaning for a physical device`, + path: [field], + }); + } + }; +} + +// why: a replay would have to be checked against a capability state that changed since. +function refuseRequiresWithIdempotencyKey(): < + T extends { readonly requires?: unknown; readonly idempotencyKey?: string | undefined }, +>( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.requires === undefined || input.idempotencyKey === undefined) return; + context.addIssue({ + code: "custom", + message: "requires cannot be combined with idempotencyKey", + path: ["requires"], + }); + }; +} + +/** why: one object-level check with no path (like `osVersion`), so every transport, MCP included, answers `BAD_REQUEST` with the message as written. */ +function refuseBadExcept(): < + T extends { + readonly except?: readonly string[] | undefined; + readonly mode?: unknown; + readonly physical?: boolean | undefined; + readonly platform: string; + }, +>( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + const { except } = input; + if (except === undefined) return; + const message = exceptRefusal(input, except); + if (message !== undefined) context.addIssue({ code: "custom", message }); + }; +} + +function exceptRefusal( + input: { + readonly mode?: unknown; + readonly physical?: boolean | undefined; + readonly platform: string; + }, + except: readonly string[], +): string | undefined { + if (except.length < 1 || except.length > EXCEPT_MAX_NAMES) { + return `except names 1 to ${EXCEPT_MAX_NAMES} categories`; + } + const bad = except.find((name) => !EXCEPT_NAME_PATTERN.test(name)); + if (bad !== undefined) return `except: "${bad}" is not a category name`; + const twice = except.find((name, index) => except.indexOf(name) !== index); + if (twice !== undefined) return `except names "${twice}" twice`; + if (input.physical === true || input.platform !== "ios") return "except is for iOS simulators"; + if (input.mode === "full") return "except needs a slim lease (mode is full)"; + return undefined; +} + +const METADATA_MAX_KEYS = 16; +const METADATA_KEY_MAX = 64; +const METADATA_VALUE_MAX = 256; +const METADATA_TOTAL_MAX = 2048; + +function metadataRefusal(metadata: Readonly>): string | undefined { + const entries = Object.entries(metadata); + if (entries.length > METADATA_MAX_KEYS) return `metadata has more than ${METADATA_MAX_KEYS} keys`; + const badKey = entries.find(([key]) => key.length < 1 || key.length > METADATA_KEY_MAX); + if (badKey !== undefined) { + return `metadata key must be 1 to ${METADATA_KEY_MAX} characters (got ${badKey[0]})`; + } + const badValue = entries.find(([, value]) => value.length > METADATA_VALUE_MAX); + if (badValue !== undefined) { + return `metadata value for ${badValue[0]} must be at most ${METADATA_VALUE_MAX} characters`; + } + const total = entries.reduce((sum, [key, value]) => sum + key.length + value.length, 0); + if (total > METADATA_TOTAL_MAX) + return `metadata is longer than ${METADATA_TOTAL_MAX} characters in all`; + return undefined; +} + +function refuseBadMetadata(): < + T extends { readonly metadata?: Readonly> | undefined }, +>( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.metadata === undefined) return; + const message = metadataRefusal(input.metadata); + if (message !== undefined) context.addIssue({ code: "custom", message }); + }; +} + +function refuseBadOsVersion(): ( + input: T, + context: z.RefinementCtx, +) => void { + return (input, context) => { + if (input.osVersion === undefined) return; + const parsed = parseOsConstraint(input.osVersion); + if (!parsed.ok) { + context.addIssue({ + code: "custom", + message: parsed.message, + path: ["osVersion"], + }); + return; + } + if ( + parsed.constraint.kind === "exact" && + !componentVersionSchema.safeParse(parsed.constraint.version).success + ) { + context.addIssue({ + code: "custom", + message: EXACT_OS_VERSION_BOUND, + path: ["osVersion"], + }); + } + }; +} + +export const leaseRequestFields = leaseRequestBaseSchema; + +const leaseRequestInputSchema = leaseRequestBaseSchema + .superRefine(refuseModelWithClass()) + .superRefine(refuseBaselineWithClass()) + .superRefine(refuseBadOsVersion()) + .superRefine(refuseBadExcept()) + .superRefine(refuseBadMetadata()) + .superRefine(refusePhysicalWithImageFields()) + .superRefine(refuseRequiresWithIdempotencyKey()); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseRequest = defineOperation({ + name: "lease.request", + role: "agent", + effect: "write", + input: leaseRequestInputSchema, + output: leaseGrantSchema, + // why: No `authorize` hook on purpose: any agent session may request under an arbitrary + // `requesterId` (printable, at most 256 characters), and `owner` is gated by role in the handler (the resource does not exist yet). +}); + +export function requestedDevice( + input: Omit, "except"> & { + readonly except?: readonly string[] | undefined; + }, +) { + return { + ...(input.model === undefined ? {} : { model: input.model }), + ...(input.class === undefined ? {} : { class: input.class }), + platform: input.platform, + ...(input.osVersion === undefined ? {} : { osVersion: input.osVersion }), + ...(input.mode === undefined ? {} : { mode: input.mode }), + ...(input.except === undefined ? {} : { except: [...input.except] }), + ...(input.imageTag === undefined ? {} : { imageTag: input.imageTag }), + ...(input.baseline === undefined ? {} : { baseline: input.baseline }), + ...(input.physical === true ? { physical: true as const } : {}), + }; +} + +export function requestedClass(request: { + readonly model?: undefined; + readonly class?: DeviceClass | undefined; +}): DeviceClass; +export function requestedClass(request: { + readonly model?: string | undefined; + readonly class?: DeviceClass | undefined; +}): DeviceClass | undefined; +export function requestedClass(request: { + readonly model?: string | undefined; + readonly class?: DeviceClass | undefined; +}): DeviceClass | undefined { + if (request.class !== undefined) return request.class; + return request.model === undefined ? "phone" : undefined; +} diff --git a/src/contract/operations-leases.ts b/src/contract/operations-leases.ts new file mode 100644 index 000000000..15b645ac5 --- /dev/null +++ b/src/contract/operations-leases.ts @@ -0,0 +1,255 @@ +import { z } from "zod"; +import { authorizeOneLeaseCall, defineOperation } from "./operation-definition.js"; +import { INPUT_ORIENTATIONS } from "./orientation.js"; +import { ownsLease } from "./roles.js"; +import { + leaseRecordSchema, + passthroughCommandSchema, + platformSchema, + requesterIdSchema, + statusLeaseSchema, +} from "./schemas.js"; + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseCancel = defineOperation({ + name: "lease.cancel", + role: "agent", + effect: "write", + input: z.object({ + requesterId: requesterIdSchema.optional(), + platform: platformSchema.optional(), + }), + output: z.object({ result: z.enum(["cancelled", "not-found", "not-cancellable"]) }), + /** + * why: ADR §9: "cancels this principal's pending request by requester id". Gated on the pending + * request's *recorded owner* (`pendingRequestOwner`, ADR §4's `ownerId` -- always the + * creating session's principal, never the caller-suppliable `requesterId`), not on comparing + * `requesterId` to the principal directly -- that comparison would forbid exactly the case + * ADR §4 exists for: one connection (`principal: "host"`) proxying many agents, each under + * its own `requesterId` (`"agent-7"`). Deliberately not `ownsLease` (that hook resolves a + * *lease's* recorded `ownerId` from the registry; a pending, not-yet-granted request has no + * lease yet to look up -- `pendingRequestOwner` is the wait-queue equivalent). An omitted + * `requesterId` defaults to the principal the same way the handler's own default does (see + * `dispatcher.ts`'s `#leaseCancel`), so it always resolves to a request this principal owns. + * A `requesterId` with no pending request resolves `pendingRequestOwner` to `undefined`, + * which is treated as authorized (same convention as `ownsLease`) so the handler's own + * `not-found` surfaces instead of a misleading `FORBIDDEN`. + */ + authorize: (input, context) => { + if (context.role === "admin") return true; + const owner = context.pendingRequestOwner( + input.requesterId ?? context.principal, + input.platform, + ); + return owner === undefined || owner === context.principal; + }, +}); + +/** + * ADR 0004 §1: the only thing that keeps a lease alive, on every transport. An omitted `ttlMs` + * re-applies the lease's own stored width -- never `lease.defaultTtlMs` -- so a lease granted + * for four hours does not shrink to fifteen minutes the first time something renews it. A + * `ttlMs` above `lease.maxTtlMs` is `BAD_REQUEST`, checked in the dispatcher's handler where + * that config value is visible. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseRenew = defineOperation({ + name: "lease.renew", + role: "agent", + effect: "write", + input: z.object({ leaseId: z.string(), ttlMs: z.number().finite().positive().optional() }), + output: leaseRecordSchema, + authorize: ownsLease((input) => input.leaseId), +}); + +export const leaseRelease = defineOperation({ + name: "lease.release", + role: "agent", + effect: "write", + input: z.object({ leaseId: z.string() }), + output: z.object({ leaseId: z.string() }), + authorize: ownsLease((input) => input.leaseId), +}); + +/** + * Builds the scoped command behind `simlock simctl` / `simlock adb`. It lives in the daemon + * rather than the CLI because only the driver that proved a device root knows which flag points + * its tool at that root, and which verbs would change a device's lifecycle behind the registry's + * back -- neither is knowledge a frontend is entitled to (ADR 0003 §11: frontends render the + * contract and nothing else). + * + * `role: "agent"`: the wrappers exist so a lease holder can reach a contained device, which is + * an agent's job. It grants no authority of its own -- the refusal list lives in the driver, and + * what comes back is a command the caller could have written had it known the root path, which + * ADR 0001 is explicit is an accident boundary rather than a security one. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const driverPassthrough = defineOperation({ + name: "driver.passthrough", + role: "agent", + effect: "read", + input: z.object({ args: z.array(z.string()), tool: z.string().min(1) }), + output: passthroughCommandSchema, +}); + +/** why: the File name rule of a lease's files (1 to 128 characters), shared by the upload routes and exec's `stdinFile`. */ +export const LEASE_FILE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +// why: the daemon parses a request as one string, so unbounded stdin exhausts the heap (#1242). +export const EXEC_STDIN_MAX_LENGTH = 8_388_608; + +/** + * Runs `simctl`, `adb` or `devicectl` **on the machine that owns the device** and streams the output back + * (ADR 0005 §19a). The difference from `driver.passthrough` is where the process runs, not + * what it is: the same driver resolves the same scoped command with the same refusals -- + * the daemon then spawns it itself instead of handing the caller a command line it could only + * run if it happened to be on that machine. That is what makes a leased device reachable from + * a remote HTTP agent today, and through a gateway later, without a second network path. + * + * `role: "agent"`, gated on the lease's requester: driving a device is an agent's job, but -- + * unlike `driver.passthrough`, which only builds a string -- this one actually runs a command + * against a machine, so it is gated on the caller being who the lease was granted to (see the + * `authorize` hook below). `leaseId` is that proof: for `simctl` and `adb` the command's own + * arguments may leave the device out and the daemon parses none of them: the driver puts the + * lease's device where the tool expects it (ADR 0047 §1) and refuses a command that names + * another one. + * + * A bare `adb shell` (nothing to run) is refused by the driver on this path, because there is + * no terminal to attach it to and it would otherwise sit until `exec.timeoutMs` -- the same + * `PASSTHROUGH_REFUSED` a lifecycle verb gets. Locally, where the CLI hands the tool its own + * terminal, it stays allowed and interactive. + * + * Every command runs with the lease's folder on the worker as its working folder (ADR 0067 §5), + * so a relative path names a file the holder uploaded. `stdinFile` names such a file to pipe to + * the command's stdin instead of `stdin`; the two together are `BAD_REQUEST`, checked by the + * worker's dispatcher, not here, because `.omit()` cannot build `run_on_device`'s input from an + * object carrying a refinement. + * + * `stdin` is a **one-shot string**, not a stream: it is written to the child's stdin and the + * pipe is then closed. ADR §19c is explicit that there is no pseudo-terminal here, so + * line-oriented commands (`adb shell getprop`, `simctl spawn`) work and full-screen or + * interactive ones (`adb shell` with no argument, a curses UI) do not -- and a client that + * could feed such a program keystrokes over time would only make that failure slower to + * discover. Locally, `simlock adb shell` still spawns with inherited stdio and stays fully + * interactive (ADR §19c). + * + * `tool` is an open string, exactly as `driver.passthrough`'s is, and for the same reason: + * which wrappers exist is the drivers' answer, not the contract's. A name no driver on that + * machine wraps comes back as `UNKNOWN_PASSTHROUGH_TOOL` -- the code that already means + * precisely that, at the 422/exit-2 it already carries -- rather than as a `BAD_REQUEST`, + * which would say the request was malformed when it was merely aimed at a tool this host does + * not have. `BAD_REQUEST` on this operation means the body's shape is wrong, nothing more. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const deviceExec = defineOperation({ + name: "device.exec", + role: "agent", + effect: "write", + input: z + .object({ + leaseId: z.string(), + tool: z.string().min(1), + args: z.array(z.string()), + stdin: z.string().max(EXEC_STDIN_MAX_LENGTH).optional(), + stdinFile: z.string().regex(LEASE_FILE_NAME_PATTERN).optional(), + /** + * why: Who this command is being run *for* -- the requester the lease is attributed to (ADR + * 0003 §4), not the connection's principal. Read **only on an admin session**, where it + * is the gateway case: a gateway holds one admin session over its uplink and proxies + * many agents through it, each under its own namespaced requester id (ADR 0005 + * §19b/§27). A non-admin session that supplies one at all is refused by `authorize` + * below rather than having it quietly ignored -- ADR 0003 §2 puts a contract + * operation's answer in the dispatcher, not a transport, and a request that names an + * identity and is answered as if it had not is the kind of silence that reads like + * authorization (round 4, F4). + */ + requesterId: requesterIdSchema.optional(), + }) + .strict(), + /** + * why: Only the exit code: every byte the command wrote already left as `output` pushes while it + * ran (see `pushes.ts`), so an answer that also carried the output would mean the daemon had + * buffered all of it -- which ADR §19e forbids, and which is why there is no size cap here + * to speak of either. + */ + output: z.object({ exitCode: z.number() }), + /** + * why: `ownsLease` for an agent, and something stricter for an admin. + * + * An **agent** session is gated exactly like `lease.renew`: its principal against the + * lease's `ownerId`. It may not supply `requesterId` at all -- round 4, F4: this used to be + * read and then silently ignored, which left the rule "an agent may not name a + * `requesterId`" living only in the HTTP route (`src/http/app.ts`) rather than here, so the + * unix socket (and the future uplink, and any `simlock/client` consumer) answered a request + * naming an identity as if it had named none. A non-admin session naming one at all is now + * `FORBIDDEN` on every transport, uniformly, whether or not the name it chose happens to + * match anything real. + * + * An **admin** session does not get `ownsLease`'s bypass on this operation. Every other + * admin-bypasses hook is about an operator reaching past ownership on their own machine; + * this one is the check a gateway's uplink session runs against, and a bypass there would + * mean any admin-role connection could drive every lease on the worker -- the exact hole + * §19b's "ownership is checked at the gateway, and again at the worker" exists to close. + * So an admin is held to the lease's `requesterId` instead: it names the agent it is + * proxying for (defaulting to its own principal), and a mismatch is `FORBIDDEN`. An + * operator who means to reach another agent's device names that agent explicitly, which is + * a deliberate act rather than an implicit privilege. + * + * An unknown lease id resolves to `undefined` on either lookup and is treated as + * authorized, the same convention `ownsLease` uses, so the handler's own `UNKNOWN_LEASE` + * surfaces instead of a misleading `FORBIDDEN`. + */ + authorize: authorizeOneLeaseCall, +}); + +/** + * ADR 0005 §20: the element carries an optional `workerId` -- absent from a worker's own + * answer, set by a gateway, which lists the leases its worker views report. Additive, so a + * client that never talks to a gateway sees exactly what it saw before. + */ +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseList = defineOperation({ + name: "lease.list", + role: "agent", + effect: "read", + input: z.object({}), + output: z.object({ leases: z.array(statusLeaseSchema) }), +}); + +// why: no `authorize` hook -- the handler answers `UNKNOWN_LEASE` for a lease the caller does not own, as `lease.list` hides it (ADR 0044 §3), so "not yours" and "not there" look the same. +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseGet = defineOperation({ + name: "lease.get", + role: "agent", + effect: "read", + input: z.object({ leaseId: z.string() }), + output: z.object({ lease: statusLeaseSchema }), +}); + +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const leaseReleaseAll = defineOperation({ + name: "lease.release-all", + role: "admin", + effect: "write", + input: z.object({}), + output: z.object({ leaseIds: z.array(z.string()) }), +}); + +/** + * why: turns a leased iOS phone or tablet simulator, through the lease's device input, whether or not the input module is on. + */ +export const deviceRotate = defineOperation({ + name: "device.rotate", + role: "agent", + effect: "write", + input: z + .object({ + leaseId: z.string().min(1).max(256), + orientation: z.enum(INPUT_ORIENTATIONS), + // why: rotate always refused an empty requesterId, so the shared format also keeps min(1). + requesterId: requesterIdSchema.pipe(z.string().min(1)).optional(), + }) + .strict(), + output: z.object({ leaseId: z.string().max(256), orientation: z.enum(INPUT_ORIENTATIONS) }), + authorize: authorizeOneLeaseCall, +}); diff --git a/src/contract/operations.ts b/src/contract/operations.ts index 804171999..3059806e9 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -1,88 +1,135 @@ import { z } from "zod"; - import { CONSOLE_URL_MAX_LENGTH } from "./console-url.js"; +import { defineOperation } from "./operation-definition.js"; +import { + baselineList, + baselineRemove, + baselineSave, + cleanupRun, + configGet, + daemonStop, + daemonUpdate, + doctorRun, + eventsReplay, + eventsSubscribe, + eventsUnsubscribe, + listGet, + nukeRun, + tokenCreate, + tokenList, + tokenRevoke, + usageGet, + warmForget, +} from "./operations-admin.js"; import { - EVENTS_PAGE_MAX, - EVENTS_WORKER_MAX, - EVENTS_PAGE_WITH_SINCE_MESSAGE, - eventsPageWithoutSince, -} from "./events-page.js"; -import { INPUT_ORIENTATIONS } from "./orientation.js"; -import { parseOsConstraint } from "./os-range.js"; -import { relayChannelSchema, relayHeadersSchema, relayIdSchema } from "./relay.js"; -import { ownsLease, type AuthorizeContext, type Role } from "./roles.js"; + alertsTest, + componentInstall, + componentList, + componentRemove, + deviceAdd, + deviceRemove, + identityGet, + inboxGet, + relayCancel, + relayRequest, + workerDrain, + workerInstallComponent, + workerList, + workerRemove, + workerUndrain, + workerUpdate, +} from "./operations-fleet.js"; +import { leaseRequest } from "./operations-lease-request.js"; import { - capabilityNameSchema, - cleanupRuleSummarySchema, - configSchema, + deviceExec, + deviceRotate, + driverPassthrough, + leaseCancel, + leaseGet, + leaseList, + leaseRelease, + leaseReleaseAll, + leaseRenew, +} from "./operations-leases.js"; +import { + capabilityListSchema, daemonHealthSchema, daemonModeSchema, daemonUpdateFields, - type DeviceClass, - deviceClassSchema, - updateRequestedVersionSchema, - deviceRecordSchema, - doctorReportSchema, - eventEnvelopeSchema, - EVENT_ID_PATTERN, - EXCEPT_MAX_NAMES, - EXCEPT_NAME_PATTERN, - hostFactsSchema, - requesterIdSchema, historyStatusSchema, + hostFactsSchema, hostLoadSchema, - baselineEntrySchema, - baselineNameSchema, - baselineNoteSchema, - imageTagSchema, - leaseGrantSchema, - leaseRecordSchema, - nukeReportSchema, - passthroughCommandSchema, platformCatalogSchema, platformSchema, - proposalSchema, statusCapacitySchema, - statusWarmPoolSchema, statusDeviceSchema, statusInstallsSchema, statusLeaseSchema, - tokenRecordSchema, - tokenRoleSchema, - usageOutputSchema, - usageWindowSchema, - inboxItemSchema, + statusWarmPoolSchema, waitingRequestSchema, - capabilityListSchema, - WARM_POOL_LEARNED_KINDS_MAX, - WARM_POOL_NAME_MAX, workerViewSchema, } from "./schemas.js"; -export type Effect = "read" | "write"; - -export interface OperationDefinition< - Name extends string = string, - InputSchema extends z.ZodType = z.ZodType, - OutputSchema extends z.ZodType = z.ZodType, -> { - readonly name: Name; - readonly role: Role | ((input: z.infer) => Role); - readonly effect: Effect | ((input: z.infer) => Effect); - readonly input: InputSchema; - readonly output: OutputSchema; - readonly authorize?: (input: z.infer, context: AuthorizeContext) => boolean; -} - -function defineOperation< - Name extends string, - InputSchema extends z.ZodType, - OutputSchema extends z.ZodType, ->( - definition: OperationDefinition, -): OperationDefinition { - return definition; -} +export type { Effect, OperationDefinition } from "./operation-definition.js"; +export { + LEASE_ID_PATTERN, + leaseIdSchema, + refuseModelWithClass, + leaseRequestFields, + leaseRequest, + requestedDevice, + requestedClass, +} from "./operations-lease-request.js"; +export { + leaseCancel, + leaseRenew, + leaseRelease, + driverPassthrough, + LEASE_FILE_NAME_PATTERN, + EXEC_STDIN_MAX_LENGTH, + deviceExec, + leaseList, + leaseGet, + leaseReleaseAll, +} from "./operations-leases.js"; +export { + doctorRun, + listGet, + cleanupRun, + nukeRun, + warmForget, + baselineSave, + baselineList, + baselineRemove, + configGet, + daemonStop, + daemonUpdate, + eventsReplay, + usageGet, + eventsSubscribe, + eventsUnsubscribe, + tokenCreate, + tokenList, + tokenRevoke, +} from "./operations-admin.js"; +export { + workerList, + workerDrain, + workerUndrain, + workerRemove, + workerUpdate, + componentInstall, + componentList, + WORKER_RESULT_MESSAGE_MAX_LENGTH, + workerInstallComponent, + componentRemove, + deviceAdd, + deviceRemove, + relayRequest, + relayCancel, + alertsTest, + inboxGet, +} from "./operations-fleet.js"; // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const catalogGet = defineOperation({ @@ -125,1158 +172,6 @@ export const statusGet = defineOperation({ }), }); -/** - * The lease ID a requester may choose (ADR 0020 §1): ASCII, 1 to 64 characters, starting with a - * letter or digit, then letters, digits, `-` and `_`, case-sensitive. It has no `.`, so it can - * never look like a gateway's `.`. The one definition: a gateway reads it too, - * to decide which reported lease it names bare. - */ -export const LEASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; - -export const leaseIdSchema = z.string().regex(LEASE_ID_PATTERN); - -/** - * why: a component's version (`26.4`, `35`) reaches the driver unread and becomes an installer - * argument, so its shape is bounded here (safety rule 10): no whitespace or control character, no - * `;` (sdkmanager's package path separator), no leading `-` (an option). An exact `osVersion` in - * `lease.request` is held to it too, because a downloadable missing runtime installs that string. - */ -const componentVersionSchema = z - .string() - .min(1) - .max(64) - .regex(/^[^\s\p{Cc};]+$/u, "must contain no whitespace, control character or ';'") - .refine((version) => !version.startsWith("-"), "must not start with '-'"); - -const EXACT_OS_VERSION_BOUND = - "an exact OS version is 1 to 64 characters, none of them whitespace, a control character or `;`, and does not start with `-`"; - -const leaseRequestBaseSchema = z - .object({ - model: z.string().min(1).max(WARM_POOL_NAME_MAX).optional(), - class: deviceClassSchema.optional(), - platform: platformSchema, - // why: A plain string, checked by `refuseBadOsVersion`: a refined field would be checked by the MCP SDK - // before the handler, and its error would stand in for `BAD_REQUEST`. - osVersion: z.string().optional(), - mode: z.enum(["slim", "full"]).optional(), - // why: A plain array of plain strings, bounded by `refuseBadExcept` as the one object-level check, like `osVersion`: a bound on the field itself would be checked by the MCP SDK before the handler. - except: z.array(z.string()).optional(), - imageTag: imageTagSchema.optional(), - baseline: baselineNameSchema.optional(), - physical: z.boolean().optional(), - requesterId: requesterIdSchema.optional(), - allowDownload: z.boolean().optional(), - noWait: z.boolean().optional(), - timeoutMs: z.number().finite().positive().optional(), - /** why: Capped by the dispatcher's handler, not here: the cap is daemon config this module cannot see. */ - ttlMs: z.number().finite().positive().optional(), - owner: z.string().optional(), - idempotencyKey: z.string().min(1).max(200).optional(), - leaseId: leaseIdSchema.optional(), - fleetRequestId: z.string().min(1).max(200).optional(), - // why: A plain record of plain strings, bounded by `refuseBadMetadata` as one object-level check, like `except`: a bound on the field itself would be checked by the MCP SDK before the handler. - metadata: z.record(z.string(), z.string()).optional(), - requires: z - .array(capabilityNameSchema) - .max(8) - .refine((names) => new Set(names).size === names.length, "a capability name appears twice") - .optional(), - }) - .strict(); - -export function refuseModelWithClass( - modelField = "model", -): ( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.model !== undefined && input.class !== undefined) { - context.addIssue({ - code: "custom", - message: `a request names a model or a class, not both: send only one of \`${modelField}\` and \`class\``, - path: ["class"], - }); - } - }; -} - -function refuseBaselineWithClass(): < - T extends { readonly baseline?: unknown; readonly class?: unknown }, ->( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.baseline !== undefined && input.class !== undefined) { - context.addIssue({ - code: "custom", - message: - "a request names a baseline or a class, not both: send only one of `baseline` and `class`", - path: ["class"], - }); - } - }; -} - -/** why: `allowDownload: false` passes because the CLI always sends it. */ -function refusePhysicalWithImageFields(): < - T extends { - readonly physical?: boolean | undefined; - readonly mode?: unknown; - readonly imageTag?: unknown; - readonly baseline?: unknown; - readonly allowDownload?: boolean | undefined; - }, ->( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.physical !== true) return; - const refused = [ - input.mode === undefined ? undefined : "mode", - input.imageTag === undefined ? undefined : "imageTag", - input.baseline === undefined ? undefined : "baseline", - input.allowDownload === true ? "allowDownload" : undefined, - ]; - for (const field of refused) { - if (field === undefined) continue; - context.addIssue({ - code: "custom", - message: `\`${field}\` has no meaning for a physical device`, - path: [field], - }); - } - }; -} - -// why: a replay would have to be checked against a capability state that changed since. -function refuseRequiresWithIdempotencyKey(): < - T extends { readonly requires?: unknown; readonly idempotencyKey?: string | undefined }, ->( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.requires === undefined || input.idempotencyKey === undefined) return; - context.addIssue({ - code: "custom", - message: "requires cannot be combined with idempotencyKey", - path: ["requires"], - }); - }; -} - -/** why: one object-level check with no path (like `osVersion`), so every transport, MCP included, answers `BAD_REQUEST` with the message as written. */ -function refuseBadExcept(): < - T extends { - readonly except?: readonly string[] | undefined; - readonly mode?: unknown; - readonly physical?: boolean | undefined; - readonly platform: string; - }, ->( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - const { except } = input; - if (except === undefined) return; - const message = exceptRefusal(input, except); - if (message !== undefined) context.addIssue({ code: "custom", message }); - }; -} - -function exceptRefusal( - input: { - readonly mode?: unknown; - readonly physical?: boolean | undefined; - readonly platform: string; - }, - except: readonly string[], -): string | undefined { - if (except.length < 1 || except.length > EXCEPT_MAX_NAMES) { - return `except names 1 to ${EXCEPT_MAX_NAMES} categories`; - } - const bad = except.find((name) => !EXCEPT_NAME_PATTERN.test(name)); - if (bad !== undefined) return `except: "${bad}" is not a category name`; - const twice = except.find((name, index) => except.indexOf(name) !== index); - if (twice !== undefined) return `except names "${twice}" twice`; - if (input.physical === true || input.platform !== "ios") return "except is for iOS simulators"; - if (input.mode === "full") return "except needs a slim lease (mode is full)"; - return undefined; -} - -const METADATA_MAX_KEYS = 16; -const METADATA_KEY_MAX = 64; -const METADATA_VALUE_MAX = 256; -const METADATA_TOTAL_MAX = 2048; - -function metadataRefusal(metadata: Readonly>): string | undefined { - const entries = Object.entries(metadata); - if (entries.length > METADATA_MAX_KEYS) return `metadata has more than ${METADATA_MAX_KEYS} keys`; - const badKey = entries.find(([key]) => key.length < 1 || key.length > METADATA_KEY_MAX); - if (badKey !== undefined) { - return `metadata key must be 1 to ${METADATA_KEY_MAX} characters (got ${badKey[0]})`; - } - const badValue = entries.find(([, value]) => value.length > METADATA_VALUE_MAX); - if (badValue !== undefined) { - return `metadata value for ${badValue[0]} must be at most ${METADATA_VALUE_MAX} characters`; - } - const total = entries.reduce((sum, [key, value]) => sum + key.length + value.length, 0); - if (total > METADATA_TOTAL_MAX) - return `metadata is longer than ${METADATA_TOTAL_MAX} characters in all`; - return undefined; -} - -function refuseBadMetadata(): < - T extends { readonly metadata?: Readonly> | undefined }, ->( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.metadata === undefined) return; - const message = metadataRefusal(input.metadata); - if (message !== undefined) context.addIssue({ code: "custom", message }); - }; -} - -function refuseBadOsVersion(): ( - input: T, - context: z.RefinementCtx, -) => void { - return (input, context) => { - if (input.osVersion === undefined) return; - const parsed = parseOsConstraint(input.osVersion); - if (!parsed.ok) { - context.addIssue({ - code: "custom", - message: parsed.message, - path: ["osVersion"], - }); - return; - } - if ( - parsed.constraint.kind === "exact" && - !componentVersionSchema.safeParse(parsed.constraint.version).success - ) { - context.addIssue({ - code: "custom", - message: EXACT_OS_VERSION_BOUND, - path: ["osVersion"], - }); - } - }; -} - -export const leaseRequestFields = leaseRequestBaseSchema; - -const leaseRequestInputSchema = leaseRequestBaseSchema - .superRefine(refuseModelWithClass()) - .superRefine(refuseBaselineWithClass()) - .superRefine(refuseBadOsVersion()) - .superRefine(refuseBadExcept()) - .superRefine(refuseBadMetadata()) - .superRefine(refusePhysicalWithImageFields()) - .superRefine(refuseRequiresWithIdempotencyKey()); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseRequest = defineOperation({ - name: "lease.request", - role: "agent", - effect: "write", - input: leaseRequestInputSchema, - output: leaseGrantSchema, - // why: No `authorize` hook on purpose: any agent session may request under an arbitrary - // `requesterId` (printable, at most 256 characters), and `owner` is gated by role in the handler (the resource does not exist yet). -}); - -export function requestedDevice( - input: Omit, "except"> & { - readonly except?: readonly string[] | undefined; - }, -) { - return { - ...(input.model === undefined ? {} : { model: input.model }), - ...(input.class === undefined ? {} : { class: input.class }), - platform: input.platform, - ...(input.osVersion === undefined ? {} : { osVersion: input.osVersion }), - ...(input.mode === undefined ? {} : { mode: input.mode }), - ...(input.except === undefined ? {} : { except: [...input.except] }), - ...(input.imageTag === undefined ? {} : { imageTag: input.imageTag }), - ...(input.baseline === undefined ? {} : { baseline: input.baseline }), - ...(input.physical === true ? { physical: true as const } : {}), - }; -} - -export function requestedClass(request: { - readonly model?: undefined; - readonly class?: DeviceClass | undefined; -}): DeviceClass; -export function requestedClass(request: { - readonly model?: string | undefined; - readonly class?: DeviceClass | undefined; -}): DeviceClass | undefined; -export function requestedClass(request: { - readonly model?: string | undefined; - readonly class?: DeviceClass | undefined; -}): DeviceClass | undefined { - if (request.class !== undefined) return request.class; - return request.model === undefined ? "phone" : undefined; -} - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseCancel = defineOperation({ - name: "lease.cancel", - role: "agent", - effect: "write", - input: z.object({ - requesterId: requesterIdSchema.optional(), - platform: platformSchema.optional(), - }), - output: z.object({ result: z.enum(["cancelled", "not-found", "not-cancellable"]) }), - /** - * why: ADR §9: "cancels this principal's pending request by requester id". Gated on the pending - * request's *recorded owner* (`pendingRequestOwner`, ADR §4's `ownerId` -- always the - * creating session's principal, never the caller-suppliable `requesterId`), not on comparing - * `requesterId` to the principal directly -- that comparison would forbid exactly the case - * ADR §4 exists for: one connection (`principal: "host"`) proxying many agents, each under - * its own `requesterId` (`"agent-7"`). Deliberately not `ownsLease` (that hook resolves a - * *lease's* recorded `ownerId` from the registry; a pending, not-yet-granted request has no - * lease yet to look up -- `pendingRequestOwner` is the wait-queue equivalent). An omitted - * `requesterId` defaults to the principal the same way the handler's own default does (see - * `dispatcher.ts`'s `#leaseCancel`), so it always resolves to a request this principal owns. - * A `requesterId` with no pending request resolves `pendingRequestOwner` to `undefined`, - * which is treated as authorized (same convention as `ownsLease`) so the handler's own - * `not-found` surfaces instead of a misleading `FORBIDDEN`. - */ - authorize: (input, context) => { - if (context.role === "admin") return true; - const owner = context.pendingRequestOwner( - input.requesterId ?? context.principal, - input.platform, - ); - return owner === undefined || owner === context.principal; - }, -}); - -/** - * ADR 0004 §1: the only thing that keeps a lease alive, on every transport. An omitted `ttlMs` - * re-applies the lease's own stored width -- never `lease.defaultTtlMs` -- so a lease granted - * for four hours does not shrink to fifteen minutes the first time something renews it. A - * `ttlMs` above `lease.maxTtlMs` is `BAD_REQUEST`, checked in the dispatcher's handler where - * that config value is visible. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseRenew = defineOperation({ - name: "lease.renew", - role: "agent", - effect: "write", - input: z.object({ leaseId: z.string(), ttlMs: z.number().finite().positive().optional() }), - output: leaseRecordSchema, - authorize: ownsLease((input) => input.leaseId), -}); - -export const leaseRelease = defineOperation({ - name: "lease.release", - role: "agent", - effect: "write", - input: z.object({ leaseId: z.string() }), - output: z.object({ leaseId: z.string() }), - authorize: ownsLease((input) => input.leaseId), -}); - -/** - * Builds the scoped command behind `simlock simctl` / `simlock adb`. It lives in the daemon - * rather than the CLI because only the driver that proved a device root knows which flag points - * its tool at that root, and which verbs would change a device's lifecycle behind the registry's - * back -- neither is knowledge a frontend is entitled to (ADR 0003 §11: frontends render the - * contract and nothing else). - * - * `role: "agent"`: the wrappers exist so a lease holder can reach a contained device, which is - * an agent's job. It grants no authority of its own -- the refusal list lives in the driver, and - * what comes back is a command the caller could have written had it known the root path, which - * ADR 0001 is explicit is an accident boundary rather than a security one. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const driverPassthrough = defineOperation({ - name: "driver.passthrough", - role: "agent", - effect: "read", - input: z.object({ args: z.array(z.string()), tool: z.string().min(1) }), - output: passthroughCommandSchema, -}); - -/** why: the File name rule of a lease's files (1 to 128 characters), shared by the upload routes and exec's `stdinFile`. */ -export const LEASE_FILE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; - -// why: the daemon parses a request as one string, so unbounded stdin exhausts the heap (#1242). -export const EXEC_STDIN_MAX_LENGTH = 8_388_608; - -/** - * Runs `simctl`, `adb` or `devicectl` **on the machine that owns the device** and streams the output back - * (ADR 0005 §19a). The difference from `driver.passthrough` is where the process runs, not - * what it is: the same driver resolves the same scoped command with the same refusals -- - * the daemon then spawns it itself instead of handing the caller a command line it could only - * run if it happened to be on that machine. That is what makes a leased device reachable from - * a remote HTTP agent today, and through a gateway later, without a second network path. - * - * `role: "agent"`, gated on the lease's requester: driving a device is an agent's job, but -- - * unlike `driver.passthrough`, which only builds a string -- this one actually runs a command - * against a machine, so it is gated on the caller being who the lease was granted to (see the - * `authorize` hook below). `leaseId` is that proof: for `simctl` and `adb` the command's own - * arguments may leave the device out and the daemon parses none of them: the driver puts the - * lease's device where the tool expects it (ADR 0047 §1) and refuses a command that names - * another one. - * - * A bare `adb shell` (nothing to run) is refused by the driver on this path, because there is - * no terminal to attach it to and it would otherwise sit until `exec.timeoutMs` -- the same - * `PASSTHROUGH_REFUSED` a lifecycle verb gets. Locally, where the CLI hands the tool its own - * terminal, it stays allowed and interactive. - * - * Every command runs with the lease's folder on the worker as its working folder (ADR 0067 §5), - * so a relative path names a file the holder uploaded. `stdinFile` names such a file to pipe to - * the command's stdin instead of `stdin`; the two together are `BAD_REQUEST`, checked by the - * worker's dispatcher, not here, because `.omit()` cannot build `run_on_device`'s input from an - * object carrying a refinement. - * - * `stdin` is a **one-shot string**, not a stream: it is written to the child's stdin and the - * pipe is then closed. ADR §19c is explicit that there is no pseudo-terminal here, so - * line-oriented commands (`adb shell getprop`, `simctl spawn`) work and full-screen or - * interactive ones (`adb shell` with no argument, a curses UI) do not -- and a client that - * could feed such a program keystrokes over time would only make that failure slower to - * discover. Locally, `simlock adb shell` still spawns with inherited stdio and stays fully - * interactive (ADR §19c). - * - * `tool` is an open string, exactly as `driver.passthrough`'s is, and for the same reason: - * which wrappers exist is the drivers' answer, not the contract's. A name no driver on that - * machine wraps comes back as `UNKNOWN_PASSTHROUGH_TOOL` -- the code that already means - * precisely that, at the 422/exit-2 it already carries -- rather than as a `BAD_REQUEST`, - * which would say the request was malformed when it was merely aimed at a tool this host does - * not have. `BAD_REQUEST` on this operation means the body's shape is wrong, nothing more. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const deviceExec = defineOperation({ - name: "device.exec", - role: "agent", - effect: "write", - input: z - .object({ - leaseId: z.string(), - tool: z.string().min(1), - args: z.array(z.string()), - stdin: z.string().max(EXEC_STDIN_MAX_LENGTH).optional(), - stdinFile: z.string().regex(LEASE_FILE_NAME_PATTERN).optional(), - /** - * why: Who this command is being run *for* -- the requester the lease is attributed to (ADR - * 0003 §4), not the connection's principal. Read **only on an admin session**, where it - * is the gateway case: a gateway holds one admin session over its uplink and proxies - * many agents through it, each under its own namespaced requester id (ADR 0005 - * §19b/§27). A non-admin session that supplies one at all is refused by `authorize` - * below rather than having it quietly ignored -- ADR 0003 §2 puts a contract - * operation's answer in the dispatcher, not a transport, and a request that names an - * identity and is answered as if it had not is the kind of silence that reads like - * authorization (round 4, F4). - */ - requesterId: requesterIdSchema.optional(), - }) - .strict(), - /** - * why: Only the exit code: every byte the command wrote already left as `output` pushes while it - * ran (see `pushes.ts`), so an answer that also carried the output would mean the daemon had - * buffered all of it -- which ADR §19e forbids, and which is why there is no size cap here - * to speak of either. - */ - output: z.object({ exitCode: z.number() }), - /** - * why: `ownsLease` for an agent, and something stricter for an admin. - * - * An **agent** session is gated exactly like `lease.renew`: its principal against the - * lease's `ownerId`. It may not supply `requesterId` at all -- round 4, F4: this used to be - * read and then silently ignored, which left the rule "an agent may not name a - * `requesterId`" living only in the HTTP route (`src/http/app.ts`) rather than here, so the - * unix socket (and the future uplink, and any `simlock/client` consumer) answered a request - * naming an identity as if it had named none. A non-admin session naming one at all is now - * `FORBIDDEN` on every transport, uniformly, whether or not the name it chose happens to - * match anything real. - * - * An **admin** session does not get `ownsLease`'s bypass on this operation. Every other - * admin-bypasses hook is about an operator reaching past ownership on their own machine; - * this one is the check a gateway's uplink session runs against, and a bypass there would - * mean any admin-role connection could drive every lease on the worker -- the exact hole - * §19b's "ownership is checked at the gateway, and again at the worker" exists to close. - * So an admin is held to the lease's `requesterId` instead: it names the agent it is - * proxying for (defaulting to its own principal), and a mismatch is `FORBIDDEN`. An - * operator who means to reach another agent's device names that agent explicitly, which is - * a deliberate act rather than an implicit privilege. - * - * An unknown lease id resolves to `undefined` on either lookup and is treated as - * authorized, the same convention `ownsLease` uses, so the handler's own `UNKNOWN_LEASE` - * surfaces instead of a misleading `FORBIDDEN`. - */ - authorize: authorizeOneLeaseCall, -}); - -/** - * ADR 0005 §20: the element carries an optional `workerId` -- absent from a worker's own - * answer, set by a gateway, which lists the leases its worker views report. Additive, so a - * client that never talks to a gateway sees exactly what it saw before. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseList = defineOperation({ - name: "lease.list", - role: "agent", - effect: "read", - input: z.object({}), - output: z.object({ leases: z.array(statusLeaseSchema) }), -}); - -// why: no `authorize` hook -- the handler answers `UNKNOWN_LEASE` for a lease the caller does not own, as `lease.list` hides it (ADR 0044 §3), so "not yours" and "not there" look the same. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseGet = defineOperation({ - name: "lease.get", - role: "agent", - effect: "read", - input: z.object({ leaseId: z.string() }), - output: z.object({ lease: statusLeaseSchema }), -}); - -const doctorRunInputSchema = z.object({ - fix: z.boolean().optional(), - /** - * why: Destroys every orphan the run finds (ADR 0001, decision 5) -- the single opt-in exception - * to registry-only destruction (safety rule 1). Its own flag rather than part of `fix`, so a - * `doctor --fix` already running unattended in CI does not acquire a destructive behaviour on - * upgrade. - */ - purgeOrphans: z.boolean().optional(), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const doctorRun = defineOperation({ - name: "doctor.run", - /** - * why: The role genuinely depends on the input: `fix: false` (read-only, shells out per device) - * is agent-visible; `fix: true` (applies safe fixes to registry-owned devices) is admin - * only -- see ADR §3's matrix, which lists `doctor.run` twice for exactly this reason. A - * function-of-input role, rather than an `authorize` hook, is the right tool here: `authorize` - * (ADR §1's `ownsLease` example) is for *ownership* checks against a specific resource once a - * role has already cleared the gate; this is a *role* decision, and it is entirely computable - * from the already-validated input with no session/resource context needed. Keeping it as - * `role` also means the dispatcher's role-check step (PR 2) stays uniform -- "compute the - * required role from the input, compare to the session role" -- for every operation, instead - * of `doctor.run` needing a second, bespoke gating path. - */ - role: (input: z.infer): Role => - input.fix || input.purgeOrphans ? "admin" : "agent", - effect: (input: z.infer): Effect => - input.fix || input.purgeOrphans ? "write" : "read", - input: doctorRunInputSchema, - output: doctorReportSchema, -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const leaseReleaseAll = defineOperation({ - name: "lease.release-all", - role: "admin", - effect: "write", - input: z.object({}), - output: z.object({ leaseIds: z.array(z.string()) }), -}); - -/** Arm order is load-bearing: a full record matches the first arm, so a worker's answer is unchanged. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const listGet = defineOperation({ - name: "list.get", - role: "admin", - effect: "read", - input: z.object({ kind: z.enum(["devices", "leases", "rules", "requests"]).optional() }), - output: z.union([ - z.array(deviceRecordSchema), - z.array(statusDeviceSchema), - z.array(statusLeaseSchema), - z.array(cleanupRuleSummarySchema), - z.array(waitingRequestSchema), - ]), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const cleanupRun = defineOperation({ - name: "cleanup.run", - role: "admin", - effect: (input: { dryRun?: boolean | undefined }): Effect => (input.dryRun ? "read" : "write"), - input: z.object({ dryRun: z.boolean().optional(), rule: z.string().optional() }), - output: z.array(proposalSchema), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const nukeRun = defineOperation({ - name: "nuke.run", - role: "admin", - effect: "write", - input: z.object({ deleteDevices: z.boolean().optional() }), - output: nukeReportSchema, -}); - -const warmKindSchema = z.object({ - platform: platformSchema, - model: z.string().max(WARM_POOL_NAME_MAX), - osVersion: z.string().max(WARM_POOL_NAME_MAX), - mode: z.enum(["slim", "full"]), - imageTag: imageTagSchema.optional(), - baseline: baselineNameSchema.optional(), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const warmForget = defineOperation({ - name: "warm.forget", - role: "admin", - effect: "write", - input: z.object({ - platform: platformSchema, - model: z.string().min(1).max(WARM_POOL_NAME_MAX), - osVersion: z.string().min(1).max(WARM_POOL_NAME_MAX).optional(), - mode: z.enum(["slim", "full"]).optional(), - imageTag: imageTagSchema.optional(), - }), - output: z.object({ forgotten: z.array(warmKindSchema).max(WARM_POOL_LEARNED_KINDS_MAX) }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const baselineSave = defineOperation({ - name: "baseline.save", - role: "admin", - effect: "write", - input: z - .object({ - leaseId: z.string().min(1).max(256), - name: baselineNameSchema, - note: baselineNoteSchema.optional(), - }) - .strict(), - output: z.object({ baseline: baselineEntrySchema, leaseId: z.string().max(256) }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const baselineList = defineOperation({ - name: "baseline.list", - role: "agent", - effect: "read", - input: z.object({}).strict(), - output: z.object({ baselines: z.array(baselineEntrySchema).max(1000) }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const baselineRemove = defineOperation({ - name: "baseline.remove", - role: "admin", - effect: "write", - // why: `workerId` names the worker a gateway forwards the remove to; a gateway requires it and strips it, a worker refuses it. - input: z - .object({ - name: baselineNameSchema, - workerId: z.string().min(1).max(EVENTS_WORKER_MAX).optional(), - }) - .strict(), - output: z.object({ - name: baselineNameSchema, - versions: z.array(z.number().int().min(1)).max(1000), - deletedDevices: z.array(z.string().max(256)).max(1000), - }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const configGet = defineOperation({ - name: "config.get", - role: "admin", - effect: "read", - input: z.object({ showSecrets: z.boolean().optional() }), - output: configSchema, -}); - -// fallow-ignore-next-line unused-export -- see the comment above. -export const daemonStop = defineOperation({ - name: "daemon.stop", - role: "admin", - effect: "write", - input: z.object({}), - output: z.object({ stopping: z.literal(true) }), -}); - -// why: `DaemonServer` dispatches it on a connection whose protocol range does not overlap (like `daemon.stop`), so its input and answer never change shape. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const daemonUpdate = defineOperation({ - name: "daemon.update", - role: "admin", - effect: "write", - input: z - .object({ - version: updateRequestedVersionSchema.optional(), - force: z.boolean().optional(), - query: z.boolean().optional(), - }) - .refine( - (input) => input.query !== true || (input.version === undefined && input.force === undefined), - { - message: "query takes no other field", - }, - ), - output: z.object({ - started: z.boolean(), - upToDate: z.boolean(), - resolved: updateRequestedVersionSchema.nullable(), - version: daemonUpdateFields.version, - health: daemonHealthSchema, - update: daemonUpdateFields.update, - lastUpdateFailure: daemonUpdateFields.lastUpdateFailure, - latestVersion: daemonUpdateFields.latestVersion, - }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const eventsReplay = defineOperation({ - name: "events.replay", - role: "admin", - effect: "read", - input: z - .object({ - beforeId: z.string().regex(EVENT_ID_PATTERN).optional(), - limit: z.number().int().min(1).max(EVENTS_PAGE_MAX).optional(), - sinceTs: z.number().optional(), - workerId: z.string().min(1).max(EVENTS_WORKER_MAX).optional(), - }) - .refine(eventsPageWithoutSince, { message: EVENTS_PAGE_WITH_SINCE_MESSAGE }), - output: z.array(eventEnvelopeSchema), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const usageGet = defineOperation({ - name: "usage.get", - role: "admin", - effect: "read", - input: usageWindowSchema, - output: usageOutputSchema, -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const eventsSubscribe = defineOperation({ - name: "events.subscribe", - role: "admin", - effect: "read", - input: z.object({}), - output: z.object({ subscribed: z.literal(true), subscriptionId: z.string() }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const eventsUnsubscribe = defineOperation({ - name: "events.unsubscribe", - role: "admin", - effect: "read", - input: z.object({}), - output: z.object({ subscribed: z.literal(false) }), -}); - -// why: ADR §11 makes the daemon the only owner of the tokens, and that move has landed: both -// dispatchers handle these three, the CLI reaches them as admin round-trips from -// `src/cli/commands/token.ts`, and no CLI code reads the token store itself. The declarations live -// here because the `OPERATIONS` map every dispatcher validates against is built from this file. - -// fallow-ignore-next-line unused-export -- reachable from outside this file only through OPERATIONS. -export const tokenCreate = defineOperation({ - name: "token.create", - role: "admin", - effect: "write", - input: z.object({ role: tokenRoleSchema, label: z.string().optional() }), - output: z.object({ secret: z.string(), token: tokenRecordSchema }), -}); - -// fallow-ignore-next-line unused-export -- same as tokenCreate above. -export const tokenList = defineOperation({ - name: "token.list", - role: "admin", - effect: "read", - input: z.object({}), - output: z.object({ tokens: z.array(tokenRecordSchema) }), -}); - -// fallow-ignore-next-line unused-export -- same as tokenCreate above. -export const tokenRevoke = defineOperation({ - name: "token.revoke", - role: "admin", - effect: "write", - input: z.object({ id: z.string() }), - output: z.object({ revoked: z.boolean() }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerList = defineOperation({ - name: "worker.list", - role: "admin", - effect: "read", - input: z.object({}), - output: z.object({ workers: z.array(workerViewSchema) }), -}); - -/** - * ADR 0005 §9: a drained worker keeps its existing leases and receives no new dispatches -- - * the operator's tool for taking a machine down without killing anyone's device. This PR - * records the flag (persisted, so it survives a worker reconnect and a gateway restart) and - * reports it; #118's dispatcher is what honours it, because this PR has no dispatch to skip. - * - * Idempotent: draining an already-drained worker succeeds and changes nothing. An id the - * gateway has no view of is `UNKNOWN_WORKER` -- draining a machine that is not in the fleet is - * a typo, and answering "done" would leave an operator believing a machine was taken out of - * service when nothing was. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerDrain = defineOperation({ - name: "worker.drain", - role: "admin", - effect: "write", - input: z.object({ workerId: z.string().min(1) }), - output: z.object({ workerId: z.string(), drained: z.literal(true) }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerUndrain = defineOperation({ - name: "worker.undrain", - role: "admin", - effect: "write", - input: z.object({ workerId: z.string().min(1) }), - output: z.object({ workerId: z.string(), drained: z.literal(false) }), -}); - -/** - * ADR 0005 §8: forgets a *disconnected* worker's view. A connected one is refused with - * `WORKER_CONNECTED` -- removing it would only make the gateway forget a machine that is about - * to announce itself again on its very next refresh, which is a confusing no-op rather than an - * operator action. - * - * `removed: false` is the answer for an id with no view: unlike drain, "forget this" is - * already true of something the gateway has never heard of or has already forgotten, so it is - * an outcome to report rather than an error to raise (the same shape `token.revoke` uses). - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerRemove = defineOperation({ - name: "worker.remove", - role: "admin", - effect: "write", - input: z.object({ workerId: z.string().min(1) }), - output: z.object({ workerId: z.string(), removed: z.boolean() }), -}); - -// why: ADR 0042 §2: the gateway sends `daemon.update` over that worker's uplink and answers with the worker's own answer plus the id. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerUpdate = defineOperation({ - name: "worker.update", - role: "admin", - effect: "write", - input: z.object({ - workerId: z.string().min(1), - version: updateRequestedVersionSchema.optional(), - force: z.boolean().optional(), - }), - output: daemonUpdate.output.extend({ workerId: z.string() }), -}); - -/** - * Installs one iOS simulator runtime or Android system image without leasing a device (ADR 0010 - * §6). Admin-only: the command is an operator's consent to a multi-GB download, which - * `downloads.policy: "never"` still refuses (§4). Not durable: a daemon stop ends the install, - * and running it again is the recovery. `component` echoes the version asked for; `version` is - * the exact one installed or found. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const componentInstall = defineOperation({ - name: "component.install", - role: "admin", - effect: "write", - input: z.object({ platform: platformSchema, version: componentVersionSchema }), - output: z.object({ - platform: platformSchema, - component: z.string(), - outcome: z.enum(["installed", "already-installed"]), - version: z.string(), - }), -}); - -/** - * Every installed iOS simulator runtime and Android system image on this machine, ordered by - * platform, then version, then variant. A read for any session. `installedBySimlock` is true only - * when Simlock's record of an install names this exact component; `installedAt` is that - * record's time. `devices` counts Simlock's own devices of this platform and version in any - * state but deleted, so two variants of one version show the same count; `foreignDevices` - * counts the devices outside Simlock that use it. `variant` tells two components of one - * version apart, in the platform's own words; `sizeBytes` is absent when it could not be read. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const componentList = defineOperation({ - name: "component.list", - role: "agent", - effect: "read", - input: z.object({ platform: platformSchema.optional() }), - output: z.object({ - components: z.array( - z.object({ - platform: platformSchema, - version: z.string(), - variant: z.string().optional(), - sizeBytes: z.number().int().nonnegative().optional(), - installedBySimlock: z.boolean(), - installedAt: z.number().int().optional(), - devices: z.number().int().nonnegative(), - foreignDevices: z.number().int().nonnegative(), - }), - ), - }), -}); - -const WORKER_ID_MAX_LENGTH = 128; - -/** The longest error message a relayed result carries. A worker's message is a claim, cut to - * this before it is stored in a result (safety rule 10). */ -export const WORKER_RESULT_MESSAGE_MAX_LENGTH = 1024; - -const workerInstallTargetsSchema = z.union([ - z.literal("all"), - z - .array(z.string().min(1).max(WORKER_ID_MAX_LENGTH)) - .min(1) - .max(64) - .refine((ids) => new Set(ids).size === ids.length, "worker ids must be distinct"), -]); - -const workerInstallResultSchema = z.object({ - workerId: z.string(), - label: z.string().optional(), - outcome: z.enum(["installed", "already-installed", "refused", "failed", "skipped", "unknown"]), - version: z.string().max(64).optional(), - error: z - .object({ - code: z.string().max(64), - message: z.string().max(WORKER_RESULT_MESSAGE_MAX_LENGTH), - }) - .optional(), -}); - -/** - * ADR 0010 §7: installs one component on named workers, or on every connected one, through a - * gateway. The gateway asks each target at the same time with `component.install`, and each - * worker answers for itself under its own `downloads.policy`. One result per worker, in - * ascending worker id. Progress arrives as `component-progress` pushes carrying `workerId`. - * A worker answers `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012 §2). - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const workerInstallComponent = defineOperation({ - name: "worker.install-component", - role: "admin", - effect: "write", - input: z.object({ - platform: platformSchema, - version: componentVersionSchema, - workers: workerInstallTargetsSchema, - }), - output: z.object({ results: z.array(workerInstallResultSchema) }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const componentRemove = defineOperation({ - name: "component.remove", - role: "admin", - effect: "write", - input: z.object({ - platform: platformSchema, - version: componentVersionSchema, - dryRun: z.boolean().optional(), - }), - output: z.object({ - platform: platformSchema, - version: z.string(), - outcome: z.enum(["removed", "would-remove"]), - sizeBytes: z.number().int().nonnegative().optional(), - residue: z.string().optional(), - }), -}); - -const ENROLLED_DEVICE_ID_MAX_LENGTH = 200; - -const enrollmentDeviceIdSchema = z - .string() - .min(1) - .max(ENROLLED_DEVICE_ID_MAX_LENGTH) - .refine( - (id) => [...id].every((character) => !/\s/u.test(character) && !isControlCharacter(character)), - "must not contain whitespace or control characters", - ) - .refine((id) => !id.startsWith("-"), "must not start with -"); - -function isControlCharacter(character: string): boolean { - const code = character.codePointAt(0) ?? 0; - return code <= 0x1f || (code >= 0x7f && code <= 0x9f); -} - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const deviceAdd = defineOperation({ - name: "device.add", - role: "admin", - effect: "write", - input: z.object({ platform: platformSchema, id: enrollmentDeviceIdSchema }).strict(), - output: z.object({ - id: z.string(), - driverDeviceId: z.string(), - platform: platformSchema, - model: z.string(), - osVersion: z.string(), - class: z.enum(["phone", "tablet"]), - appCount: z.number().int().nonnegative(), - reenrolled: z.boolean(), - }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const deviceRemove = defineOperation({ - name: "device.remove", - role: "admin", - effect: "write", - input: z.object({ id: enrollmentDeviceIdSchema }).strict(), - output: z.object({ id: z.string(), driverDeviceId: z.string() }), -}); - -const RELAY_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"] as const; - -const relayUrlSchema = z - .string() - .max(4096) - .refine((value) => { - try { - const { protocol } = new URL(value); - return protocol === "http:" || protocol === "https:"; - } catch { - return false; - } - }); - -/** - * One HTTP exchange a gateway hands to a module on a worker, over the gateway's uplink. The - * request body arrives, and the response body leaves, on the byte channel `channel` names (ADR 0028); - * the head leaves as a `relay-head` push keyed by this call's frame id. The - * answer comes once the response body has ended, or `{ cancelled: true }` once a `relay.cancel` took - * effect. With `twoWay` (GET, no body) both directions of the channel stay open: the answer comes once - * both have ended. Admin only, and on a worker only the gateway's own uplink session may call it. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const relayRequest = defineOperation({ - name: "relay.request", - role: "admin", - effect: "write", - input: z - .object({ - relayId: relayIdSchema, - target: capabilityNameSchema, - method: z.enum(RELAY_METHODS), - url: relayUrlSchema, - headers: relayHeadersSchema, - tenant: z.string().min(1).max(300), - leaseId: leaseIdSchema.optional(), - hasBody: z.boolean(), - twoWay: z.literal(true).optional(), - channel: relayChannelSchema, - }) - .strict() - .refine((input) => !input.hasBody || (input.method !== "GET" && input.method !== "HEAD"), { - message: "hasBody is not allowed with GET or HEAD", - }) - .refine((input) => input.twoWay !== true || (input.method === "GET" && !input.hasBody), { - message: "twoWay needs GET and no body", - }), - output: z.object({ cancelled: z.literal(true).optional() }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const relayCancel = defineOperation({ - name: "relay.cancel", - role: "admin", - effect: "write", - input: z.object({ relayId: relayIdSchema }).strict(), - output: z.object({}), -}); - -/** - * why: ADR 0035 §1: asks the gateway to post one sample message to one of its alert destinations, - * once, outside the destination's queue. `destination` is only looked up by name among the - * configured ones, never used in a URL. - */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const alertsTest = defineOperation({ - name: "alerts.test", - role: "admin", - effect: "write", - input: z.object({ destination: z.string().min(1).max(32) }), - output: z.object({ destination: z.string(), status: z.number() }), -}); - -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. -export const inboxGet = defineOperation({ - name: "inbox.get", - role: "admin", - effect: "read", - input: z.object({}), - output: z.object({ items: z.array(inboxItemSchema) }), -}); - -const identityGet = defineOperation({ - name: "identity.get", - role: "agent", - effect: "read", - input: z.object({}), - output: z.object({ - principal: z.string(), - name: z.string(), - role: z.enum(["admin", "agent"]), - via: z.enum(["token", "proxy", "socket", "uplink"]), - signOutUrl: z.string().optional(), - }), -}); - -/** - * why: `device.exec`'s and `device.rotate`'s one rule, so a gateway's admin session is held to the lease's requester on both. - */ -function authorizeOneLeaseCall( - input: { readonly leaseId: string; readonly requesterId?: string | undefined }, - context: AuthorizeContext, -): boolean { - if (context.role !== "admin") { - if (input.requesterId !== undefined) return false; - const ownerId = context.ownerId(input.leaseId); - return ownerId === undefined || ownerId === context.principal; - } - const requesterId = input.requesterId ?? context.principal; - const leaseRequesterId = context.leaseRequesterId(input.leaseId); - return leaseRequesterId === undefined || leaseRequesterId === requesterId; -} - -/** - * why: turns a leased iOS phone or tablet simulator, through the lease's device input, whether or not the input module is on. - */ -const deviceRotate = defineOperation({ - name: "device.rotate", - role: "agent", - effect: "write", - input: z - .object({ - leaseId: z.string().min(1).max(256), - orientation: z.enum(INPUT_ORIENTATIONS), - // why: rotate always refused an empty requesterId, so the shared format also keeps min(1). - requesterId: requesterIdSchema.pipe(z.string().min(1)).optional(), - }) - .strict(), - output: z.object({ leaseId: z.string().max(256), orientation: z.enum(INPUT_ORIENTATIONS) }), - authorize: authorizeOneLeaseCall, -}); - export const OPERATIONS = { "identity.get": identityGet, "catalog.get": catalogGet, From 3d83603ef779db2ee8db8fe1251763c0b7284953 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:05:56 +0100 Subject: [PATCH 08/83] refactor(http): split app.ts into parts --- src/http/app-helpers.ts | 164 +++++++ src/http/app-lease-request-routes.ts | 126 +++++ src/http/app-lease-routes.ts | 262 +++++++++++ src/http/app-types.ts | 111 +++++ src/http/app.ts | 657 ++------------------------- 5 files changed, 701 insertions(+), 619 deletions(-) create mode 100644 src/http/app-helpers.ts create mode 100644 src/http/app-lease-request-routes.ts create mode 100644 src/http/app-lease-routes.ts create mode 100644 src/http/app-types.ts diff --git a/src/http/app-helpers.ts b/src/http/app-helpers.ts new file mode 100644 index 000000000..199b32d88 --- /dev/null +++ b/src/http/app-helpers.ts @@ -0,0 +1,164 @@ +import type { Context } from "hono"; +import type { z } from "zod"; + +import type { Identity } from "../auth/index.js"; +import { parseDurationMs, type UsageWindowRefusal } from "../contract/index.js"; +import type { buildHttpSession } from "./dispatcher-session.js"; +import { badRequest, forbidden } from "./errors.js"; +import type { leaseRequestBodySchema, leaseRequestViewSchema } from "./lease-declarations.js"; +import type { HttpGatewayDeps, HttpRegistryDevice } from "./app-types.js"; +import type { LeaseRequestInput, TrackedRequestView } from "./tracker.js"; + +export const MAX_LONG_POLL_SECONDS = 60; + +export function toLeaseRequestInput( + body: z.infer, +): LeaseRequestInput { + return { + ...(body.device === undefined ? {} : { device: body.device }), + ...(body.class === undefined ? {} : { class: body.class }), + platform: body.platform, + ...leaseRequestOptionFields(body), + ...leaseRequestNameFields(body), + }; +} + +function leaseRequestNameFields(body: z.infer) { + return { + ...(body.owner === undefined ? {} : { owner: body.owner }), + ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), + ...(body.metadata === undefined ? {} : { metadata: body.metadata }), + }; +} + +function leaseRequestOptionFields(body: z.infer) { + return { + ...(body.os === undefined ? {} : { os: body.os }), + ...(body.ttlMs === undefined ? {} : { ttlMs: body.ttlMs }), + ...(body.timeoutMs === undefined ? {} : { timeoutMs: body.timeoutMs }), + ...(body.noWait === undefined ? {} : { noWait: body.noWait }), + ...(body.allowDownload === undefined ? {} : { allowDownload: body.allowDownload }), + ...(body.mode === undefined ? {} : { mode: body.mode }), + ...(body.except === undefined ? {} : { except: body.except }), + ...(body.physical === undefined ? {} : { physical: body.physical }), + ...leaseRequestImageFields(body), + }; +} + +function leaseRequestImageFields(body: z.infer) { + return { + ...(body.imageTag === undefined ? {} : { imageTag: body.imageTag }), + ...(body.baseline === undefined ? {} : { baseline: body.baseline }), + }; +} + +// why: no dispatcher operation reads a stored lease request by id, so there is no `authorize` hook to reuse; compared against the record's `ownerId`, the principal the daemon saw send it, never its caller-claimed `requesterId`. +export function requireOwnRequest(identity: Identity, ownerId: string): void { + if (identity.role === "operator" || identity.principal === ownerId) return; + throw forbidden("Not permitted to access another requester's resource"); +} + +// why: `lease.get` answers `UNKNOWN_LEASE` for a lease that does not exist and for one the session does not own alike, so the 404 does not tell them apart (the narrower, deliberate divergence from the mutating routes below that the known gaps in src/http/AGENTS.md describe). +export async function findOwnedLease( + deps: HttpGatewayDeps, + session: ReturnType, + id: string, +) { + const { lease } = await deps.dispatch("lease.get", { leaseId: id }, session); + return lease; +} + +/** why: Best-effort lease lookup for a lease id, scoped to what the session's `lease.list` would + * show it (own leases; admin sees all) -- used only to decorate a response after an + * authoritative dispatch has already decided whether the request is allowed. Never used to + * gate access itself (see the `DELETE /v1/leases/:id` route). */ +export async function findLeaseForDecoration( + deps: HttpGatewayDeps, + session: ReturnType, + id: string, +) { + const { leases } = await deps.dispatch("lease.list", {}, session); + return leases.find((candidate) => candidate.id === id); +} + +export function serializeRequest( + view: Pick, +): z.infer { + const base = { createdAt: view.createdAt, id: view.id }; + switch (view.state.stage) { + case "queued": + return { ...base, queuePosition: view.state.queuePosition, state: "queued" }; + case "downloading": { + const { stage: _stage, ...download } = view.state; + return { ...base, ...download, state: "downloading" }; + } + case "provisioning": + case "booting": + case "reclaiming": + return { ...base, etaSeconds: view.state.etaSeconds, state: view.state.stage }; + case "granted": + return { ...base, lease: view.state.lease, state: "granted" }; + case "failed": + return { ...base, error: view.state.error, state: "failed" }; + case "cancelled": + return { ...base, state: "cancelled" }; + } +} + +/** why: The device a lease names. On a gateway both carry the `workerId` they came from, and a + * device matches only on the lease's own worker: a device id is that worker's claim, so another + * worker reporting the same id never answers for it. On a worker neither carries one. */ +export function findDevice( + deps: HttpGatewayDeps, + lease: { readonly deviceId: string; readonly workerId?: string | undefined }, +): HttpRegistryDevice | undefined { + return deps.registry.snapshot.devices.find( + (device) => device.id === lease.deviceId && device.workerId === lease.workerId, + ); +} + +// why: an empty body is no body whatever its content type: a renew without one keeps the lease's own width. +export async function emptyBodyAsNone(c: Context, next: () => Promise): Promise { + const isJson = /^application\/([a-z-.]+\+)?json/i.test(c.req.header("content-type") ?? ""); + if ((await c.req.text()).trim() === "") { + // why: the validator reads the body only under a JSON content type, so an empty one is relabelled; `bodyCache.text` is typed as the string but holds the pending read, and the JSON validator parses what it holds, so `{}` stands in for the empty text. + if (!isJson) c.req.raw.headers.set("content-type", "application/json"); + Object.assign(c.req.bodyCache, { text: Promise.resolve("{}") }); + } + await next(); +} + +/** why: HTTP-only sugar translating `?since=5m` into the absolute `sinceTs` `events.replay` takes -- + * not a duplicate of any daemon-side logic (the operation itself takes an absolute timestamp), + * so it is not one of the re-implementations ADR §2 says fall away with the dispatcher move. */ +export function parseDuration(value: string): number { + const milliseconds = parseDurationMs(value); + if (milliseconds === undefined) throw badRequest(`Invalid duration: ${value}`); + return milliseconds; +} + +export function optionalQuery( + value: string | undefined, + name: Name, +): { [Key in Name]?: string } { + return value === undefined ? {} : ({ [name]: value } as { [Key in Name]?: string }); +} + +export function statsRefusal(refusal: UsageWindowRefusal): string { + switch (refusal.problem) { + case "since-and-from": + return "stats takes since or from, not both"; + case "to-without-from": + return "to needs from"; + case "not-before-to": + return "from must be earlier than to, or than now when to is left out"; + case "bad-duration": + return `Invalid duration: ${refusal.value}`; + case "bad-time": + return `Invalid time: ${refusal.value} (use ISO 8601 with Z or an offset)`; + } +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/http/app-lease-request-routes.ts b/src/http/app-lease-request-routes.ts new file mode 100644 index 000000000..a356f3496 --- /dev/null +++ b/src/http/app-lease-request-routes.ts @@ -0,0 +1,126 @@ +import type { OpenAPIHono } from "@hono/zod-openapi"; + +import { type HttpAppContext, type Env } from "./app-types.js"; +import { + MAX_LONG_POLL_SECONDS, + requireOwnRequest, + serializeRequest, + toLeaseRequestInput, +} from "./app-helpers.js"; +import { buildHttpSession } from "./dispatcher-session.js"; +import { requestNotCancellable, unknownRequest } from "./errors.js"; +import { + leaseRequestCancelRoute, + leaseRequestCreateRoute, + leaseRequestEventsRoute, + leaseRequestGetRoute, + leaseRequestListRoute, + requestsReplySchema, +} from "./lease-declarations.js"; +import { pipeSse } from "./sse.js"; +import { isTerminalStage } from "./tracker.js"; + +// why: the lease-request resource routes, registered in the order the app serves them: each one's +// middleware and matching depend on that order, so the calls here must stay exactly as they were +// inline in `createHttpApp`. +export function registerLeaseRequestRoutes( + app: OpenAPIHono, + { agentAuth, deps, tracker }: HttpAppContext, +): void { + app.openapi({ ...leaseRequestCreateRoute, middleware: [agentAuth] }, async (c) => { + const identity = c.get("identity"); + const body = c.req.valid("json"); + const idempotencyKey = c.req.header("Idempotency-Key"); + const outcome = await tracker.submit(identity, toLeaseRequestInput(body), idempotencyKey); + if (outcome.kind === "rejected") throw outcome.error; + c.header("Location", `/v1/lease-requests/${outcome.view.id}`); + return c.json({ request: serializeRequest(outcome.view) }, 201); + }); + + // why: operator-role in effect: `list.get` is an admin operation, so an agent token gets `FORBIDDEN`/403 from the shared dispatcher. + app.openapi({ ...leaseRequestListRoute, middleware: [agentAuth] }, async (c) => { + const requests = await deps.dispatch( + "list.get", + { kind: "requests" }, + buildHttpSession(c.get("identity")), + ); + return c.json(requestsReplySchema.parse({ requests }), 200); + }); + + app.openapi({ ...leaseRequestGetRoute, middleware: [agentAuth] }, async (c) => { + const id = c.req.valid("param").id; + const initial = tracker.get(id); + if (initial === undefined) throw unknownRequest(id); + requireOwnRequest(c.get("identity"), initial.ownerId); + + const { wait } = c.req.valid("query"); + if (wait !== undefined) { + // why: Clamped, not rejected: an oversized wait still long-polls correctly -- the client + // simply re-polls sooner than it asked -- and the clamp bounds how long an abandoned + // poll can pin its listener and timer. Aborting the request releases them immediately. + await tracker.waitForChange( + id, + Math.min(Number(wait), MAX_LONG_POLL_SECONDS), + c.req.raw.signal, + ); + } + + const view = tracker.get(id) ?? initial; + return c.json({ request: serializeRequest(view) }, 200); + }); + + app.openapi({ ...leaseRequestEventsRoute, middleware: [agentAuth] }, (c) => { + const id = c.req.valid("param").id; + const initial = tracker.get(id); + if (initial === undefined) throw unknownRequest(id); + requireOwnRequest(c.get("identity"), initial.ownerId); + + return pipeSse( + c, + deps.clock, + { + subscribe(send, end) { + const current = tracker.get(id) ?? initial; + send({ data: serializeRequest(current), event: current.state.stage }); + if (isTerminalStage(current.state)) { + end(); + return () => {}; + } + const unsubscribe = tracker.subscribe(id, (state) => { + send({ data: serializeRequest({ ...current, state }), event: state.stage }); + if (isTerminalStage(state)) end(); + }); + // why: nothing in this process is driving the request -- one a restarted daemon has not + // settled yet -- so no change will ever arrive on this stream: end it rather than + // hold it open. The client reconnects and reads the settled state. + if (unsubscribe === undefined) { + end(); + return () => {}; + } + return unsubscribe; + }, + }, + c.get("identity").expiresAt, + ); + }); + + app.openapi({ ...leaseRequestCancelRoute, middleware: [agentAuth] }, async (c) => { + const id = c.req.valid("param").id; + const existing = tracker.get(id); + if (existing === undefined) throw unknownRequest(id); + requireOwnRequest(c.get("identity"), existing.ownerId); + + const outcome = await tracker.cancel(id, c.get("identity")); + if (outcome.kind === "cancelled") return c.body(null, 204); + if (outcome.kind === "not-found") throw unknownRequest(id); + if (outcome.leaseId !== undefined) { + throw requestNotCancellable( + `Request already granted lease ${outcome.leaseId}; release it instead`, + { leaseId: outcome.leaseId }, + ); + } + throw requestNotCancellable( + "Request is no longer cancellable -- device work is already in flight for it", + ); + }); +} diff --git a/src/http/app-lease-routes.ts b/src/http/app-lease-routes.ts new file mode 100644 index 000000000..da3ca7ea9 --- /dev/null +++ b/src/http/app-lease-routes.ts @@ -0,0 +1,262 @@ +import type { OpenAPIHono } from "@hono/zod-openapi"; + +import type { ComponentProgress } from "../contract/index.js"; +import { componentInstallRoute } from "./admin-declarations.js"; +import type { Env, HttpAppContext } from "./app-types.js"; +import { + emptyBodyAsNone, + findDevice, + findLeaseForDecoration, + findOwnedLease, +} from "./app-helpers.js"; +import { buildHttpSession } from "./dispatcher-session.js"; +import { errorResponse, mapError, unknownLease } from "./errors.js"; +import { + leaseEventsRoute, + leaseExecRoute, + leaseGetRoute, + leaseListRoute, + leaseRenewRoute, + leaseReleaseRoute, + leaseRotateRoute, +} from "./lease-declarations.js"; +import { OutputRelay } from "./output-relay.js"; +import { pipeSse } from "./sse.js"; +import { buildLeasePayload } from "./tracker.js"; + +// why: the lease resource routes and the component-install stream, registered in the order the +// app serves them: each one's middleware and matching depend on that order, so the calls here +// must stay exactly as they were inline in `createHttpApp`. +export function registerLeaseRoutes( + app: OpenAPIHono, + { agentAuth, deps, notices, tracker }: HttpAppContext, +): void { + app.openapi({ ...leaseGetRoute, middleware: [agentAuth] }, async (c) => { + const identity = c.get("identity"); + const session = buildHttpSession(identity); + const lease = await findOwnedLease(deps, session, c.req.valid("param").id); + const device = findDevice(deps, lease); + if (device === undefined) throw unknownLease(lease.id); + // why: ADR 0004: `ttlMs` comes off the lease record itself, so a payload served after a daemon + // restart reports the lease's real width rather than a mode default standing in for a + // per-request value this gateway no longer remembers. `requestId` is still gateway-side + // bookkeeping, and is still the one field a restart can drop. + const requestId = tracker.requestIdForLease(lease.id); + return c.json( + { lease: buildLeasePayload(device, lease, requestId === undefined ? {} : { requestId }) }, + 200, + ); + }); + + app.openapi({ ...leaseRenewRoute, middleware: [agentAuth, emptyBodyAsNone] }, async (c) => { + const identity = c.get("identity"); + const session = buildHttpSession(identity); + const id = c.req.valid("param").id; + const { ttlMs } = c.req.valid("json"); + + // why: Dispatched directly (not via `findOwnedLease`/`lease.get`) so `lease.renew`'s own + // `ownsLease` authorize hook answers -- an unowned lease is `FORBIDDEN`/403, matching the + // socket transport, rather than the `UNKNOWN_LEASE`/404 `lease.get`'s hidden-lease answer would produce + // (S6). An unknown id still surfaces as `UNKNOWN_LEASE`/404 from the handler itself, and a + // `ttlMs` above `lease.maxTtlMs` as `BAD_REQUEST`/400 -- the cap lives in the shared + // dispatcher, and this route awaits it, so there is nothing to restate here. Answering it + // earlier would answer it *before* `ownsLease` and turn another requester's lease into a + // 400 about its TTL where the socket and the HTTP API both say 403. An omitted + // `ttlMs` re-applies the lease's own stored width (ADR 0004 §4), which is why nothing is + // recorded here afterwards. + const renewed = await deps.dispatch( + "lease.renew", + { leaseId: id, ...(ttlMs === undefined ? {} : { ttlMs }) }, + session, + ); + + return c.json( + { + expiresAt: new Date(renewed.ttlDeadline).toISOString(), + leaseId: renewed.id, + notices: notices.drain(id), + }, + 200, + ); + }); + + app.openapi({ ...leaseExecRoute, middleware: [agentAuth] }, async (c) => { + const identity = c.get("identity"); + const leaseId = c.req.valid("param").id; + const { requesterId, ...body } = c.req.valid("json"); + // why: Requester identity is the token's over HTTP, never the body's -- except for an + // `operator` token, which is the proxying case `device.exec` reads it for. An agent + // token that supplies one at all is refused, but by `device.exec`'s own `authorize` + // hook (round 4, F4) rather than a route-level check: ADR 0003 §2 puts a contract + // operation's answer in the dispatcher, not a transport, so the same request over the + // unix socket or a future uplink gets the same `FORBIDDEN` this route now inherits + // rather than defining. + + // why: dispatched directly, like `renew`/`release` and unlike the single-lease *reads*: this + // route mutates a device, so it answers `device.exec`'s own ownership hook -- 403 for + // another requester's lease, 404 for an id that names none -- rather than the 404-for-both + // `lease.get` would produce (see `findOwnedLease`'s comment). + // Where a chunk goes at each stage of this request's life, including after the client + // disconnects -- see `OutputRelay`, which is where the "retain nothing then" rule lives + // and is tested. + const relay = new OutputRelay(); + let onStarted!: () => void; + const started = new Promise((resolve) => { + onStarted = resolve; + }); + + const settled = deps + .dispatch( + "device.exec", + { leaseId, ...body, ...(requesterId === undefined ? {} : { requesterId }) }, + buildHttpSession(identity, { + onStarted: () => onStarted(), + onOutput: (stream, chunk) => relay.push({ chunk, stream }), + }), + ) + .then( + (result) => ({ exitCode: result.exitCode, kind: "exited" }) as const, + (error: unknown) => ({ error, kind: "failed" }) as const, + ); + + // why: An SSE response cannot take back its status code, so the decision point is the moment + // the process exists: every failure that can precede one -- an unowned lease, an unknown + // one, a refused verb, an unknown tool, a daemon still starting -- lands before + // `onStarted` and is answered as an ordinary JSON error with its own status. After it, + // the stream is committed, which is what gets a silent long-running command its `200` + // and its keepalives immediately, and what makes `EXEC_TIMEOUT` always arrive as the + // stream's terminal `error` event (ADR 0005 §19e). + const outcome = await Promise.race([settled, started.then(() => undefined)]); + if (outcome?.kind === "failed") return errorResponse(c, outcome.error); + + return pipeSse(c, deps.clock, { + subscribe(send, end) { + relay.attach((chunk) => send({ data: chunk, event: "output" })); + void settled.then((result) => { + if (result.kind === "exited") { + send({ data: { exitCode: result.exitCode }, event: "exit" }); + } else { + const mapped = mapError(result.error); + send({ + data: { error: { code: mapped.code, message: mapped.message } }, + event: "error", + }); + } + end(); + }); + return () => relay.drop(); + }, + }); + }); + + app.openapi({ ...componentInstallRoute, middleware: [agentAuth] }, async (c) => { + const { workers, ...component } = c.req.valid("json"); + type ProgressEvent = ComponentProgress & { readonly workerId?: string }; + const early: ProgressEvent[] = []; + let deliver: ((progress: ProgressEvent) => void) | undefined; + let onAdmitted!: () => void; + const admitted = new Promise((resolve) => { + onAdmitted = resolve; + }); + const session = buildHttpSession(c.get("identity"), { + onStarted: () => onAdmitted(), + onComponentProgress: (progress, workerId) => { + const event = workerId === undefined ? progress : { ...progress, workerId }; + if (deliver === undefined) early.push(event); + else deliver(event); + }, + }); + + const dispatched: Promise = + workers === undefined + ? deps.dispatch("component.install", component, session) + : deps.dispatch("worker.install-component", { ...component, workers }, session); + const settled = dispatched.then( + (result) => ({ kind: "done", result }) as const, + (error: unknown) => ({ error, kind: "failed" }) as const, + ); + + const outcome = await Promise.race([settled, admitted.then(() => undefined)]); + if (outcome?.kind === "failed") return errorResponse(c, outcome.error); + + return pipeSse(c, deps.clock, { + subscribe(send, end) { + deliver = (progress) => void send({ data: progress, event: "progress" }); + for (const progress of early.splice(0)) deliver(progress); + void settled.then((result) => { + if (result.kind === "done") { + send({ data: result.result, event: "result" }); + } else { + const mapped = mapError(result.error); + send({ + data: { error: { code: mapped.code, message: mapped.message } }, + event: "error", + }); + } + end(); + }); + return () => undefined; + }, + }); + }); + + app.openapi({ ...leaseEventsRoute, middleware: [agentAuth] }, async (c) => { + const identity = c.get("identity"); + const lease = await findOwnedLease(deps, buildHttpSession(identity), c.req.valid("param").id); + + return pipeSse( + c, + deps.clock, + { + subscribe(send, end) { + return notices.subscribe(lease.id, (notice) => { + send({ data: notice, event: notice.event }); + if (notice.event === "lease_lost") end(); + }); + }, + }, + identity.expiresAt, + ); + }); + + app.openapi({ ...leaseRotateRoute, middleware: [agentAuth] }, async (c) => { + const { orientation } = c.req.valid("json"); + // why: dispatched directly, like `exec`, so `device.rotate`'s own authorize answers 403 for another requester's lease and the handler 404 for an unknown one. + const rotated = await deps.dispatch( + "device.rotate", + { leaseId: c.req.valid("param").id, orientation }, + buildHttpSession(c.get("identity")), + ); + return c.json(rotated, 200); + }); + + app.openapi({ ...leaseReleaseRoute, middleware: [agentAuth] }, async (c) => { + const identity = c.get("identity"); + const session = buildHttpSession(identity); + const id = c.req.valid("param").id; + + // why: best-effort lookup, only to decorate the response with the device id/state -- + // `lease.release`'s output schema is `{ leaseId }` alone (no `deviceId`), so this is still + // needed for the response body. It is *not* the authorization decision: `lease.release` + // below is dispatched directly and is the sole source of truth for whether this request is + // allowed (S6). Its `ownsLease` hook answers `FORBIDDEN`/403 for someone else's lease and + // the handler answers `UNKNOWN_LEASE`/404 for a nonexistent one, matching the socket + // transport exactly. + const lease = await findLeaseForDecoration(deps, session, id); + + await deps.dispatch("lease.release", { leaseId: id }, session); + + const device = lease === undefined ? undefined : findDevice(deps, lease); + return c.json( + { + device: { id: lease?.deviceId ?? id, state: device?.state ?? "reclaiming" }, + released: true as const, + }, + 202, + ); + }); + + app.openapi({ ...leaseListRoute, middleware: [agentAuth] }, async (c) => + c.json(await deps.dispatch("lease.list", {}, buildHttpSession(c.get("identity"))), 200), + ); +} diff --git a/src/http/app-types.ts b/src/http/app-types.ts new file mode 100644 index 000000000..1089709f0 --- /dev/null +++ b/src/http/app-types.ts @@ -0,0 +1,111 @@ +import type { MiddlewareHandler } from "hono"; + +import type { Auth, Identity } from "../auth/index.js"; +import type { EventBus } from "../bus/index.js"; +import type { DeviceRecord } from "../core/index.js"; +import type { OwnerRoutedFacts } from "../daemon/owner-routed-facts.js"; +import type { Clock, IdGenerator, LeaseSocketAccept, Logger } from "../ports/index.js"; +import type { AuthEnv } from "./auth.js"; +import type { HttpDispatch } from "./dispatcher-session.js"; +import type { LeaseNoticeBuffer } from "./notices.js"; +import type { HttpLeaseDevice, LeaseRequestReader, LeaseRequestTracker } from "./tracker.js"; + +/** + * why: The device fields the lease routes read: what a lease payload is built from, plus two that + * only one kind of daemon has. A worker's own registry record carries `state`, which `DELETE + * /v1/leases/:id` reports; a gateway holds no device state of its own and leaves it out rather + * than inventing one. A gateway's device carries the `workerId` it lives on, because a device + * id is one worker's claim and is matched only together with the lease's own `workerId`. + */ +export interface HttpRegistryDevice extends HttpLeaseDevice { + readonly state?: DeviceRecord["state"]; + readonly workerId?: string; +} + +export interface HttpRegistryReader { + readonly snapshot: { + readonly devices: readonly HttpRegistryDevice[]; + }; +} + +export interface HttpGatewayDeps { + readonly dispatch: HttpDispatch; + readonly registry: HttpRegistryReader; + readonly leaseRequests: LeaseRequestReader; + readonly eventBus: EventBus; + readonly ownerRoutedFacts: OwnerRoutedFacts; + readonly clock: Clock; + readonly idGenerator: IdGenerator; + readonly logger: Logger; + readonly auth: Pick; + readonly answerDownloadsEarly?: boolean | undefined; + readonly mounts?: readonly HttpMount[]; + readonly leaseRoutes?: readonly HttpLeaseRoute[]; + readonly leaseFiles: HttpLeaseFiles; +} + +export interface HttpLeaseFiles { + handle( + request: Request, + identity: Identity, + lease: HttpLeaseSocketLease, + name: string | undefined, + ): Promise; +} + +// why: the lease is resolved here, with `lease.get`'s own owner-or-operator answer, so a module's +// lease handler never checks who owns it; http knows a module only by its name. +export interface HttpLeaseRoute { + readonly name: string; + // why: the module's two-step socket handler, run for a WebSocket upgrade of the same path. + readonly socket?: { + open( + request: Request, + identity: Identity, + lease: HttpLeaseSocketLease, + ): Promise; + }; + handle(request: Request, identity: Identity, lease: HttpLeaseSocketLease): Promise; +} + +interface HttpLeaseSocketLease { + readonly id: string; + readonly deviceId: string; + readonly workerId?: string; + readonly workerLeaseId?: string; +} + +export type HttpLeaseSocketAccept = LeaseSocketAccept; + +// why: what the upgrade router needs of a lease socket request: the refusal to send before the +// upgrade, or the accept to run on the upgraded socket. +export type HttpLeaseSocketOutcome = + | { readonly response: Response } + | { readonly accept: HttpLeaseSocketAccept }; + +export interface LeaseSocketCapture { + accept?: HttpLeaseSocketAccept; +} + +export interface HttpMount { + readonly prefix: string; + handle(request: Request, identity: Identity): Promise; +} + +export type Env = AuthEnv; + +export interface HttpAppDisposable { + readonly dispose: () => void; + // why: a WebSocket upgrade of a lease route goes through the app's own auth and lease resolution, + // so a refusal is the answer any request to the path would get; only the accept is new. + readonly leaseSocket: (request: Request) => Promise; +} + +// why: what the lease route blocks close over: the app's dependencies, its request tracker and +// notice buffer, and the agent auth middleware every route in them runs. +export interface HttpAppContext { + readonly deps: HttpGatewayDeps; + readonly tracker: LeaseRequestTracker; + readonly notices: LeaseNoticeBuffer; + readonly agentAuth: MiddlewareHandler; +} diff --git a/src/http/app.ts b/src/http/app.ts index 046e6c3b8..937fc4556 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -1,70 +1,52 @@ import { OpenAPIHono } from "@hono/zod-openapi"; import type { Context } from "hono"; import { etag } from "hono/etag"; -import { z } from "zod"; -import type { EventBus } from "../bus/index.js"; +import type { Identity } from "../auth/index.js"; +import { eventsPageQuery, OPERATIONS, usageWindow } from "../contract/index.js"; +import { registerLeaseRequestRoutes } from "./app-lease-request-routes.js"; +import { registerLeaseRoutes } from "./app-lease-routes.js"; import { - type ComponentProgress, - eventsPageQuery, - OPERATIONS, - parseDurationMs, - usageWindow, - type UsageWindowRefusal, -} from "../contract/index.js"; -import type { DeviceRecord } from "../core/index.js"; -import type { OwnerRoutedFacts } from "../daemon/owner-routed-facts.js"; -import type { Clock, IdGenerator, LeaseSocketAccept, Logger } from "../ports/index.js"; -import { type AuthEnv, requireAuth } from "./auth.js"; -import { buildHttpSession, type HttpDispatch } from "./dispatcher-session.js"; + emptyBodyAsNone, + errorMessage, + findOwnedLease, + optionalQuery, + parseDuration, + statsRefusal, +} from "./app-helpers.js"; +import type { + Env, + HttpAppDisposable, + HttpGatewayDeps, + HttpLeaseSocketOutcome, + LeaseSocketCapture, +} from "./app-types.js"; +import { requireAuth } from "./auth.js"; import { - componentInstallRoute, baselineListRoute, baselineRemoveRoute, baselineSaveRoute, componentListRoute, componentRemoveRoute, + configRoute, eventsRoute, eventsStreamRoute, + inboxRoute, tokenListRoute, updateRoute, workerDrainRoute, - inboxRoute, - configRoute, workerListRoute, workerRemoveRoute, workerUndrainRoute, workerUpdateRoute, } from "./admin-declarations.js"; +import { buildHttpSession } from "./dispatcher-session.js"; +import { badRequest, errorResponse, mapError } from "./errors.js"; import { - badRequest, - errorResponse, - forbidden, - mapError, - requestNotCancellable, - unknownLease, - unknownRequest, -} from "./errors.js"; -import { - type leaseRequestBodySchema, - leaseEventsRoute, - leaseExecRoute, leaseFileDeleteRoute, leaseFileGetRoute, leaseFileListRoute, leaseFilePutRoute, - leaseGetRoute, - leaseListRoute, - leaseReleaseRoute, - leaseRotateRoute, - leaseRenewRoute, - leaseRequestCancelRoute, - leaseRequestCreateRoute, - leaseRequestEventsRoute, - leaseRequestGetRoute, - leaseRequestListRoute, - type leaseRequestViewSchema, - requestsReplySchema, } from "./lease-declarations.js"; import { LeaseNoticeBuffer } from "./notices.js"; import { @@ -79,150 +61,20 @@ import { statusRoute, } from "./route-declarations.js"; import { validationHook } from "./routes.js"; -import { OutputRelay } from "./output-relay.js"; import { pipeSse } from "./sse.js"; -import type { Auth, Identity } from "../auth/index.js"; -import { - buildLeasePayload, - type HttpLeaseDevice, - isTerminalStage, - type LeaseRequestInput, - type LeaseRequestReader, - LeaseRequestTracker, - type TrackedRequestView, -} from "./tracker.js"; - -/** - * why: The device fields the lease routes read: what a lease payload is built from, plus two that - * only one kind of daemon has. A worker's own registry record carries `state`, which `DELETE - * /v1/leases/:id` reports; a gateway holds no device state of its own and leaves it out rather - * than inventing one. A gateway's device carries the `workerId` it lives on, because a device - * id is one worker's claim and is matched only together with the lease's own `workerId`. - */ -export interface HttpRegistryDevice extends HttpLeaseDevice { - readonly state?: DeviceRecord["state"]; - readonly workerId?: string; -} - -export interface HttpRegistryReader { - readonly snapshot: { - readonly devices: readonly HttpRegistryDevice[]; - }; -} - -export interface HttpGatewayDeps { - readonly dispatch: HttpDispatch; - readonly registry: HttpRegistryReader; - readonly leaseRequests: LeaseRequestReader; - readonly eventBus: EventBus; - readonly ownerRoutedFacts: OwnerRoutedFacts; - readonly clock: Clock; - readonly idGenerator: IdGenerator; - readonly logger: Logger; - readonly auth: Pick; - readonly answerDownloadsEarly?: boolean | undefined; - readonly mounts?: readonly HttpMount[]; - readonly leaseRoutes?: readonly HttpLeaseRoute[]; - readonly leaseFiles: HttpLeaseFiles; -} - -export interface HttpLeaseFiles { - handle( - request: Request, - identity: Identity, - lease: HttpLeaseSocketLease, - name: string | undefined, - ): Promise; -} - -// why: the lease is resolved here, with `lease.get`'s own owner-or-operator answer, so a module's -// lease handler never checks who owns it; http knows a module only by its name. -export interface HttpLeaseRoute { - readonly name: string; - // why: the module's two-step socket handler, run for a WebSocket upgrade of the same path. - readonly socket?: { - open( - request: Request, - identity: Identity, - lease: HttpLeaseSocketLease, - ): Promise; - }; - handle(request: Request, identity: Identity, lease: HttpLeaseSocketLease): Promise; -} - -interface HttpLeaseSocketLease { - readonly id: string; - readonly deviceId: string; - readonly workerId?: string; - readonly workerLeaseId?: string; -} - -export type HttpLeaseSocketAccept = LeaseSocketAccept; - -// why: what the upgrade router needs of a lease socket request: the refusal to send before the -// upgrade, or the accept to run on the upgraded socket. -export type HttpLeaseSocketOutcome = - | { readonly response: Response } - | { readonly accept: HttpLeaseSocketAccept }; - -interface LeaseSocketCapture { - accept?: HttpLeaseSocketAccept; -} - -export interface HttpMount { - readonly prefix: string; - handle(request: Request, identity: Identity): Promise; -} - -type Env = AuthEnv; - -const MAX_LONG_POLL_SECONDS = 60; - -function toLeaseRequestInput(body: z.infer): LeaseRequestInput { - return { - ...(body.device === undefined ? {} : { device: body.device }), - ...(body.class === undefined ? {} : { class: body.class }), - platform: body.platform, - ...leaseRequestOptionFields(body), - ...leaseRequestNameFields(body), - }; -} - -function leaseRequestNameFields(body: z.infer) { - return { - ...(body.owner === undefined ? {} : { owner: body.owner }), - ...(body.leaseId === undefined ? {} : { leaseId: body.leaseId }), - ...(body.metadata === undefined ? {} : { metadata: body.metadata }), - }; -} - -function leaseRequestOptionFields(body: z.infer) { - return { - ...(body.os === undefined ? {} : { os: body.os }), - ...(body.ttlMs === undefined ? {} : { ttlMs: body.ttlMs }), - ...(body.timeoutMs === undefined ? {} : { timeoutMs: body.timeoutMs }), - ...(body.noWait === undefined ? {} : { noWait: body.noWait }), - ...(body.allowDownload === undefined ? {} : { allowDownload: body.allowDownload }), - ...(body.mode === undefined ? {} : { mode: body.mode }), - ...(body.except === undefined ? {} : { except: body.except }), - ...(body.physical === undefined ? {} : { physical: body.physical }), - ...leaseRequestImageFields(body), - }; -} - -function leaseRequestImageFields(body: z.infer) { - return { - ...(body.imageTag === undefined ? {} : { imageTag: body.imageTag }), - ...(body.baseline === undefined ? {} : { baseline: body.baseline }), - }; -} - -export interface HttpAppDisposable { - readonly dispose: () => void; - // why: a WebSocket upgrade of a lease route goes through the app's own auth and lease resolution, - // so a refusal is the answer any request to the path would get; only the accept is new. - readonly leaseSocket: (request: Request) => Promise; -} +import { LeaseRequestTracker } from "./tracker.js"; + +export type { + HttpAppDisposable, + HttpGatewayDeps, + HttpLeaseFiles, + HttpLeaseRoute, + HttpLeaseSocketAccept, + HttpLeaseSocketOutcome, + HttpMount, + HttpRegistryDevice, + HttpRegistryReader, +} from "./app-types.js"; // fallow-ignore-next-line complexity -- route wiring for one focused resource surface; splitting it would scatter the shared closures (tracker, notices) across files for no clarity gain. export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpAppDisposable { @@ -367,331 +219,9 @@ export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpApp ), ); - app.openapi({ ...leaseRequestCreateRoute, middleware: [agentAuth] }, async (c) => { - const identity = c.get("identity"); - const body = c.req.valid("json"); - const idempotencyKey = c.req.header("Idempotency-Key"); - const outcome = await tracker.submit(identity, toLeaseRequestInput(body), idempotencyKey); - if (outcome.kind === "rejected") throw outcome.error; - c.header("Location", `/v1/lease-requests/${outcome.view.id}`); - return c.json({ request: serializeRequest(outcome.view) }, 201); - }); - - // why: operator-role in effect: `list.get` is an admin operation, so an agent token gets `FORBIDDEN`/403 from the shared dispatcher. - app.openapi({ ...leaseRequestListRoute, middleware: [agentAuth] }, async (c) => { - const requests = await deps.dispatch( - "list.get", - { kind: "requests" }, - buildHttpSession(c.get("identity")), - ); - return c.json(requestsReplySchema.parse({ requests }), 200); - }); - - app.openapi({ ...leaseRequestGetRoute, middleware: [agentAuth] }, async (c) => { - const id = c.req.valid("param").id; - const initial = tracker.get(id); - if (initial === undefined) throw unknownRequest(id); - requireOwnRequest(c.get("identity"), initial.ownerId); - - const { wait } = c.req.valid("query"); - if (wait !== undefined) { - // why: Clamped, not rejected: an oversized wait still long-polls correctly -- the client - // simply re-polls sooner than it asked -- and the clamp bounds how long an abandoned - // poll can pin its listener and timer. Aborting the request releases them immediately. - await tracker.waitForChange( - id, - Math.min(Number(wait), MAX_LONG_POLL_SECONDS), - c.req.raw.signal, - ); - } - - const view = tracker.get(id) ?? initial; - return c.json({ request: serializeRequest(view) }, 200); - }); - - app.openapi({ ...leaseRequestEventsRoute, middleware: [agentAuth] }, (c) => { - const id = c.req.valid("param").id; - const initial = tracker.get(id); - if (initial === undefined) throw unknownRequest(id); - requireOwnRequest(c.get("identity"), initial.ownerId); - - return pipeSse( - c, - deps.clock, - { - subscribe(send, end) { - const current = tracker.get(id) ?? initial; - send({ data: serializeRequest(current), event: current.state.stage }); - if (isTerminalStage(current.state)) { - end(); - return () => {}; - } - const unsubscribe = tracker.subscribe(id, (state) => { - send({ data: serializeRequest({ ...current, state }), event: state.stage }); - if (isTerminalStage(state)) end(); - }); - // why: nothing in this process is driving the request -- one a restarted daemon has not - // settled yet -- so no change will ever arrive on this stream: end it rather than - // hold it open. The client reconnects and reads the settled state. - if (unsubscribe === undefined) { - end(); - return () => {}; - } - return unsubscribe; - }, - }, - c.get("identity").expiresAt, - ); - }); - - app.openapi({ ...leaseRequestCancelRoute, middleware: [agentAuth] }, async (c) => { - const id = c.req.valid("param").id; - const existing = tracker.get(id); - if (existing === undefined) throw unknownRequest(id); - requireOwnRequest(c.get("identity"), existing.ownerId); - - const outcome = await tracker.cancel(id, c.get("identity")); - if (outcome.kind === "cancelled") return c.body(null, 204); - if (outcome.kind === "not-found") throw unknownRequest(id); - if (outcome.leaseId !== undefined) { - throw requestNotCancellable( - `Request already granted lease ${outcome.leaseId}; release it instead`, - { leaseId: outcome.leaseId }, - ); - } - throw requestNotCancellable( - "Request is no longer cancellable -- device work is already in flight for it", - ); - }); - - app.openapi({ ...leaseGetRoute, middleware: [agentAuth] }, async (c) => { - const identity = c.get("identity"); - const session = buildHttpSession(identity); - const lease = await findOwnedLease(deps, session, c.req.valid("param").id); - const device = findDevice(deps, lease); - if (device === undefined) throw unknownLease(lease.id); - // why: ADR 0004: `ttlMs` comes off the lease record itself, so a payload served after a daemon - // restart reports the lease's real width rather than a mode default standing in for a - // per-request value this gateway no longer remembers. `requestId` is still gateway-side - // bookkeeping, and is still the one field a restart can drop. - const requestId = tracker.requestIdForLease(lease.id); - return c.json( - { lease: buildLeasePayload(device, lease, requestId === undefined ? {} : { requestId }) }, - 200, - ); - }); + registerLeaseRequestRoutes(app, { agentAuth, deps, notices, tracker }); - app.openapi({ ...leaseRenewRoute, middleware: [agentAuth, emptyBodyAsNone] }, async (c) => { - const identity = c.get("identity"); - const session = buildHttpSession(identity); - const id = c.req.valid("param").id; - const { ttlMs } = c.req.valid("json"); - - // why: Dispatched directly (not via `findOwnedLease`/`lease.get`) so `lease.renew`'s own - // `ownsLease` authorize hook answers -- an unowned lease is `FORBIDDEN`/403, matching the - // socket transport, rather than the `UNKNOWN_LEASE`/404 `lease.get`'s hidden-lease answer would produce - // (S6). An unknown id still surfaces as `UNKNOWN_LEASE`/404 from the handler itself, and a - // `ttlMs` above `lease.maxTtlMs` as `BAD_REQUEST`/400 -- the cap lives in the shared - // dispatcher, and this route awaits it, so there is nothing to restate here. Answering it - // earlier would answer it *before* `ownsLease` and turn another requester's lease into a - // 400 about its TTL where the socket and the HTTP API both say 403. An omitted - // `ttlMs` re-applies the lease's own stored width (ADR 0004 §4), which is why nothing is - // recorded here afterwards. - const renewed = await deps.dispatch( - "lease.renew", - { leaseId: id, ...(ttlMs === undefined ? {} : { ttlMs }) }, - session, - ); - - return c.json( - { - expiresAt: new Date(renewed.ttlDeadline).toISOString(), - leaseId: renewed.id, - notices: notices.drain(id), - }, - 200, - ); - }); - - app.openapi({ ...leaseExecRoute, middleware: [agentAuth] }, async (c) => { - const identity = c.get("identity"); - const leaseId = c.req.valid("param").id; - const { requesterId, ...body } = c.req.valid("json"); - // why: Requester identity is the token's over HTTP, never the body's -- except for an - // `operator` token, which is the proxying case `device.exec` reads it for. An agent - // token that supplies one at all is refused, but by `device.exec`'s own `authorize` - // hook (round 4, F4) rather than a route-level check: ADR 0003 §2 puts a contract - // operation's answer in the dispatcher, not a transport, so the same request over the - // unix socket or a future uplink gets the same `FORBIDDEN` this route now inherits - // rather than defining. - - // why: dispatched directly, like `renew`/`release` and unlike the single-lease *reads*: this - // route mutates a device, so it answers `device.exec`'s own ownership hook -- 403 for - // another requester's lease, 404 for an id that names none -- rather than the 404-for-both - // `lease.get` would produce (see `findOwnedLease`'s comment). - // Where a chunk goes at each stage of this request's life, including after the client - // disconnects -- see `OutputRelay`, which is where the "retain nothing then" rule lives - // and is tested. - const relay = new OutputRelay(); - let onStarted!: () => void; - const started = new Promise((resolve) => { - onStarted = resolve; - }); - - const settled = deps - .dispatch( - "device.exec", - { leaseId, ...body, ...(requesterId === undefined ? {} : { requesterId }) }, - buildHttpSession(identity, { - onStarted: () => onStarted(), - onOutput: (stream, chunk) => relay.push({ chunk, stream }), - }), - ) - .then( - (result) => ({ exitCode: result.exitCode, kind: "exited" }) as const, - (error: unknown) => ({ error, kind: "failed" }) as const, - ); - - // why: An SSE response cannot take back its status code, so the decision point is the moment - // the process exists: every failure that can precede one -- an unowned lease, an unknown - // one, a refused verb, an unknown tool, a daemon still starting -- lands before - // `onStarted` and is answered as an ordinary JSON error with its own status. After it, - // the stream is committed, which is what gets a silent long-running command its `200` - // and its keepalives immediately, and what makes `EXEC_TIMEOUT` always arrive as the - // stream's terminal `error` event (ADR 0005 §19e). - const outcome = await Promise.race([settled, started.then(() => undefined)]); - if (outcome?.kind === "failed") return errorResponse(c, outcome.error); - - return pipeSse(c, deps.clock, { - subscribe(send, end) { - relay.attach((chunk) => send({ data: chunk, event: "output" })); - void settled.then((result) => { - if (result.kind === "exited") { - send({ data: { exitCode: result.exitCode }, event: "exit" }); - } else { - const mapped = mapError(result.error); - send({ - data: { error: { code: mapped.code, message: mapped.message } }, - event: "error", - }); - } - end(); - }); - return () => relay.drop(); - }, - }); - }); - - app.openapi({ ...componentInstallRoute, middleware: [agentAuth] }, async (c) => { - const { workers, ...component } = c.req.valid("json"); - type ProgressEvent = ComponentProgress & { readonly workerId?: string }; - const early: ProgressEvent[] = []; - let deliver: ((progress: ProgressEvent) => void) | undefined; - let onAdmitted!: () => void; - const admitted = new Promise((resolve) => { - onAdmitted = resolve; - }); - const session = buildHttpSession(c.get("identity"), { - onStarted: () => onAdmitted(), - onComponentProgress: (progress, workerId) => { - const event = workerId === undefined ? progress : { ...progress, workerId }; - if (deliver === undefined) early.push(event); - else deliver(event); - }, - }); - - const dispatched: Promise = - workers === undefined - ? deps.dispatch("component.install", component, session) - : deps.dispatch("worker.install-component", { ...component, workers }, session); - const settled = dispatched.then( - (result) => ({ kind: "done", result }) as const, - (error: unknown) => ({ error, kind: "failed" }) as const, - ); - - const outcome = await Promise.race([settled, admitted.then(() => undefined)]); - if (outcome?.kind === "failed") return errorResponse(c, outcome.error); - - return pipeSse(c, deps.clock, { - subscribe(send, end) { - deliver = (progress) => void send({ data: progress, event: "progress" }); - for (const progress of early.splice(0)) deliver(progress); - void settled.then((result) => { - if (result.kind === "done") { - send({ data: result.result, event: "result" }); - } else { - const mapped = mapError(result.error); - send({ - data: { error: { code: mapped.code, message: mapped.message } }, - event: "error", - }); - } - end(); - }); - return () => undefined; - }, - }); - }); - - app.openapi({ ...leaseEventsRoute, middleware: [agentAuth] }, async (c) => { - const identity = c.get("identity"); - const lease = await findOwnedLease(deps, buildHttpSession(identity), c.req.valid("param").id); - - return pipeSse( - c, - deps.clock, - { - subscribe(send, end) { - return notices.subscribe(lease.id, (notice) => { - send({ data: notice, event: notice.event }); - if (notice.event === "lease_lost") end(); - }); - }, - }, - identity.expiresAt, - ); - }); - - app.openapi({ ...leaseRotateRoute, middleware: [agentAuth] }, async (c) => { - const { orientation } = c.req.valid("json"); - // why: dispatched directly, like `exec`, so `device.rotate`'s own authorize answers 403 for another requester's lease and the handler 404 for an unknown one. - const rotated = await deps.dispatch( - "device.rotate", - { leaseId: c.req.valid("param").id, orientation }, - buildHttpSession(c.get("identity")), - ); - return c.json(rotated, 200); - }); - - app.openapi({ ...leaseReleaseRoute, middleware: [agentAuth] }, async (c) => { - const identity = c.get("identity"); - const session = buildHttpSession(identity); - const id = c.req.valid("param").id; - - // why: best-effort lookup, only to decorate the response with the device id/state -- - // `lease.release`'s output schema is `{ leaseId }` alone (no `deviceId`), so this is still - // needed for the response body. It is *not* the authorization decision: `lease.release` - // below is dispatched directly and is the sole source of truth for whether this request is - // allowed (S6). Its `ownsLease` hook answers `FORBIDDEN`/403 for someone else's lease and - // the handler answers `UNKNOWN_LEASE`/404 for a nonexistent one, matching the socket - // transport exactly. - const lease = await findLeaseForDecoration(deps, session, id); - - await deps.dispatch("lease.release", { leaseId: id }, session); - - const device = lease === undefined ? undefined : findDevice(deps, lease); - return c.json( - { - device: { id: lease?.deviceId ?? id, state: device?.state ?? "reclaiming" }, - released: true as const, - }, - 202, - ); - }); - - app.openapi({ ...leaseListRoute, middleware: [agentAuth] }, async (c) => - c.json(await deps.dispatch("lease.list", {}, buildHttpSession(c.get("identity"))), 200), - ); + registerLeaseRoutes(app, { agentAuth, deps, notices, tracker }); app.openapi({ ...tokenListRoute, middleware: [agentAuth] }, async (c) => c.json(await deps.dispatch("token.list", {}, buildHttpSession(c.get("identity"))), 200), @@ -912,114 +442,3 @@ export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpApp }, }); } - -// why: no dispatcher operation reads a stored lease request by id, so there is no `authorize` hook to reuse; compared against the record's `ownerId`, the principal the daemon saw send it, never its caller-claimed `requesterId`. -function requireOwnRequest(identity: Identity, ownerId: string): void { - if (identity.role === "operator" || identity.principal === ownerId) return; - throw forbidden("Not permitted to access another requester's resource"); -} - -// why: `lease.get` answers `UNKNOWN_LEASE` for a lease that does not exist and for one the session does not own alike, so the 404 does not tell them apart (the narrower, deliberate divergence from the mutating routes below that the known gaps in src/http/AGENTS.md describe). -async function findOwnedLease( - deps: HttpGatewayDeps, - session: ReturnType, - id: string, -) { - const { lease } = await deps.dispatch("lease.get", { leaseId: id }, session); - return lease; -} - -/** why: Best-effort lease lookup for a lease id, scoped to what the session's `lease.list` would - * show it (own leases; admin sees all) -- used only to decorate a response after an - * authoritative dispatch has already decided whether the request is allowed. Never used to - * gate access itself (see the `DELETE /v1/leases/:id` route). */ -async function findLeaseForDecoration( - deps: HttpGatewayDeps, - session: ReturnType, - id: string, -) { - const { leases } = await deps.dispatch("lease.list", {}, session); - return leases.find((candidate) => candidate.id === id); -} - -function serializeRequest( - view: Pick, -): z.infer { - const base = { createdAt: view.createdAt, id: view.id }; - switch (view.state.stage) { - case "queued": - return { ...base, queuePosition: view.state.queuePosition, state: "queued" }; - case "downloading": { - const { stage: _stage, ...download } = view.state; - return { ...base, ...download, state: "downloading" }; - } - case "provisioning": - case "booting": - case "reclaiming": - return { ...base, etaSeconds: view.state.etaSeconds, state: view.state.stage }; - case "granted": - return { ...base, lease: view.state.lease, state: "granted" }; - case "failed": - return { ...base, error: view.state.error, state: "failed" }; - case "cancelled": - return { ...base, state: "cancelled" }; - } -} - -/** why: The device a lease names. On a gateway both carry the `workerId` they came from, and a - * device matches only on the lease's own worker: a device id is that worker's claim, so another - * worker reporting the same id never answers for it. On a worker neither carries one. */ -function findDevice( - deps: HttpGatewayDeps, - lease: { readonly deviceId: string; readonly workerId?: string | undefined }, -): HttpRegistryDevice | undefined { - return deps.registry.snapshot.devices.find( - (device) => device.id === lease.deviceId && device.workerId === lease.workerId, - ); -} - -// why: an empty body is no body whatever its content type: a renew without one keeps the lease's own width. -async function emptyBodyAsNone(c: Context, next: () => Promise): Promise { - const isJson = /^application\/([a-z-.]+\+)?json/i.test(c.req.header("content-type") ?? ""); - if ((await c.req.text()).trim() === "") { - // why: the validator reads the body only under a JSON content type, so an empty one is relabelled; `bodyCache.text` is typed as the string but holds the pending read, and the JSON validator parses what it holds, so `{}` stands in for the empty text. - if (!isJson) c.req.raw.headers.set("content-type", "application/json"); - Object.assign(c.req.bodyCache, { text: Promise.resolve("{}") }); - } - await next(); -} - -/** why: HTTP-only sugar translating `?since=5m` into the absolute `sinceTs` `events.replay` takes -- - * not a duplicate of any daemon-side logic (the operation itself takes an absolute timestamp), - * so it is not one of the re-implementations ADR §2 says fall away with the dispatcher move. */ -function parseDuration(value: string): number { - const milliseconds = parseDurationMs(value); - if (milliseconds === undefined) throw badRequest(`Invalid duration: ${value}`); - return milliseconds; -} - -function optionalQuery( - value: string | undefined, - name: Name, -): { [Key in Name]?: string } { - return value === undefined ? {} : ({ [name]: value } as { [Key in Name]?: string }); -} - -function statsRefusal(refusal: UsageWindowRefusal): string { - switch (refusal.problem) { - case "since-and-from": - return "stats takes since or from, not both"; - case "to-without-from": - return "to needs from"; - case "not-before-to": - return "from must be earlier than to, or than now when to is left out"; - case "bad-duration": - return `Invalid duration: ${refusal.value}`; - case "bad-time": - return `Invalid time: ${refusal.value} (use ISO 8601 with Z or an offset)`; - } -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} From 2389bfe62f0b11f56f807ef7952c3c090a92c9ba Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:06:09 +0100 Subject: [PATCH 09/83] refactor(core): split registry.ts into parts --- src/core/registry-errors.ts | 20 +++ src/core/registry-records.ts | 100 ++++++++++++++ src/core/registry-types.ts | 120 ++++++++++++++++ src/core/registry.ts | 260 +++++------------------------------ 4 files changed, 275 insertions(+), 225 deletions(-) create mode 100644 src/core/registry-errors.ts create mode 100644 src/core/registry-records.ts create mode 100644 src/core/registry-types.ts diff --git a/src/core/registry-errors.ts b/src/core/registry-errors.ts new file mode 100644 index 000000000..ce6a79ef9 --- /dev/null +++ b/src/core/registry-errors.ts @@ -0,0 +1,20 @@ +export class UnknownDeviceError extends Error { + constructor(readonly deviceId: string) { + super(`Unknown device: ${deviceId}`); + this.name = "UnknownDeviceError"; + } +} + +export class UnknownLeaseError extends Error { + constructor(readonly leaseId: string) { + super(`Unknown lease: ${leaseId}`); + this.name = "UnknownLeaseError"; + } +} + +export class RegistryEventError extends Error { + constructor(message: string) { + super(message); + this.name = "RegistryEventError"; + } +} diff --git a/src/core/registry-records.ts b/src/core/registry-records.ts new file mode 100644 index 000000000..1de18d224 --- /dev/null +++ b/src/core/registry-records.ts @@ -0,0 +1,100 @@ +import type { + DeviceRecord, + DeviceState, + LeaseRecord, + LeaseRequestRecord, + VirtualDevice, +} from "./domain.js"; +import { RegistryEventError } from "./registry-errors.js"; +import type { ComponentRecord, RegistryDeviceEvent } from "./registry-types.js"; + +export function isExpired(device: DeviceRecord, now: number, retentionMs: number): boolean { + return device.deletedAt !== undefined && now - device.deletedAt > retentionMs; +} + +export function componentKey(record: ComponentRecord): string { + return `${record.platform}\0${record.version}`; +} + +/** why: records are immutable, so a record that is not the committed object has changed and one that is not in `after` is gone. */ +export function changes( + before: readonly T[], + after: readonly T[], + keyOf: (record: T) => string, +): { readonly changed: T[]; readonly removed: string[] } { + const committed = new Map(before.map((record) => [keyOf(record), record])); + const kept = new Set(after.map(keyOf)); + return { + changed: after.filter((record) => committed.get(keyOf(record)) !== record), + removed: [...committed.keys()].filter((key) => !kept.has(key)), + }; +} + +export function cloneDevice(device: T): T { + return { ...device, spec: { ...device.spec } } as T; +} + +export function isVirtual(device: DeviceRecord): device is VirtualDevice { + return !device.physical; +} + +/** why: the one place a cause goes when its device leaves quarantine, however it left. */ +export function withoutCauseUnlessQuarantined(device: DeviceRecord): DeviceRecord { + if (device.state === "quarantined" || device.quarantineCause === undefined) return device; + const { quarantineCause: _cause, ...rest } = device; + return rest as DeviceRecord; +} + +export function cloneLease(lease: LeaseRecord): LeaseRecord { + return { ...lease }; +} + +export function cloneComponent(record: ComponentRecord): ComponentRecord { + return { ...record, receipt: { ...record.receipt } }; +} + +export function cloneLeaseRequest(record: LeaseRequestRecord): LeaseRequestRecord { + return { ...record, request: { ...record.request } }; +} + +/** The one device event a transition carries must be the one its states call for, about this device. */ +export function assertTransitionEvent( + device: DeviceRecord, + to: DeviceState, + event: RegistryDeviceEvent | undefined, +): void { + const expectedEvent = eventForTransition(device.state, to); + if (event?.event !== expectedEvent) { + throw new RegistryEventError( + `Transition from ${device.state} to ${to} requires ${expectedEvent ?? "no"} device event`, + ); + } + if (event !== undefined && event.payload.deviceId !== device.id) { + throw new RegistryEventError(`Device event payload does not match device: ${device.id}`); + } +} + +function eventForTransition( + from: DeviceState, + to: DeviceState, +): RegistryDeviceEvent["event"] | undefined { + if (to === "absent") { + return "device.absence-detected"; + } + if (from === "absent" && to === "reclaiming") { + return "device.returned"; + } + if (from === "reclaiming") { + return "device.reclaimed"; + } + if (to === "ready") { + return "device.ready"; + } + if (to === "shutdown") { + return "device.shutdown"; + } + if (to === "deleted") { + return "device.deleted"; + } + return undefined; +} diff --git a/src/core/registry-types.ts b/src/core/registry-types.ts new file mode 100644 index 000000000..2692d3661 --- /dev/null +++ b/src/core/registry-types.ts @@ -0,0 +1,120 @@ +import type { EventBus, EventMap } from "../bus/index.js"; +import type { Clock, IdGenerator } from "../ports/index.js"; +import type { StateDatabase } from "../state/index.js"; +import type { ComponentReceipt, DeviceDataTable } from "./driver.js"; +import type { LeaseRequestLimits } from "./lease-request-store.js"; +import type { + DeviceRecord, + LeaseGrant, + LeaseIdentity, + LeaseRecord, + PhysicalClass, + Platform, + VirtualDevice, + VirtualSpec, +} from "./domain.js"; + +export interface RegistryOptions { + readonly clock: Clock; + readonly idGenerator: IdGenerator; + readonly eventBus: EventBus; + readonly state: StateDatabase; + readonly deviceData: ReadonlyMap; + /** + * `lease.identity`: the policy a newly registered device is stamped with, looked up by its + * spec's platform. Only registration reads it -- a loaded record keeps the policy it was + * created under. Defaults to `reusable` for both platforms. + */ + readonly leaseIdentity?: Readonly>; + /** + * `lease.requestRetentionMs` and `lease.maxRequestRecords`: how long a settled lease request is + * kept and how many are kept at most. Defaults to the config defaults. + */ + readonly leaseRequestLimits?: LeaseRequestLimits; + /** why: `devices.deletedRetentionMs`, how long a `deleted` device record is kept; defaults to the config default. */ + readonly deletedRetentionMs?: number; + /** why: `lease.endedIdRetentionMs`, how long a chosen lease id is remembered after its lease ended; defaults to the config default. */ + readonly endedIdRetentionMs?: number; +} + +export interface RegistrySnapshot { + /** Every device: the records of `devices`, then those of `physicalDevices`. */ + readonly devices: readonly DeviceRecord[]; + readonly virtualDevices: readonly VirtualDevice[]; + readonly leases: readonly LeaseRecord[]; + readonly components: readonly ComponentRecord[]; +} + +/** + * The fourth record type: a component Simlock installed (ADR 0010 §5). `receipt` is the + * driver's own, opaque to the core and compared only for equality -- a component is Simlock's + * when a record's receipt equals the receipt of something installed now. One record per platform + * and version: a later install of the same version replaces it. + */ +export interface ComponentRecord { + readonly platform: Platform; + readonly version: string; + readonly installedAt: number; + readonly receipt: ComponentReceipt; +} + +export interface RegisterDeviceInput { + readonly driverDeviceId: string; + readonly spec: VirtualSpec; + readonly driverData: unknown; + readonly provisionDuration: number; +} + +export interface PhysicalEnrollmentInput { + readonly platform: Platform; + readonly driverDeviceId: string; + readonly model: string; + readonly osVersion: string; + readonly class: PhysicalClass; + readonly apps: readonly string[]; + /** why: who enrolled the phone, for `device.enrolled`; an operator's `device add` unless the worker's auto-enroller says otherwise. */ + readonly initiator?: "operator" | "auto"; +} + +export interface ReleasedLease { + readonly device: DeviceRecord; + readonly lease: LeaseRecord; +} + +export interface CreateLeaseInput { + readonly deviceId: string; + readonly requesterId: string; + readonly ownerId: string; + readonly ttlMs: number; + readonly ttlDeadline: number; + /** why: the caller has already refused an ID that is in use: one place enforces that rule. */ + readonly leaseId?: string; + /** why: the categories the lease keeps running, written to the device in the commit that leases it (ADR 0066 §4). */ + readonly except?: readonly string[]; + /** + * why: when given, the commit that adds the lease also marks the request `granted`, so no crash can + * fall between the two. + */ + readonly request?: { + readonly id: string; + readonly environment: LeaseGrant["environment"]; + readonly timing: LeaseGrant["timing"]; + }; +} + +export type RegistryDeviceEvent = + | { + readonly event: "device.ready"; + readonly payload: Omit; + } + | { readonly event: "device.reclaimed"; readonly payload: EventMap["device.reclaimed"] } + | { readonly event: "device.shutdown"; readonly payload: EventMap["device.shutdown"] } + | { + readonly event: "device.absence-detected"; + readonly payload: EventMap["device.absence-detected"]; + } + | { readonly event: "device.returned"; readonly payload: EventMap["device.returned"] } + | { + readonly event: "device.deleted"; + readonly payload: Omit; + }; diff --git a/src/core/registry.ts b/src/core/registry.ts index 5b941e251..4e4e42322 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -1,6 +1,3 @@ -import type { EventBus, EventMap } from "../bus/index.js"; -import type { Clock, IdGenerator } from "../ports/index.js"; -import type { StateDatabase } from "../state/index.js"; import { DEFAULT_DEVICES_DELETED_RETENTION_MS, DEFAULT_LEASE_ENDED_ID_RETENTION_MS, @@ -16,20 +13,15 @@ import { IllegalTransition, isSettled, type LeaseGrant, - type LeaseIdentity, type LeaseRecord, type LeaseRequestFailure, type LeaseRequestRecord, - type PhysicalClass, type PhysicalDevice, type Platform, transition, type VirtualDevice, - type VirtualSpec, } from "./domain.js"; -import type { ComponentReceipt, DeviceDataTable } from "./driver.js"; import { - type LeaseRequestLimits, type LeaseRequestOutcome, type LeaseRequestStore, type NewLeaseRequest, @@ -53,132 +45,41 @@ import { writeLease, writeLeaseRequest, } from "./registry-rows.js"; - -export interface RegistryOptions { - readonly clock: Clock; - readonly idGenerator: IdGenerator; - readonly eventBus: EventBus; - readonly state: StateDatabase; - readonly deviceData: ReadonlyMap; - /** - * `lease.identity`: the policy a newly registered device is stamped with, looked up by its - * spec's platform. Only registration reads it -- a loaded record keeps the policy it was - * created under. Defaults to `reusable` for both platforms. - */ - readonly leaseIdentity?: Readonly>; - /** - * `lease.requestRetentionMs` and `lease.maxRequestRecords`: how long a settled lease request is - * kept and how many are kept at most. Defaults to the config defaults. - */ - readonly leaseRequestLimits?: LeaseRequestLimits; - /** why: `devices.deletedRetentionMs`, how long a `deleted` device record is kept; defaults to the config default. */ - readonly deletedRetentionMs?: number; - /** why: `lease.endedIdRetentionMs`, how long a chosen lease id is remembered after its lease ended; defaults to the config default. */ - readonly endedIdRetentionMs?: number; -} - -export interface RegistrySnapshot { - /** Every device: the records of `devices`, then those of `physicalDevices`. */ - readonly devices: readonly DeviceRecord[]; - readonly virtualDevices: readonly VirtualDevice[]; - readonly leases: readonly LeaseRecord[]; - readonly components: readonly ComponentRecord[]; -} - -/** - * The fourth record type: a component Simlock installed (ADR 0010 §5). `receipt` is the - * driver's own, opaque to the core and compared only for equality -- a component is Simlock's - * when a record's receipt equals the receipt of something installed now. One record per platform - * and version: a later install of the same version replaces it. - */ -export interface ComponentRecord { - readonly platform: Platform; - readonly version: string; - readonly installedAt: number; - readonly receipt: ComponentReceipt; -} - -export interface RegisterDeviceInput { - readonly driverDeviceId: string; - readonly spec: VirtualSpec; - readonly driverData: unknown; - readonly provisionDuration: number; -} - -export interface PhysicalEnrollmentInput { - readonly platform: Platform; - readonly driverDeviceId: string; - readonly model: string; - readonly osVersion: string; - readonly class: PhysicalClass; - readonly apps: readonly string[]; - /** why: who enrolled the phone, for `device.enrolled`; an operator's `device add` unless the worker's auto-enroller says otherwise. */ - readonly initiator?: "operator" | "auto"; -} - -export interface ReleasedLease { - readonly device: DeviceRecord; - readonly lease: LeaseRecord; -} - -export interface CreateLeaseInput { - readonly deviceId: string; - readonly requesterId: string; - readonly ownerId: string; - readonly ttlMs: number; - readonly ttlDeadline: number; - /** why: the caller has already refused an ID that is in use: one place enforces that rule. */ - readonly leaseId?: string; - /** why: the categories the lease keeps running, written to the device in the commit that leases it (ADR 0066 §4). */ - readonly except?: readonly string[]; - /** - * why: when given, the commit that adds the lease also marks the request `granted`, so no crash can - * fall between the two. - */ - readonly request?: { - readonly id: string; - readonly environment: LeaseGrant["environment"]; - readonly timing: LeaseGrant["timing"]; - }; -} - -export type RegistryDeviceEvent = - | { - readonly event: "device.ready"; - readonly payload: Omit; - } - | { readonly event: "device.reclaimed"; readonly payload: EventMap["device.reclaimed"] } - | { readonly event: "device.shutdown"; readonly payload: EventMap["device.shutdown"] } - | { - readonly event: "device.absence-detected"; - readonly payload: EventMap["device.absence-detected"]; - } - | { readonly event: "device.returned"; readonly payload: EventMap["device.returned"] } - | { - readonly event: "device.deleted"; - readonly payload: Omit; - }; - -export class UnknownDeviceError extends Error { - constructor(readonly deviceId: string) { - super(`Unknown device: ${deviceId}`); - this.name = "UnknownDeviceError"; - } -} - -export class UnknownLeaseError extends Error { - constructor(readonly leaseId: string) { - super(`Unknown lease: ${leaseId}`); - this.name = "UnknownLeaseError"; - } -} - -export class RegistryEventError extends Error { - constructor(message: string) { - super(message); - this.name = "RegistryEventError"; - } -} +import { RegistryEventError, UnknownDeviceError, UnknownLeaseError } from "./registry-errors.js"; +import type { + ComponentRecord, + CreateLeaseInput, + PhysicalEnrollmentInput, + RegisterDeviceInput, + RegistryDeviceEvent, + RegistryOptions, + RegistrySnapshot, + ReleasedLease, +} from "./registry-types.js"; +import { + assertTransitionEvent, + changes, + cloneComponent, + cloneDevice, + cloneLease, + cloneLeaseRequest, + componentKey, + isExpired, + isVirtual, + withoutCauseUnlessQuarantined, +} from "./registry-records.js"; + +export type { + ComponentRecord, + CreateLeaseInput, + PhysicalEnrollmentInput, + RegisterDeviceInput, + RegistryDeviceEvent, + RegistryOptions, + RegistrySnapshot, + ReleasedLease, +} from "./registry-types.js"; +export { RegistryEventError, UnknownDeviceError, UnknownLeaseError } from "./registry-errors.js"; /** * Devices, leases, lease requests (stored before they are queued, see `LeaseRequestBook`) and @@ -1089,94 +990,3 @@ export class Registry implements LeaseRequestStore { } } } - -function isExpired(device: DeviceRecord, now: number, retentionMs: number): boolean { - return device.deletedAt !== undefined && now - device.deletedAt > retentionMs; -} - -function componentKey(record: ComponentRecord): string { - return `${record.platform}\0${record.version}`; -} - -/** why: records are immutable, so a record that is not the committed object has changed and one that is not in `after` is gone. */ -function changes( - before: readonly T[], - after: readonly T[], - keyOf: (record: T) => string, -): { readonly changed: T[]; readonly removed: string[] } { - const committed = new Map(before.map((record) => [keyOf(record), record])); - const kept = new Set(after.map(keyOf)); - return { - changed: after.filter((record) => committed.get(keyOf(record)) !== record), - removed: [...committed.keys()].filter((key) => !kept.has(key)), - }; -} - -function cloneDevice(device: T): T { - return { ...device, spec: { ...device.spec } } as T; -} - -function isVirtual(device: DeviceRecord): device is VirtualDevice { - return !device.physical; -} - -/** why: the one place a cause goes when its device leaves quarantine, however it left. */ -function withoutCauseUnlessQuarantined(device: DeviceRecord): DeviceRecord { - if (device.state === "quarantined" || device.quarantineCause === undefined) return device; - const { quarantineCause: _cause, ...rest } = device; - return rest as DeviceRecord; -} - -function cloneLease(lease: LeaseRecord): LeaseRecord { - return { ...lease }; -} - -function cloneComponent(record: ComponentRecord): ComponentRecord { - return { ...record, receipt: { ...record.receipt } }; -} - -function cloneLeaseRequest(record: LeaseRequestRecord): LeaseRequestRecord { - return { ...record, request: { ...record.request } }; -} - -/** The one device event a transition carries must be the one its states call for, about this device. */ -function assertTransitionEvent( - device: DeviceRecord, - to: DeviceState, - event: RegistryDeviceEvent | undefined, -): void { - const expectedEvent = eventForTransition(device.state, to); - if (event?.event !== expectedEvent) { - throw new RegistryEventError( - `Transition from ${device.state} to ${to} requires ${expectedEvent ?? "no"} device event`, - ); - } - if (event !== undefined && event.payload.deviceId !== device.id) { - throw new RegistryEventError(`Device event payload does not match device: ${device.id}`); - } -} - -function eventForTransition( - from: DeviceState, - to: DeviceState, -): RegistryDeviceEvent["event"] | undefined { - if (to === "absent") { - return "device.absence-detected"; - } - if (from === "absent" && to === "reclaiming") { - return "device.returned"; - } - if (from === "reclaiming") { - return "device.reclaimed"; - } - if (to === "ready") { - return "device.ready"; - } - if (to === "shutdown") { - return "device.shutdown"; - } - if (to === "deleted") { - return "device.deleted"; - } - return undefined; -} From 3c4a3a65a6a59b9be1cd60ea9bd6596e2a8fe29b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:11:02 +0100 Subject: [PATCH 10/83] refactor(simlock-client): split client.ts into parts --- src/simlock-client/client-api.ts | 256 ++++++++ src/simlock-client/client-degraded.ts | 119 ++++ src/simlock-client/client-errors.ts | 23 + src/simlock-client/client-impl.ts | 561 +++++++++++++++++ src/simlock-client/client.ts | 864 +------------------------- 5 files changed, 966 insertions(+), 857 deletions(-) create mode 100644 src/simlock-client/client-api.ts create mode 100644 src/simlock-client/client-degraded.ts create mode 100644 src/simlock-client/client-errors.ts create mode 100644 src/simlock-client/client-impl.ts diff --git a/src/simlock-client/client-api.ts b/src/simlock-client/client-api.ts new file mode 100644 index 000000000..308bdd16c --- /dev/null +++ b/src/simlock-client/client-api.ts @@ -0,0 +1,256 @@ +import type { z } from "zod"; + +import type { AnySimlockError, OperationName, OPERATIONS, Role } from "../contract/index.js"; +import type { LinkChannel } from "../link/index.js"; +import type { CallHooks } from "./wire.js"; +import type { + AlertsTestInput, + AlertsTestOutput, + BaselineListOutput, + BaselineRemoveInput, + BaselineRemoveOutput, + BaselineSaveInput, + BaselineSaveOutput, + CatalogGetInput, + CatalogGetOutput, + CleanupRunInput, + CleanupRunOutput, + ComponentInstallInput, + ComponentInstallOutput, + ComponentListInput, + ComponentListOutput, + ComponentRemoveInput, + ComponentRemoveOutput, + ConfigGetInput, + DaemonStopOutput, + DaemonUpdateInput, + DaemonUpdateOutput, + DeviceAddInput, + DeviceAddOutput, + DeviceRecoveredPush, + DeviceRemoveInput, + DeviceRemoveOutput, + DeviceUnhealthyPush, + DoctorReport, + DoctorRunInput, + DriverPassthroughInput, + EventPush, + EventsReplayInput, + EventsReplayOutput, + ExecInput, + ExecOptions, + ExecOutput, + IdentityOutput, + InstallComponentOnWorkersOptions, + InstallComponentOptions, + LeaseCancelInput, + LeaseCancelOutput, + LeaseGrant, + LeaseListOutput, + LeaseLostPush, + LeaseRecord, + LeaseReleaseAllOutput, + LeaseReleaseInput, + LeaseReleaseOutput, + LeaseRenewInput, + LeaseRequestInput, + ListGetInput, + ListGetOutput, + NukeReport, + NukeRunInput, + PassthroughCommand, + RequestLeaseOptions, + RotateInput, + RotateOutput, + SimlockConfig, + StatusGetOutput, + TokenCreateInput, + TokenCreateOutput, + TokenListOutput, + TokenRevokeInput, + TokenRevokeOutput, + UsageGetInput, + UsageGetOutput, + WarmForgetInput, + WarmForgetOutput, + WorkerDrainInput, + WorkerDrainOutput, + WorkerInstallComponentInput, + WorkerInstallComponentOutput, + WorkerListOutput, + WorkerRemoveOutput, + WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, +} from "./types.js"; + +/** The full agent-visible method surface (ADR §3's operation matrix, `agent` rows) plus the + * pushes and lifecycle every frontend needs. `simlock/admin`'s `SimlockAdminClient` extends + * this with the `admin` rows. */ +export interface SimlockClient { + readonly principal: string; + readonly role: Role; + readonly daemonVersion: string; + + getCatalog(input?: CatalogGetInput): Promise; + getStatus(): Promise; + identity(): Promise; + /** + * ADR 0010 §8: every installed iOS simulator runtime and Android system image on the daemon's + * machine, with its size, whether Simlock installed it, and how many devices use it. + */ + listComponents(input?: ComponentListInput): Promise; + requestLease(input: LeaseRequestInput, options?: RequestLeaseOptions): Promise; + cancelLease(input?: LeaseCancelInput): Promise; + renewLease(input: LeaseRenewInput): Promise; + releaseLease(input: LeaseReleaseInput): Promise; + listLeases(): Promise; + runDoctor(input?: DoctorRunInput): Promise; + /** Resolves the scoped command behind `simlock simctl` / `simlock adb` (ADR 0001, decision + * 7). Resolution only -- the caller is the process with a terminal, so it runs it. */ + resolvePassthrough(input: DriverPassthroughInput): Promise; + /** + * why: runs `simctl`, `adb` or `devicectl` **on the daemon's machine** against a device this + * client's principal leases, streaming the output to `options.onOutput` as it arrives and resolving with the + * command's exit code (ADR 0005 §19a). The counterpart to `resolvePassthrough`: that one + * hands back a command line for a caller sharing the machine, this one is for a caller that + * does not. `stdin` is written once and closed -- there is no pseudo-terminal, so + * line-oriented commands work and full-screen ones do not (§19c). + */ + exec(input: ExecInput, options?: ExecOptions): Promise; + /** + * why: turns the lease's iOS phone or tablet simulator to `orientation`, whether or not the daemon's input module is on. `requesterId` is for an operator that acts for the lease's requester, as `exec`'s is. + */ + rotate( + leaseId: string, + orientation: RotateInput["orientation"], + options?: { readonly requesterId?: string }, + ): Promise; + + /** + * Reports a lease the *daemon* ended -- an expiry, an operator release, an unrecoverable + * device. ADR 0004 §3 narrows ADR 0003 §10 here: a dead connection is not a dead lease, so + * this no longer fires with a synthesized `daemon-connection-lost` when the socket dies. The + * leases this client held are still granted and still counting down; `onConnectionLost` is + * what reports the connection, and reconnecting and renewing is the frontend's call. + */ + onLeaseLost(listener: (push: LeaseLostPush) => void): () => void; + onDeviceUnhealthy(listener: (push: DeviceUnhealthyPush) => void): () => void; + onDeviceRecovered(listener: (push: DeviceRecoveredPush) => void): () => void; + /** Fires exactly once, when the connection dies for any reason (including a deliberate + * `close()`). Every in-flight call has already rejected `DAEMON_CONNECTION_LOST` by the time + * this listener runs. Any lease this client was holding is still alive on the daemon. */ + onConnectionLost(listener: (error: AnySimlockError) => void): () => void; + + close(): Promise; +} + +/** `simlock/admin`'s extension: the `admin` rows of ADR §3's matrix. */ +export interface SimlockAdminClient extends SimlockClient { + releaseAllLeases(): Promise; + list(input?: ListGetInput): Promise; + runCleanup(input?: CleanupRunInput): Promise; + runNuke(input?: NukeRunInput): Promise; + forgetWarmKind(input: WarmForgetInput): Promise; + getConfig(input?: ConfigGetInput): Promise; + /** ADR §6's frozen exception: accepted by the daemon at any protocol version it has ever + * spoken, even before `hello`/role resolve. Exposed here rather than on the base client + * purely as an API-surface choice matching the ADR's operation matrix (the daemon itself + * does not gate this operation by role at all -- see the caveat in the PR report). */ + stopDaemon(): Promise; + updateDaemon(input?: DaemonUpdateInput): Promise; + replayEvents(input?: EventsReplayInput): Promise; + usage(window: UsageGetInput): Promise; + subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise>; + createToken(input: TokenCreateInput): Promise; + listTokens(): Promise; + revokeToken(input: TokenRevokeInput): Promise; + + /** why: ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker`, `removeWorker` and + * `updateWorker` (a gateway updating one of its workers, ADR 0042 §2) reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ + listWorkers(): Promise; + drainWorker(input: WorkerDrainInput): Promise; + undrainWorker(input: WorkerDrainInput): Promise; + removeWorker(input: WorkerDrainInput): Promise; + updateWorker(input: WorkerUpdateInput): Promise; + + /** + * why: ADR 0035 §1: asks a gateway to post one sample message to the named alert destination, once. + * Rejects with `ALERT_SEND_FAILED` when the receiver does not take it, `UNKNOWN_ALERT_DESTINATION` + * for a name the gateway has not configured, and on a worker with `UNSUPPORTED_IN_WORKER_MODE`. + */ + testAlertDestination(input: AlertsTestInput): Promise; + + /** + * ADR 0010 §6: installs one iOS simulator runtime or Android system image, with no lease. + * Resolves once it is installed or found already there; `options.onProgress` hears `waiting` + * while another install on the platform runs ahead, then `downloading`. Refused with + * `DOWNLOADS_DISABLED` under `downloads.policy: "never"`. + */ + installComponent( + input: ComponentInstallInput, + options?: InstallComponentOptions, + ): Promise; + + /** + * ADR 0010 §7, gateway-only: installs one component on named workers (`workers: [ids]`) or on + * every connected one (`workers: "all"`). Every targeted worker is asked at the same time and + * answers for itself; the result lists one entry per worker. `options.onProgress` hears each + * worker's progress with its `workerId`. A worker answers `UNSUPPORTED_IN_WORKER_MODE`. + */ + installComponentOnWorkers( + input: WorkerInstallComponentInput, + options?: InstallComponentOnWorkersOptions, + ): Promise; + + /** + * ADR 0010 §8: removes one component Simlock installed, or with `dryRun: true` runs every + * check and removes nothing. Refused with `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE` (with the + * counts as details) or `COMPONENT_BUSY`. + */ + removeComponent(input: ComponentRemoveInput): Promise; + + /** why: ADR 0061 §4: ends the held lease and keeps its device as a named baseline; admin only. */ + saveBaseline(input: BaselineSaveInput): Promise; + + /** why: the newest version of each baseline on the worker, with its stale reason. */ + listBaselines(): Promise; + + /** why: deletes every version of a baseline and the idle devices made from it; refused while a lease uses it. */ + removeBaseline(input: BaselineRemoveInput): Promise; + + /** + * Enrols the physical device plugged into the host this daemon runs on, named by + * any ID the platform's driver reads. A device it cannot enrol is `DEVICE_NOT_ENROLLABLE` with + * a reason; a leased, reclaiming or busy one is `DEVICE_BUSY`. A device already + * enrolled is enrolled again: its record is kept and its model, OS version, class and apps are + * read anew (`reenrolled: true`). A gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`. + */ + addDevice(input: DeviceAddInput): Promise; + + /** + * Forgets an enrolled physical device, named by its canonical ID. Nothing is sent + * to the device. `UNKNOWN_DEVICE` for an ID with no enrolled device, `DEVICE_BUSY` for a leased, + * reclaiming or busy one (another operation holds it). A gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`. + */ + removeDevice(input: DeviceRemoveInput): Promise; +} + +export type OperationInput = z.infer< + (typeof OPERATIONS)[Name]["input"] +>; +export type OperationOutput = z.infer< + (typeof OPERATIONS)[Name]["output"] +>; + +// why: not part of `SimlockAdminClient`, so `simlock/admin` does not export it (ADR 0026). + +export interface RelayClientMethods { + relayRequest( + input: OperationInput<"relay.request">, + hooks: Pick, + ): Promise>; + // why: rejects on a connection that holds no link, since a channel is a link's (ADR 0028). + openChannel(options: { readonly signal: AbortSignal }): Promise; + relayCancel(input: OperationInput<"relay.cancel">): Promise>; +} diff --git a/src/simlock-client/client-degraded.ts b/src/simlock-client/client-degraded.ts new file mode 100644 index 000000000..9737e6bd7 --- /dev/null +++ b/src/simlock-client/client-degraded.ts @@ -0,0 +1,119 @@ +import { OPERATIONS, SimlockError } from "../contract/index.js"; +import type { SimlockWire } from "./wire.js"; +import type { RelayClientMethods, SimlockAdminClient, SimlockClient } from "./client-api.js"; +import { toSimlockError } from "./client-errors.js"; +import type { DaemonStopOutput, DaemonUpdateInput, DaemonUpdateOutput } from "./types.js"; + +/** + * why: ADR §6's escape hatch on the client side. A mismatch leaves the connection open so an admin can still send the + * two frozen calls, `stopDaemon()` and `updateDaemon()` (the daemon answers them at any protocol version, ahead of its + * mismatch gate); everything else rejects with the captured `PROTOCOL_VERSION_UNSUPPORTED` error without being sent, + * and `close()` still works. `role` reflects which entry point the caller used and `principal` what it supplied, + * because the daemon resolves them but the failed `hello` never reports them back; the daemon's own role check on the + * credential it already verified is what gates the two calls. + */ +export function buildDegradedClient( + wire: SimlockWire, + error: SimlockError<"PROTOCOL_VERSION_UNSUPPORTED">, + principal: string, + admin: boolean, +): SimlockClient | SimlockAdminClient { + const rejected = (): Promise => Promise.reject(error); + const client: SimlockAdminClient & RelayClientMethods = { + principal, + role: admin ? "admin" : "agent", + daemonVersion: error.details.daemonVersion, + + getCatalog: () => rejected(), + getStatus: () => rejected(), + identity: () => rejected(), + listComponents: () => rejected(), + requestLease: () => rejected(), + cancelLease: () => rejected(), + renewLease: () => rejected(), + releaseLease: () => rejected(), + listLeases: () => rejected(), + runDoctor: () => rejected(), + resolvePassthrough: () => rejected(), + exec: () => rejected(), + rotate: () => rejected(), + + onLeaseLost: () => () => {}, + onDeviceUnhealthy: () => () => {}, + onDeviceRecovered: () => () => {}, + // The one live fact this connection can still report: it can still die later even though + // `hello` never fully succeeded (e.g. the daemon exits while this degraded client sits + // idle). + onConnectionLost: (listener) => wire.onDeath(listener), + + close: () => wire.close(), + + releaseAllLeases: () => rejected(), + list: () => rejected(), + runCleanup: () => rejected(), + runNuke: () => rejected(), + forgetWarmKind: () => rejected(), + getConfig: () => rejected(), + stopDaemon: () => frozenCall(wire, "daemon.stop", {}), + updateDaemon: (input = {}) => frozenCall(wire, "daemon.update", input), + replayEvents: () => rejected(), + usage: () => rejected(), + subscribeEvents: () => rejected(), + createToken: () => rejected(), + listTokens: () => rejected(), + revokeToken: () => rejected(), + listWorkers: () => rejected(), + drainWorker: () => rejected(), + undrainWorker: () => rejected(), + removeWorker: () => rejected(), + updateWorker: () => rejected(), + testAlertDestination: () => rejected(), + installComponent: () => rejected(), + installComponentOnWorkers: () => rejected(), + removeComponent: () => rejected(), + saveBaseline: () => rejected(), + listBaselines: () => rejected(), + removeBaseline: () => rejected(), + addDevice: () => rejected(), + removeDevice: () => rejected(), + relayRequest: () => rejected(), + openChannel: () => rejected(), + relayCancel: () => rejected(), + }; + return client; +} + +// why: the two frozen calls (`daemon.stop`, `daemon.update`) are the only ones a degraded client sends, and the daemon answers them at any protocol version. +function frozenCall( + wire: SimlockWire, + operation: "daemon.stop", + input: Record, +): Promise; +function frozenCall( + wire: SimlockWire, + operation: "daemon.update", + input: DaemonUpdateInput, +): Promise; +function frozenCall( + wire: SimlockWire, + operation: "daemon.stop" | "daemon.update", + input: unknown, +): Promise { + return wire.call(operation, input).then( + (payload) => { + const result = OPERATIONS[operation].output.safeParse(payload); + if (!result.success) { + throw new SimlockError( + "BAD_FRAME", + "protocol", + `Daemon's ${operation} response did not match the contract output schema`, + {}, + ); + } + return result.data; + }, + (callError: unknown) => { + throw toSimlockError(callError); + }, + ); +} diff --git a/src/simlock-client/client-errors.ts b/src/simlock-client/client-errors.ts new file mode 100644 index 000000000..cedeb4213 --- /dev/null +++ b/src/simlock-client/client-errors.ts @@ -0,0 +1,23 @@ +import { + type AnySimlockError, + fromWireError, + isSimlockError, + SimlockError, +} from "../contract/index.js"; +import { WireCallError } from "./wire.js"; + +export function cancelledError(): SimlockError<"CANCELLED"> { + return new SimlockError("CANCELLED", "domain", "Lease request was cancelled", {}); +} + +export function toSimlockError(error: unknown): AnySimlockError { + if (isSimlockError(error)) return error; + if (error instanceof WireCallError) + return fromWireError(error.code, error.message, error.details); + return new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + error instanceof Error ? error.message : String(error), + {}, + ); +} diff --git a/src/simlock-client/client-impl.ts b/src/simlock-client/client-impl.ts new file mode 100644 index 000000000..10c2a4119 --- /dev/null +++ b/src/simlock-client/client-impl.ts @@ -0,0 +1,561 @@ +import type { z } from "zod"; + +import { + type AnySimlockError, + describeSchemaIssues, + isSimlockError, + type OperationName, + OPERATIONS, + type Role, + SimlockError, +} from "../contract/index.js"; +import type { LinkChannel, LinkConnection } from "../link/index.js"; +import type { CallHooks, LeaseScopedPush, SimlockWire } from "./wire.js"; +import type { OperationInput, OperationOutput, SimlockAdminClient } from "./client-api.js"; +import { cancelledError, toSimlockError } from "./client-errors.js"; +import type { + AlertsTestInput, + AlertsTestOutput, + BaselineListOutput, + BaselineRemoveInput, + BaselineRemoveOutput, + BaselineSaveInput, + BaselineSaveOutput, + CatalogGetInput, + CatalogGetOutput, + CleanupRunInput, + CleanupRunOutput, + ComponentInstallInput, + ComponentInstallOutput, + ComponentListInput, + ComponentListOutput, + ComponentRemoveInput, + ComponentRemoveOutput, + ConfigGetInput, + DaemonStopOutput, + DaemonUpdateInput, + DaemonUpdateOutput, + DeviceAddInput, + DeviceAddOutput, + DeviceRecoveredPush, + DeviceRemoveInput, + DeviceRemoveOutput, + DeviceUnhealthyPush, + DoctorReport, + DoctorRunInput, + DriverPassthroughInput, + EventPush, + EventsReplayInput, + EventsReplayOutput, + ExecInput, + ExecOptions, + ExecOutput, + IdentityOutput, + InstallComponentOnWorkersOptions, + InstallComponentOptions, + LeaseCancelInput, + LeaseCancelOutput, + LeaseGrant, + LeaseListOutput, + LeaseLostPush, + LeaseRecord, + LeaseReleaseAllOutput, + LeaseReleaseInput, + LeaseReleaseOutput, + LeaseRenewInput, + LeaseRequestInput, + ListGetInput, + ListGetOutput, + NukeReport, + NukeRunInput, + PassthroughCommand, + RequestLeaseOptions, + RotateInput, + RotateOutput, + SimlockConfig, + StatusGetOutput, + TokenCreateInput, + TokenCreateOutput, + TokenListOutput, + TokenRevokeInput, + TokenRevokeOutput, + UsageGetInput, + UsageGetOutput, + WarmForgetInput, + WarmForgetOutput, + WorkerDrainInput, + WorkerDrainOutput, + WorkerInstallComponentInput, + WorkerInstallComponentOutput, + WorkerListOutput, + WorkerRemoveOutput, + WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, +} from "./types.js"; + +export class SimlockClientImpl { + readonly #wire: SimlockWire; + readonly #link: LinkConnection | undefined; + readonly #leaseLostListeners = new Set<(push: LeaseLostPush) => void>(); + readonly #deviceUnhealthyListeners = new Set<(push: DeviceUnhealthyPush) => void>(); + readonly #deviceRecoveredListeners = new Set<(push: DeviceRecoveredPush) => void>(); + readonly #connectionLostListeners = new Set<(error: AnySimlockError) => void>(); + #eventSubscriptionId: string | undefined; + + constructor( + wire: SimlockWire, + readonly role: Role, + readonly principal: string, + readonly daemonVersion: string, + link?: LinkConnection, + ) { + this.#wire = wire; + this.#link = link; + wire.onLeaseScopedPush((push) => this.#handleLeaseScopedPush(push)); + wire.onDeath((error) => this.#handleConnectionLost(error)); + } + + asAdmin(): SimlockAdminClient { + // `this` already implements every method the interface below needs (see the class body) -- + // this cast documents that the split is purely at the type layer. + return this as unknown as SimlockAdminClient; + } + + // ---- agent-visible operations -------------------------------------------------------------- + + getCatalog(input: CatalogGetInput = {}): Promise { + return this.#call("catalog.get", input); + } + + getStatus(): Promise { + return this.#call("status.get", {}); + } + + identity(): Promise { + return this.#call("identity.get", {}); + } + + listComponents(input: ComponentListInput = {}): Promise { + return this.#call("component.list", input); + } + + async requestLease( + input: LeaseRequestInput, + options: RequestLeaseOptions = {}, + ): Promise { + const parsed = this.#parseInput("lease.request", input); + const requesterId = parsed.requesterId ?? this.principal; + const { signal, onProgress } = options; + + if (signal?.aborted === true) throw cancelledError(); + + const requestPromise = this.#callRaw("lease.request", parsed, { onProgress }).then((grant) => + this.#parseOutput("lease.request", grant), + ); + + if (signal === undefined) return requestPromise; + return this.#requestLeaseWithAbort(requestPromise, signal, { + platform: parsed.platform, + requesterId, + }); + } + + cancelLease(input: LeaseCancelInput = {}): Promise { + return this.#call("lease.cancel", input); + } + + renewLease(input: LeaseRenewInput): Promise { + return this.#call("lease.renew", input); + } + + async releaseLease(input: LeaseReleaseInput): Promise { + const parsed = this.#parseInput("lease.release", input); + // The daemon suppresses the matching `lease-lost` push to this connection itself (ADR + // §8) -- see the comment on `SimlockWire`'s `#deliveredLeaseLost`. Nothing to mark here. + return this.#call("lease.release", parsed); + } + + listLeases(): Promise { + return this.#call("lease.list", {}); + } + + runDoctor(input: DoctorRunInput = {}): Promise { + return this.#call("doctor.run", input); + } + + resolvePassthrough(input: DriverPassthroughInput): Promise { + return this.#call("driver.passthrough", input); + } + + async exec(input: ExecInput, options: ExecOptions = {}): Promise { + const payload = await this.#callRaw("device.exec", this.#parseInput("device.exec", input), { + onOutput: options.onOutput, + onStarted: options.onStarted, + }); + return this.#parseOutput("device.exec", payload); + } + + rotate( + leaseId: string, + orientation: RotateInput["orientation"], + options: { readonly requesterId?: string } = {}, + ): Promise { + return this.#call("device.rotate", { leaseId, orientation, ...options }); + } + + onLeaseLost(listener: (push: LeaseLostPush) => void): () => void { + this.#leaseLostListeners.add(listener); + return () => this.#leaseLostListeners.delete(listener); + } + + onDeviceUnhealthy(listener: (push: DeviceUnhealthyPush) => void): () => void { + this.#deviceUnhealthyListeners.add(listener); + return () => this.#deviceUnhealthyListeners.delete(listener); + } + + onDeviceRecovered(listener: (push: DeviceRecoveredPush) => void): () => void { + this.#deviceRecoveredListeners.add(listener); + return () => this.#deviceRecoveredListeners.delete(listener); + } + + onConnectionLost(listener: (error: AnySimlockError) => void): () => void { + this.#connectionLostListeners.add(listener); + return () => this.#connectionLostListeners.delete(listener); + } + + async close(): Promise { + await this.#wire.close(); + } + + async relayRequest( + input: OperationInput<"relay.request">, + hooks: Pick, + ): Promise> { + const payload = await this.#callRaw("relay.request", input, hooks); + return this.#parseOutput("relay.request", payload); + } + + openChannel(options: { readonly signal: AbortSignal }): Promise { + const link = this.#link; + if (link === undefined) return Promise.reject(new Error("The connection carries no link")); + return link.openChannel(options); + } + + relayCancel(input: OperationInput<"relay.cancel">): Promise> { + return this.#call("relay.cancel", input); + } + + // ---- admin-only operations ------------------------------------------------------------------ + + releaseAllLeases(): Promise { + return this.#call("lease.release-all", {}); + } + + list(input: ListGetInput = {}): Promise { + return this.#call("list.get", input); + } + + runCleanup(input: CleanupRunInput = {}): Promise { + return this.#call("cleanup.run", input); + } + + runNuke(input: NukeRunInput = {}): Promise { + return this.#call("nuke.run", input); + } + + forgetWarmKind(input: WarmForgetInput): Promise { + return this.#call("warm.forget", input); + } + + getConfig(input: ConfigGetInput = {}): Promise { + return this.#call("config.get", input); + } + + stopDaemon(): Promise { + return this.#call("daemon.stop", {}); + } + + updateDaemon(input: DaemonUpdateInput = {}): Promise { + return this.#call("daemon.update", input); + } + + replayEvents(input: EventsReplayInput = {}): Promise { + return this.#call("events.replay", input); + } + + usage(window: UsageGetInput): Promise { + return this.#call("usage.get", window); + } + + async subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise> { + const unsubscribePush = this.#wire.onEvent((payload) => listener(payload as EventPush)); + const result = await this.#call("events.subscribe", {}); + this.#eventSubscriptionId = result.subscriptionId; + return async () => { + unsubscribePush(); + if (this.#eventSubscriptionId === undefined) return; + this.#eventSubscriptionId = undefined; + await this.#call("events.unsubscribe", {}); + }; + } + + createToken(input: TokenCreateInput): Promise { + return this.#call("token.create", input); + } + + listTokens(): Promise { + return this.#call("token.list", {}); + } + + revokeToken(input: TokenRevokeInput): Promise { + return this.#call("token.revoke", input); + } + + listWorkers(): Promise { + return this.#call("worker.list", {}); + } + + drainWorker(input: WorkerDrainInput): Promise { + return this.#call("worker.drain", input); + } + + undrainWorker(input: WorkerDrainInput): Promise { + return this.#call("worker.undrain", input); + } + + updateWorker(input: WorkerUpdateInput): Promise { + return this.#call("worker.update", input); + } + + testAlertDestination(input: AlertsTestInput): Promise { + return this.#call("alerts.test", input); + } + + async installComponent( + input: ComponentInstallInput, + options: InstallComponentOptions = {}, + ): Promise { + const payload = await this.#callRaw( + "component.install", + this.#parseInput("component.install", input), + { onComponentProgress: options.onProgress }, + ); + return this.#parseOutput("component.install", payload); + } + + async installComponentOnWorkers( + input: WorkerInstallComponentInput, + options: InstallComponentOnWorkersOptions = {}, + ): Promise { + const { onProgress } = options; + const payload = await this.#callRaw( + "worker.install-component", + this.#parseInput("worker.install-component", input), + { + // A gateway always names the worker; an update that names none cannot be attributed + // to one, so it is dropped rather than guessed. + onComponentProgress: + onProgress === undefined + ? undefined + : (progress, workerId) => { + if (workerId !== undefined) onProgress({ ...progress, workerId }); + }, + }, + ); + return this.#parseOutput("worker.install-component", payload); + } + + removeWorker(input: WorkerDrainInput): Promise { + return this.#call("worker.remove", input); + } + + removeComponent(input: ComponentRemoveInput): Promise { + return this.#call("component.remove", input); + } + + saveBaseline(input: BaselineSaveInput): Promise { + return this.#call("baseline.save", input); + } + + listBaselines(): Promise { + return this.#call("baseline.list", {}); + } + + removeBaseline(input: BaselineRemoveInput): Promise { + return this.#call("baseline.remove", input); + } + + addDevice(input: DeviceAddInput): Promise { + return this.#call("device.add", input); + } + + removeDevice(input: DeviceRemoveInput): Promise { + return this.#call("device.remove", input); + } + + #requestLeaseWithAbort( + requestPromise: Promise, + signal: AbortSignal, + cancel: LeaseCancelInput, + ): Promise { + return new Promise((resolve, reject) => { + let requestDone = false; + let aborted = false; + + const onAbort = (): void => { + if (requestDone) return; // after the grant resolved (or it already failed) -> ignored. + aborted = true; + void this.#cancelOnAbort(requestPromise, cancel).then(resolve, reject); + }; + signal.addEventListener("abort", onAbort, { once: true }); + + requestPromise.then( + (grant) => { + requestDone = true; + signal.removeEventListener("abort", onAbort); + if (!aborted) resolve(grant); + }, + (error: unknown) => { + requestDone = true; + signal.removeEventListener("abort", onAbort); + if (!aborted) reject(error as Error); + }, + ); + }); + } + + async #cancelOnAbort( + requestPromise: Promise, + cancel: LeaseCancelInput, + ): Promise { + let outcome: LeaseCancelOutput["result"]; + try { + outcome = (await this.#call("lease.cancel", cancel)).result; + } catch (error: unknown) { + // Only a connection that actually died mid-cancel falls through to let awaiting + // `requestPromise` below surface whatever that produces (most likely + // `DAEMON_CONNECTION_LOST`, a more accurate rejection than a manufactured `CANCELLED` + // would be here). Every other rejection -- `FORBIDDEN`, `BAD_REQUEST` -- is a real + // answer from the daemon and must surface as-is: this `catch` used to swallow those + // unconditionally, which is what let an aborted `requestLease` silently resolve with a + // real grant instead of `CANCELLED` (see the PR report's B3). + if (isSimlockError(error) && error.kind === "transport") { + return requestPromise; + } + throw error; + } + + if (outcome === "not-found") { + // Either genuinely never existed, or -- per ADR §10's fourth case -- the grant already + // resolved before the cancel reached the daemon: "after the grant resolved -> ignored". + // Both cases defer entirely to the original outcome. + return requestPromise; + } + + if (outcome === "cancelled") { + // Queued: the daemon rejects the original waiter; wait for that, then surface CANCELLED + // regardless of the rejection's own code/message. + await requestPromise.catch(() => undefined); + throw cancelledError(); + } + + // "not-cancellable": device work already in flight. Wait for the outcome; a grant that + // still arrives is abandoned immediately so the caller never holds a lease it walked away + // from, then CANCELLED is surfaced either way. + try { + const grant = await requestPromise; + await this.#releaseAbandonedGrant(grant); + } catch { + // The request failed on its own (e.g. ran out of capacity mid-flight) -- nothing to + // release, and CANCELLED is still the right thing to tell a caller who asked to abort. + } + throw cancelledError(); + } + + async #releaseAbandonedGrant(grant: LeaseGrant): Promise { + try { + await this.releaseLease({ leaseId: grant.lease.id }); + } catch { + // Best-effort: the caller already gets CANCELLED either way. A failed release here means + // the daemon may still show the lease held -- see the PR report's weak-spots list. + } + } + + // ---- pushes ----------------------------------------------------------------------------- + + #handleLeaseScopedPush(push: LeaseScopedPush): void { + switch (push.kind) { + case "lease-lost": + for (const listener of this.#leaseLostListeners) { + listener({ deviceId: push.deviceId, leaseId: push.leaseId, reason: push.reason }); + } + return; + case "device-unhealthy": + for (const listener of this.#deviceUnhealthyListeners) { + listener({ deviceId: push.deviceId, leaseId: push.leaseId }); + } + return; + case "device-recovered": + for (const listener of this.#deviceRecoveredListeners) { + listener({ attempts: push.attempts, deviceId: push.deviceId, leaseId: push.leaseId }); + } + return; + } + } + + /** + * ADR 0004 §3: a dead connection ends no lease, so nothing is synthesized here. The daemon + * releases nothing on close, so every lease this client was holding is still granted, still + * counting down its own TTL, and still renewable by whatever connects next -- which is the + * frontend's decision to make, not this client's (ADR 0003 §10). + */ + #handleConnectionLost(error: AnySimlockError): void { + for (const listener of this.#connectionLostListeners) listener(error); + } + + // ---- call plumbing -------------------------------------------------------------------------- + + #parseInput( + name: Name, + input: unknown, + ): z.infer<(typeof OPERATIONS)[Name]["input"]> { + const result = OPERATIONS[name].input.safeParse(input); + if (result.success) return result.data as z.infer<(typeof OPERATIONS)[Name]["input"]>; + throw new SimlockError( + "BAD_REQUEST", + "protocol", + describeSchemaIssues(result.error.issues), + {}, + ); + } + + #parseOutput( + name: Name, + payload: unknown, + ): z.infer<(typeof OPERATIONS)[Name]["output"]> { + const result = OPERATIONS[name].output.safeParse(payload); + if (result.success) return result.data as z.infer<(typeof OPERATIONS)[Name]["output"]>; + throw new SimlockError( + "BAD_FRAME", + "protocol", + `Daemon's ${name} response did not match the contract output schema`, + {}, + ); + } + + async #call( + name: Name, + input: z.infer<(typeof OPERATIONS)[Name]["input"]>, + ): Promise> { + const payload = await this.#callRaw(name, input); + return this.#parseOutput(name, payload); + } + + #callRaw(name: OperationName, input: unknown, hooks: CallHooks = {}): Promise { + const parsed = this.#parseInput(name, input); + return this.#wire.call(name, parsed, hooks).catch((error: unknown) => { + throw toSimlockError(error); + }); + } +} diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index 0e5bfa608..00d4e2797 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -12,101 +12,13 @@ * whole second policy (and would make "reconnecting never implicitly acquires a device" a * claim this module could no longer prove). */ -import type { z } from "zod"; - -import { - describeSchemaIssues, - fromWireError, - isSimlockError, - OPERATIONS, - SimlockError, - type AnySimlockError, - type OperationName, - type Role, -} from "../contract/index.js"; -import type { LinkChannel, LinkConnection } from "../link/index.js"; +import { isSimlockError } from "../contract/index.js"; +import type { LinkConnection } from "../link/index.js"; import type { IpcConnection, IpcConnector } from "../ports/index.js"; -import { SimlockWire, WireCallError, type CallHooks, type LeaseScopedPush } from "./wire.js"; -import type { - AlertsTestInput, - AlertsTestOutput, - CatalogGetInput, - CatalogGetOutput, - CleanupRunInput, - BaselineListOutput, - BaselineRemoveInput, - BaselineRemoveOutput, - BaselineSaveInput, - BaselineSaveOutput, - CleanupRunOutput, - ComponentInstallInput, - ComponentInstallOutput, - ComponentListInput, - ComponentListOutput, - ComponentRemoveInput, - ComponentRemoveOutput, - DaemonStopOutput, - DaemonUpdateInput, - DaemonUpdateOutput, - DeviceAddInput, - DeviceAddOutput, - DeviceRecoveredPush, - DeviceRemoveInput, - DeviceRemoveOutput, - DeviceUnhealthyPush, - ExecInput, - ExecOutput, - RotateInput, - RotateOutput, - DoctorReport, - DoctorRunInput, - DriverPassthroughInput, - EventPush, - ExecOptions, - EventsReplayInput, - EventsReplayOutput, - UsageGetInput, - UsageGetOutput, - InstallComponentOnWorkersOptions, - IdentityOutput, - InstallComponentOptions, - LeaseCancelInput, - LeaseCancelOutput, - PassthroughCommand, - LeaseGrant, - LeaseListOutput, - LeaseLostPush, - LeaseReleaseAllOutput, - LeaseReleaseInput, - LeaseReleaseOutput, - LeaseRenewInput, - LeaseRecord, - LeaseRequestInput, - ListGetInput, - ListGetOutput, - NukeReport, - NukeRunInput, - ConfigGetInput, - WarmForgetInput, - WarmForgetOutput, - RequestLeaseOptions, - SimlockConfig, - StatusGetOutput, - TokenCreateInput, - TokenCreateOutput, - TokenListOutput, - TokenRevokeInput, - TokenRevokeOutput, - WorkerDrainInput, - WorkerDrainOutput, - WorkerInstallComponentInput, - WorkerInstallComponentOutput, - WorkerListOutput, - WorkerRemoveOutput, - WorkerUndrainOutput, - WorkerUpdateInput, - WorkerUpdateOutput, -} from "./types.js"; +import { SimlockWire } from "./wire.js"; +import type { SimlockAdminClient, SimlockClient } from "./client-api.js"; +import { buildDegradedClient } from "./client-degraded.js"; +import { SimlockClientImpl } from "./client-impl.js"; export type { BaselineListOutput, @@ -200,6 +112,7 @@ export { type AnySimlockError, type SimlockErrorCode, } from "../contract/index.js"; +export type { RelayClientMethods, SimlockAdminClient, SimlockClient } from "./client-api.js"; export interface ConnectOptions { /** Pre-connected transport (a scripted `IpcConnection` in tests, or the real result of an @@ -221,173 +134,6 @@ export interface ConnectOptions { readonly maxLineChars?: number; } -/** The full agent-visible method surface (ADR §3's operation matrix, `agent` rows) plus the - * pushes and lifecycle every frontend needs. `simlock/admin`'s `SimlockAdminClient` extends - * this with the `admin` rows. */ -export interface SimlockClient { - readonly principal: string; - readonly role: Role; - readonly daemonVersion: string; - - getCatalog(input?: CatalogGetInput): Promise; - getStatus(): Promise; - identity(): Promise; - /** - * ADR 0010 §8: every installed iOS simulator runtime and Android system image on the daemon's - * machine, with its size, whether Simlock installed it, and how many devices use it. - */ - listComponents(input?: ComponentListInput): Promise; - requestLease(input: LeaseRequestInput, options?: RequestLeaseOptions): Promise; - cancelLease(input?: LeaseCancelInput): Promise; - renewLease(input: LeaseRenewInput): Promise; - releaseLease(input: LeaseReleaseInput): Promise; - listLeases(): Promise; - runDoctor(input?: DoctorRunInput): Promise; - /** Resolves the scoped command behind `simlock simctl` / `simlock adb` (ADR 0001, decision - * 7). Resolution only -- the caller is the process with a terminal, so it runs it. */ - resolvePassthrough(input: DriverPassthroughInput): Promise; - /** - * why: runs `simctl`, `adb` or `devicectl` **on the daemon's machine** against a device this - * client's principal leases, streaming the output to `options.onOutput` as it arrives and resolving with the - * command's exit code (ADR 0005 §19a). The counterpart to `resolvePassthrough`: that one - * hands back a command line for a caller sharing the machine, this one is for a caller that - * does not. `stdin` is written once and closed -- there is no pseudo-terminal, so - * line-oriented commands work and full-screen ones do not (§19c). - */ - exec(input: ExecInput, options?: ExecOptions): Promise; - /** - * why: turns the lease's iOS phone or tablet simulator to `orientation`, whether or not the daemon's input module is on. `requesterId` is for an operator that acts for the lease's requester, as `exec`'s is. - */ - rotate( - leaseId: string, - orientation: RotateInput["orientation"], - options?: { readonly requesterId?: string }, - ): Promise; - - /** - * Reports a lease the *daemon* ended -- an expiry, an operator release, an unrecoverable - * device. ADR 0004 §3 narrows ADR 0003 §10 here: a dead connection is not a dead lease, so - * this no longer fires with a synthesized `daemon-connection-lost` when the socket dies. The - * leases this client held are still granted and still counting down; `onConnectionLost` is - * what reports the connection, and reconnecting and renewing is the frontend's call. - */ - onLeaseLost(listener: (push: LeaseLostPush) => void): () => void; - onDeviceUnhealthy(listener: (push: DeviceUnhealthyPush) => void): () => void; - onDeviceRecovered(listener: (push: DeviceRecoveredPush) => void): () => void; - /** Fires exactly once, when the connection dies for any reason (including a deliberate - * `close()`). Every in-flight call has already rejected `DAEMON_CONNECTION_LOST` by the time - * this listener runs. Any lease this client was holding is still alive on the daemon. */ - onConnectionLost(listener: (error: AnySimlockError) => void): () => void; - - close(): Promise; -} - -/** `simlock/admin`'s extension: the `admin` rows of ADR §3's matrix. */ -export interface SimlockAdminClient extends SimlockClient { - releaseAllLeases(): Promise; - list(input?: ListGetInput): Promise; - runCleanup(input?: CleanupRunInput): Promise; - runNuke(input?: NukeRunInput): Promise; - forgetWarmKind(input: WarmForgetInput): Promise; - getConfig(input?: ConfigGetInput): Promise; - /** ADR §6's frozen exception: accepted by the daemon at any protocol version it has ever - * spoken, even before `hello`/role resolve. Exposed here rather than on the base client - * purely as an API-surface choice matching the ADR's operation matrix (the daemon itself - * does not gate this operation by role at all -- see the caveat in the PR report). */ - stopDaemon(): Promise; - updateDaemon(input?: DaemonUpdateInput): Promise; - replayEvents(input?: EventsReplayInput): Promise; - usage(window: UsageGetInput): Promise; - subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise>; - createToken(input: TokenCreateInput): Promise; - listTokens(): Promise; - revokeToken(input: TokenRevokeInput): Promise; - - /** why: ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker`, `removeWorker` and - * `updateWorker` (a gateway updating one of its workers, ADR 0042 §2) reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ - listWorkers(): Promise; - drainWorker(input: WorkerDrainInput): Promise; - undrainWorker(input: WorkerDrainInput): Promise; - removeWorker(input: WorkerDrainInput): Promise; - updateWorker(input: WorkerUpdateInput): Promise; - - /** - * why: ADR 0035 §1: asks a gateway to post one sample message to the named alert destination, once. - * Rejects with `ALERT_SEND_FAILED` when the receiver does not take it, `UNKNOWN_ALERT_DESTINATION` - * for a name the gateway has not configured, and on a worker with `UNSUPPORTED_IN_WORKER_MODE`. - */ - testAlertDestination(input: AlertsTestInput): Promise; - - /** - * ADR 0010 §6: installs one iOS simulator runtime or Android system image, with no lease. - * Resolves once it is installed or found already there; `options.onProgress` hears `waiting` - * while another install on the platform runs ahead, then `downloading`. Refused with - * `DOWNLOADS_DISABLED` under `downloads.policy: "never"`. - */ - installComponent( - input: ComponentInstallInput, - options?: InstallComponentOptions, - ): Promise; - - /** - * ADR 0010 §7, gateway-only: installs one component on named workers (`workers: [ids]`) or on - * every connected one (`workers: "all"`). Every targeted worker is asked at the same time and - * answers for itself; the result lists one entry per worker. `options.onProgress` hears each - * worker's progress with its `workerId`. A worker answers `UNSUPPORTED_IN_WORKER_MODE`. - */ - installComponentOnWorkers( - input: WorkerInstallComponentInput, - options?: InstallComponentOnWorkersOptions, - ): Promise; - - /** - * ADR 0010 §8: removes one component Simlock installed, or with `dryRun: true` runs every - * check and removes nothing. Refused with `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE` (with the - * counts as details) or `COMPONENT_BUSY`. - */ - removeComponent(input: ComponentRemoveInput): Promise; - - /** why: ADR 0061 §4: ends the held lease and keeps its device as a named baseline; admin only. */ - saveBaseline(input: BaselineSaveInput): Promise; - - /** why: the newest version of each baseline on the worker, with its stale reason. */ - listBaselines(): Promise; - - /** why: deletes every version of a baseline and the idle devices made from it; refused while a lease uses it. */ - removeBaseline(input: BaselineRemoveInput): Promise; - - /** - * Enrols the physical device plugged into the host this daemon runs on, named by - * any ID the platform's driver reads. A device it cannot enrol is `DEVICE_NOT_ENROLLABLE` with - * a reason; a leased, reclaiming or busy one is `DEVICE_BUSY`. A device already - * enrolled is enrolled again: its record is kept and its model, OS version, class and apps are - * read anew (`reenrolled: true`). A gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`. - */ - addDevice(input: DeviceAddInput): Promise; - - /** - * Forgets an enrolled physical device, named by its canonical ID. Nothing is sent - * to the device. `UNKNOWN_DEVICE` for an ID with no enrolled device, `DEVICE_BUSY` for a leased, - * reclaiming or busy one (another operation holds it). A gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`. - */ - removeDevice(input: DeviceRemoveInput): Promise; -} - -type OperationInput = z.infer<(typeof OPERATIONS)[Name]["input"]>; -type OperationOutput = z.infer<(typeof OPERATIONS)[Name]["output"]>; - -// why: not part of `SimlockAdminClient`, so `simlock/admin` does not export it (ADR 0026). - -export interface RelayClientMethods { - relayRequest( - input: OperationInput<"relay.request">, - hooks: Pick, - ): Promise>; - // why: rejects on a connection that holds no link, since a channel is a link's (ADR 0028). - openChannel(options: { readonly signal: AbortSignal }): Promise; - relayCancel(input: OperationInput<"relay.cancel">): Promise>; -} - export async function connectSimlockClient( options: ConnectOptions, admin: false, @@ -433,120 +179,6 @@ export async function connectSimlockClient( return admin ? impl.asAdmin() : impl; } -/** - * why: ADR §6's escape hatch on the client side. A mismatch leaves the connection open so an admin can still send the - * two frozen calls, `stopDaemon()` and `updateDaemon()` (the daemon answers them at any protocol version, ahead of its - * mismatch gate); everything else rejects with the captured `PROTOCOL_VERSION_UNSUPPORTED` error without being sent, - * and `close()` still works. `role` reflects which entry point the caller used and `principal` what it supplied, - * because the daemon resolves them but the failed `hello` never reports them back; the daemon's own role check on the - * credential it already verified is what gates the two calls. - */ -function buildDegradedClient( - wire: SimlockWire, - error: SimlockError<"PROTOCOL_VERSION_UNSUPPORTED">, - principal: string, - admin: boolean, -): SimlockClient | SimlockAdminClient { - const rejected = (): Promise => Promise.reject(error); - const client: SimlockAdminClient & RelayClientMethods = { - principal, - role: admin ? "admin" : "agent", - daemonVersion: error.details.daemonVersion, - - getCatalog: () => rejected(), - getStatus: () => rejected(), - identity: () => rejected(), - listComponents: () => rejected(), - requestLease: () => rejected(), - cancelLease: () => rejected(), - renewLease: () => rejected(), - releaseLease: () => rejected(), - listLeases: () => rejected(), - runDoctor: () => rejected(), - resolvePassthrough: () => rejected(), - exec: () => rejected(), - rotate: () => rejected(), - - onLeaseLost: () => () => {}, - onDeviceUnhealthy: () => () => {}, - onDeviceRecovered: () => () => {}, - // The one live fact this connection can still report: it can still die later even though - // `hello` never fully succeeded (e.g. the daemon exits while this degraded client sits - // idle). - onConnectionLost: (listener) => wire.onDeath(listener), - - close: () => wire.close(), - - releaseAllLeases: () => rejected(), - list: () => rejected(), - runCleanup: () => rejected(), - runNuke: () => rejected(), - forgetWarmKind: () => rejected(), - getConfig: () => rejected(), - stopDaemon: () => frozenCall(wire, "daemon.stop", {}), - updateDaemon: (input = {}) => frozenCall(wire, "daemon.update", input), - replayEvents: () => rejected(), - usage: () => rejected(), - subscribeEvents: () => rejected(), - createToken: () => rejected(), - listTokens: () => rejected(), - revokeToken: () => rejected(), - listWorkers: () => rejected(), - drainWorker: () => rejected(), - undrainWorker: () => rejected(), - removeWorker: () => rejected(), - updateWorker: () => rejected(), - testAlertDestination: () => rejected(), - installComponent: () => rejected(), - installComponentOnWorkers: () => rejected(), - removeComponent: () => rejected(), - saveBaseline: () => rejected(), - listBaselines: () => rejected(), - removeBaseline: () => rejected(), - addDevice: () => rejected(), - removeDevice: () => rejected(), - relayRequest: () => rejected(), - openChannel: () => rejected(), - relayCancel: () => rejected(), - }; - return client; -} - -// why: the two frozen calls (`daemon.stop`, `daemon.update`) are the only ones a degraded client sends, and the daemon answers them at any protocol version. -function frozenCall( - wire: SimlockWire, - operation: "daemon.stop", - input: Record, -): Promise; -function frozenCall( - wire: SimlockWire, - operation: "daemon.update", - input: DaemonUpdateInput, -): Promise; -function frozenCall( - wire: SimlockWire, - operation: "daemon.stop" | "daemon.update", - input: unknown, -): Promise { - return wire.call(operation, input).then( - (payload) => { - const result = OPERATIONS[operation].output.safeParse(payload); - if (!result.success) { - throw new SimlockError( - "BAD_FRAME", - "protocol", - `Daemon's ${operation} response did not match the contract output schema`, - {}, - ); - } - return result.data; - }, - (callError: unknown) => { - throw toSimlockError(callError); - }, - ); -} - function isLinkConnection(connection: IpcConnection): connection is LinkConnection { return typeof (connection as Partial).openChannel === "function"; } @@ -560,485 +192,3 @@ async function resolveConnection(options: ConnectOptions): Promise void>(); - readonly #deviceUnhealthyListeners = new Set<(push: DeviceUnhealthyPush) => void>(); - readonly #deviceRecoveredListeners = new Set<(push: DeviceRecoveredPush) => void>(); - readonly #connectionLostListeners = new Set<(error: AnySimlockError) => void>(); - #eventSubscriptionId: string | undefined; - - constructor( - wire: SimlockWire, - readonly role: Role, - readonly principal: string, - readonly daemonVersion: string, - link?: LinkConnection, - ) { - this.#wire = wire; - this.#link = link; - wire.onLeaseScopedPush((push) => this.#handleLeaseScopedPush(push)); - wire.onDeath((error) => this.#handleConnectionLost(error)); - } - - asAdmin(): SimlockAdminClient { - // `this` already implements every method the interface below needs (see the class body) -- - // this cast documents that the split is purely at the type layer. - return this as unknown as SimlockAdminClient; - } - - // ---- agent-visible operations -------------------------------------------------------------- - - getCatalog(input: CatalogGetInput = {}): Promise { - return this.#call("catalog.get", input); - } - - getStatus(): Promise { - return this.#call("status.get", {}); - } - - identity(): Promise { - return this.#call("identity.get", {}); - } - - listComponents(input: ComponentListInput = {}): Promise { - return this.#call("component.list", input); - } - - async requestLease( - input: LeaseRequestInput, - options: RequestLeaseOptions = {}, - ): Promise { - const parsed = this.#parseInput("lease.request", input); - const requesterId = parsed.requesterId ?? this.principal; - const { signal, onProgress } = options; - - if (signal?.aborted === true) throw cancelledError(); - - const requestPromise = this.#callRaw("lease.request", parsed, { onProgress }).then((grant) => - this.#parseOutput("lease.request", grant), - ); - - if (signal === undefined) return requestPromise; - return this.#requestLeaseWithAbort(requestPromise, signal, { - platform: parsed.platform, - requesterId, - }); - } - - cancelLease(input: LeaseCancelInput = {}): Promise { - return this.#call("lease.cancel", input); - } - - renewLease(input: LeaseRenewInput): Promise { - return this.#call("lease.renew", input); - } - - async releaseLease(input: LeaseReleaseInput): Promise { - const parsed = this.#parseInput("lease.release", input); - // The daemon suppresses the matching `lease-lost` push to this connection itself (ADR - // §8) -- see the comment on `SimlockWire`'s `#deliveredLeaseLost`. Nothing to mark here. - return this.#call("lease.release", parsed); - } - - listLeases(): Promise { - return this.#call("lease.list", {}); - } - - runDoctor(input: DoctorRunInput = {}): Promise { - return this.#call("doctor.run", input); - } - - resolvePassthrough(input: DriverPassthroughInput): Promise { - return this.#call("driver.passthrough", input); - } - - async exec(input: ExecInput, options: ExecOptions = {}): Promise { - const payload = await this.#callRaw("device.exec", this.#parseInput("device.exec", input), { - onOutput: options.onOutput, - onStarted: options.onStarted, - }); - return this.#parseOutput("device.exec", payload); - } - - rotate( - leaseId: string, - orientation: RotateInput["orientation"], - options: { readonly requesterId?: string } = {}, - ): Promise { - return this.#call("device.rotate", { leaseId, orientation, ...options }); - } - - onLeaseLost(listener: (push: LeaseLostPush) => void): () => void { - this.#leaseLostListeners.add(listener); - return () => this.#leaseLostListeners.delete(listener); - } - - onDeviceUnhealthy(listener: (push: DeviceUnhealthyPush) => void): () => void { - this.#deviceUnhealthyListeners.add(listener); - return () => this.#deviceUnhealthyListeners.delete(listener); - } - - onDeviceRecovered(listener: (push: DeviceRecoveredPush) => void): () => void { - this.#deviceRecoveredListeners.add(listener); - return () => this.#deviceRecoveredListeners.delete(listener); - } - - onConnectionLost(listener: (error: AnySimlockError) => void): () => void { - this.#connectionLostListeners.add(listener); - return () => this.#connectionLostListeners.delete(listener); - } - - async close(): Promise { - await this.#wire.close(); - } - - async relayRequest( - input: OperationInput<"relay.request">, - hooks: Pick, - ): Promise> { - const payload = await this.#callRaw("relay.request", input, hooks); - return this.#parseOutput("relay.request", payload); - } - - openChannel(options: { readonly signal: AbortSignal }): Promise { - const link = this.#link; - if (link === undefined) return Promise.reject(new Error("The connection carries no link")); - return link.openChannel(options); - } - - relayCancel(input: OperationInput<"relay.cancel">): Promise> { - return this.#call("relay.cancel", input); - } - - // ---- admin-only operations ------------------------------------------------------------------ - - releaseAllLeases(): Promise { - return this.#call("lease.release-all", {}); - } - - list(input: ListGetInput = {}): Promise { - return this.#call("list.get", input); - } - - runCleanup(input: CleanupRunInput = {}): Promise { - return this.#call("cleanup.run", input); - } - - runNuke(input: NukeRunInput = {}): Promise { - return this.#call("nuke.run", input); - } - - forgetWarmKind(input: WarmForgetInput): Promise { - return this.#call("warm.forget", input); - } - - getConfig(input: ConfigGetInput = {}): Promise { - return this.#call("config.get", input); - } - - stopDaemon(): Promise { - return this.#call("daemon.stop", {}); - } - - updateDaemon(input: DaemonUpdateInput = {}): Promise { - return this.#call("daemon.update", input); - } - - replayEvents(input: EventsReplayInput = {}): Promise { - return this.#call("events.replay", input); - } - - usage(window: UsageGetInput): Promise { - return this.#call("usage.get", window); - } - - async subscribeEvents(listener: (event: EventPush) => void): Promise<() => Promise> { - const unsubscribePush = this.#wire.onEvent((payload) => listener(payload as EventPush)); - const result = await this.#call("events.subscribe", {}); - this.#eventSubscriptionId = result.subscriptionId; - return async () => { - unsubscribePush(); - if (this.#eventSubscriptionId === undefined) return; - this.#eventSubscriptionId = undefined; - await this.#call("events.unsubscribe", {}); - }; - } - - createToken(input: TokenCreateInput): Promise { - return this.#call("token.create", input); - } - - listTokens(): Promise { - return this.#call("token.list", {}); - } - - revokeToken(input: TokenRevokeInput): Promise { - return this.#call("token.revoke", input); - } - - listWorkers(): Promise { - return this.#call("worker.list", {}); - } - - drainWorker(input: WorkerDrainInput): Promise { - return this.#call("worker.drain", input); - } - - undrainWorker(input: WorkerDrainInput): Promise { - return this.#call("worker.undrain", input); - } - - updateWorker(input: WorkerUpdateInput): Promise { - return this.#call("worker.update", input); - } - - testAlertDestination(input: AlertsTestInput): Promise { - return this.#call("alerts.test", input); - } - - async installComponent( - input: ComponentInstallInput, - options: InstallComponentOptions = {}, - ): Promise { - const payload = await this.#callRaw( - "component.install", - this.#parseInput("component.install", input), - { onComponentProgress: options.onProgress }, - ); - return this.#parseOutput("component.install", payload); - } - - async installComponentOnWorkers( - input: WorkerInstallComponentInput, - options: InstallComponentOnWorkersOptions = {}, - ): Promise { - const { onProgress } = options; - const payload = await this.#callRaw( - "worker.install-component", - this.#parseInput("worker.install-component", input), - { - // A gateway always names the worker; an update that names none cannot be attributed - // to one, so it is dropped rather than guessed. - onComponentProgress: - onProgress === undefined - ? undefined - : (progress, workerId) => { - if (workerId !== undefined) onProgress({ ...progress, workerId }); - }, - }, - ); - return this.#parseOutput("worker.install-component", payload); - } - - removeWorker(input: WorkerDrainInput): Promise { - return this.#call("worker.remove", input); - } - - removeComponent(input: ComponentRemoveInput): Promise { - return this.#call("component.remove", input); - } - - saveBaseline(input: BaselineSaveInput): Promise { - return this.#call("baseline.save", input); - } - - listBaselines(): Promise { - return this.#call("baseline.list", {}); - } - - removeBaseline(input: BaselineRemoveInput): Promise { - return this.#call("baseline.remove", input); - } - - addDevice(input: DeviceAddInput): Promise { - return this.#call("device.add", input); - } - - removeDevice(input: DeviceRemoveInput): Promise { - return this.#call("device.remove", input); - } - - #requestLeaseWithAbort( - requestPromise: Promise, - signal: AbortSignal, - cancel: LeaseCancelInput, - ): Promise { - return new Promise((resolve, reject) => { - let requestDone = false; - let aborted = false; - - const onAbort = (): void => { - if (requestDone) return; // after the grant resolved (or it already failed) -> ignored. - aborted = true; - void this.#cancelOnAbort(requestPromise, cancel).then(resolve, reject); - }; - signal.addEventListener("abort", onAbort, { once: true }); - - requestPromise.then( - (grant) => { - requestDone = true; - signal.removeEventListener("abort", onAbort); - if (!aborted) resolve(grant); - }, - (error: unknown) => { - requestDone = true; - signal.removeEventListener("abort", onAbort); - if (!aborted) reject(error as Error); - }, - ); - }); - } - - async #cancelOnAbort( - requestPromise: Promise, - cancel: LeaseCancelInput, - ): Promise { - let outcome: LeaseCancelOutput["result"]; - try { - outcome = (await this.#call("lease.cancel", cancel)).result; - } catch (error: unknown) { - // Only a connection that actually died mid-cancel falls through to let awaiting - // `requestPromise` below surface whatever that produces (most likely - // `DAEMON_CONNECTION_LOST`, a more accurate rejection than a manufactured `CANCELLED` - // would be here). Every other rejection -- `FORBIDDEN`, `BAD_REQUEST` -- is a real - // answer from the daemon and must surface as-is: this `catch` used to swallow those - // unconditionally, which is what let an aborted `requestLease` silently resolve with a - // real grant instead of `CANCELLED` (see the PR report's B3). - if (isSimlockError(error) && error.kind === "transport") { - return requestPromise; - } - throw error; - } - - if (outcome === "not-found") { - // Either genuinely never existed, or -- per ADR §10's fourth case -- the grant already - // resolved before the cancel reached the daemon: "after the grant resolved -> ignored". - // Both cases defer entirely to the original outcome. - return requestPromise; - } - - if (outcome === "cancelled") { - // Queued: the daemon rejects the original waiter; wait for that, then surface CANCELLED - // regardless of the rejection's own code/message. - await requestPromise.catch(() => undefined); - throw cancelledError(); - } - - // "not-cancellable": device work already in flight. Wait for the outcome; a grant that - // still arrives is abandoned immediately so the caller never holds a lease it walked away - // from, then CANCELLED is surfaced either way. - try { - const grant = await requestPromise; - await this.#releaseAbandonedGrant(grant); - } catch { - // The request failed on its own (e.g. ran out of capacity mid-flight) -- nothing to - // release, and CANCELLED is still the right thing to tell a caller who asked to abort. - } - throw cancelledError(); - } - - async #releaseAbandonedGrant(grant: LeaseGrant): Promise { - try { - await this.releaseLease({ leaseId: grant.lease.id }); - } catch { - // Best-effort: the caller already gets CANCELLED either way. A failed release here means - // the daemon may still show the lease held -- see the PR report's weak-spots list. - } - } - - // ---- pushes ----------------------------------------------------------------------------- - - #handleLeaseScopedPush(push: LeaseScopedPush): void { - switch (push.kind) { - case "lease-lost": - for (const listener of this.#leaseLostListeners) { - listener({ deviceId: push.deviceId, leaseId: push.leaseId, reason: push.reason }); - } - return; - case "device-unhealthy": - for (const listener of this.#deviceUnhealthyListeners) { - listener({ deviceId: push.deviceId, leaseId: push.leaseId }); - } - return; - case "device-recovered": - for (const listener of this.#deviceRecoveredListeners) { - listener({ attempts: push.attempts, deviceId: push.deviceId, leaseId: push.leaseId }); - } - return; - } - } - - /** - * ADR 0004 §3: a dead connection ends no lease, so nothing is synthesized here. The daemon - * releases nothing on close, so every lease this client was holding is still granted, still - * counting down its own TTL, and still renewable by whatever connects next -- which is the - * frontend's decision to make, not this client's (ADR 0003 §10). - */ - #handleConnectionLost(error: AnySimlockError): void { - for (const listener of this.#connectionLostListeners) listener(error); - } - - // ---- call plumbing -------------------------------------------------------------------------- - - #parseInput( - name: Name, - input: unknown, - ): z.infer<(typeof OPERATIONS)[Name]["input"]> { - const result = OPERATIONS[name].input.safeParse(input); - if (result.success) return result.data as z.infer<(typeof OPERATIONS)[Name]["input"]>; - throw new SimlockError( - "BAD_REQUEST", - "protocol", - describeSchemaIssues(result.error.issues), - {}, - ); - } - - #parseOutput( - name: Name, - payload: unknown, - ): z.infer<(typeof OPERATIONS)[Name]["output"]> { - const result = OPERATIONS[name].output.safeParse(payload); - if (result.success) return result.data as z.infer<(typeof OPERATIONS)[Name]["output"]>; - throw new SimlockError( - "BAD_FRAME", - "protocol", - `Daemon's ${name} response did not match the contract output schema`, - {}, - ); - } - - async #call( - name: Name, - input: z.infer<(typeof OPERATIONS)[Name]["input"]>, - ): Promise> { - const payload = await this.#callRaw(name, input); - return this.#parseOutput(name, payload); - } - - #callRaw(name: OperationName, input: unknown, hooks: CallHooks = {}): Promise { - const parsed = this.#parseInput(name, input); - return this.#wire.call(name, parsed, hooks).catch((error: unknown) => { - throw toSimlockError(error); - }); - } -} - -function cancelledError(): SimlockError<"CANCELLED"> { - return new SimlockError("CANCELLED", "domain", "Lease request was cancelled", {}); -} - -function toSimlockError(error: unknown): AnySimlockError { - if (isSimlockError(error)) return error; - if (error instanceof WireCallError) - return fromWireError(error.code, error.message, error.details); - return new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - error instanceof Error ? error.message : String(error), - {}, - ); -} From 870e11764c8931b32e8fec6d02b10636170f7a1d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:13:16 +0100 Subject: [PATCH 11/83] test(core): the provision call-site scan names the android test-* parts --- src/core/device-provisioner.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/device-provisioner.test.ts b/src/core/device-provisioner.test.ts index a6675dd12..77b987dcf 100644 --- a/src/core/device-provisioner.test.ts +++ b/src/core/device-provisioner.test.ts @@ -461,7 +461,8 @@ describe("DeviceProvisioner and a component removal (ADR 0010 §8)", () => { // why: not a second creator: the Android driver's own `provision` hands the call to its seeds // part, and its test helpers (never loaded in production) drive the driver to build fixtures. [join("drivers", "android", "index.ts")]: 1, - [join("drivers", "android", "testing.ts")]: 2, + [join("drivers", "android", "test-driver-harness.ts")]: 1, + [join("drivers", "android", "test-seed-harness.ts")]: 1, // why: the harness's fixture devices, as for registerDevice above. [join("leasing", "test-lease-acquisition.ts")]: 1, [join("daemon", "test-dispatcher.ts")]: 1, From cb322483740c60e09ca1be7025d46caadeb16ac9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:17:10 +0100 Subject: [PATCH 12/83] refactor(drivers): move the iOS driver's runtime, data and passthrough helpers out of index.ts --- src/drivers/ios/index.ts | 601 +++------------------------ src/drivers/ios/passthrough-rules.ts | 156 +++++++ src/drivers/ios/runtime-images.ts | 200 +++++++++ src/drivers/ios/runtime-versions.ts | 196 +++++++++ src/drivers/ios/simulator-data.ts | 39 ++ 5 files changed, 644 insertions(+), 548 deletions(-) create mode 100644 src/drivers/ios/passthrough-rules.ts create mode 100644 src/drivers/ios/runtime-images.ts create mode 100644 src/drivers/ios/runtime-versions.ts create mode 100644 src/drivers/ios/simulator-data.ts diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 1a0014653..17a28b575 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -1,5 +1,4 @@ import { dirname, join } from "node:path"; - import { BootTimeoutError, KeptCategoriesError, @@ -35,11 +34,8 @@ import { OwnedRootError, type PassthroughCommand, type PassthroughContext, - PassthroughRefusedError, removeListedComponent, - RuntimeMissingError, sameReceipt, - UnknownModelError, UnsupportedRequestOptionError, type ObservedMark, type DeviceClass, @@ -79,8 +75,6 @@ import { UDID_PATTERN, } from "./device-set-leftovers.js"; import { createIosDeviceTcpFor } from "./device-tcp.js"; -import { devicectlExecCommand } from "./devicectl-exec.js"; -import { fillSimctlDevice } from "./simctl-device.js"; import { IosPhysicalHandler } from "./physical.js"; import { createIosDeviceInput } from "./input/index.js"; import { iosRuntimeVersionFromId } from "./runtime-version.js"; @@ -101,9 +95,51 @@ import { type DeviceType, type Runtime, type SimctlCatalog, - UNBOUNDED_VERSION, } from "./simctl-parse.js"; import { parseOverridesPlist, parsePrintDisabled, renderOverridesPlist } from "./slim-overrides.js"; +import { type IosDriverData, iosDriverData, requireIosPlatform } from "./simulator-data.js"; +import { IOS_PASSTHROUGH_TOOLS, passthroughCommand } from "./passthrough-rules.js"; +import { + IOS_RUNTIME_MIN_FREE_BYTES, + IOS_RUNTIME_ASSET_ROOT, + type RuntimeAsset, + runtimeMountsAsset, + parseRuntimeAsset, + type RuntimeImage, + isBeingDeleted, + parseRuntimeImages, + runtimeReceipt, + imageReceipt, + isIosImage, + type ListedImage, + leftoverDownloadsSentence, + unavailableDevicesResidue, + type DefaultSetDevices, + devicesPerRuntime, + installedComponent, + withBareMajorHint, +} from "./runtime-images.js"; +import { + LATEST_COMPONENT, + missingRuntime, + belowDownloadFloor, + IosUnknownModelError, + IosVersionOutOfRangeError, + IosRuntimeUnpairedError, + IosDownloadFloorError, + IosRuntimeUnavailableForDownloadError, + runtimeForComponent, + findInstalledRuntime, + findDeviceType, + pairedInstalledRuntimes, + findPairedRuntime, + pairedVersions, + isUnboundedMax, + isVersionInRange, + majorVersionString, + newestRuntime, + compareVersions, +} from "./runtime-versions.js"; /** why: ADR 0061 §1: a `making` row of this process older than this is a make that never ended. */ const STALE_MAKING_MS = 300_000; @@ -120,20 +156,6 @@ const PROVISION_ESTIMATE_MS = 500; // why: How often a removal lists again while `simctl runtime list` shows the deleted image as still // being deleted, which it does for a few seconds after `simctl runtime delete` answers (#259). const DELETING_POLL_MS = 1_000; -const LATEST_COMPONENT = "latest"; -// why: `xcodebuild -downloadPlatform iOS -buildVersion` only reaches back to iOS 16.0 (Xcode -// 16.1+); older runtimes must be installed through Xcode itself. -const IOS_DOWNLOAD_FLOOR: readonly [number, number, number] = [16, 0, 0]; -// why: Conservative estimate for a simulator runtime download+install (~7 GB observed, rounded up -// with headroom) -- reserved by `ComponentInstaller` before `xcodebuild -downloadPlatform` ever -// starts, so a full disk fails fast instead of filling up mid-download. -const IOS_RUNTIME_MIN_FREE_BYTES = 8 * 1024 ** 3; -// why: Where macOS's own asset daemon keeps every simulator runtime it has ever downloaded, one -// `.asset` bundle per build. `simctl runtime delete` only unregisters a runtime from -// CoreSimulator; the bundle it was mounted from stays here, marked never-collected, spending -// the same free space `IOS_RUNTIME_MIN_FREE_BYTES` measures. Nothing Simlock may do can -// reclaim it (issue #79), so the driver only ever reads this path. -const IOS_RUNTIME_ASSET_ROOT = "/System/Library/AssetsV2/com_apple_MobileAsset_iOSSimulatorRuntime"; // why: A cold `simctl boot` to `bootstatus` measures roughly 30s on a fast, idle machine and up to // a minute on a loaded or slower one. The upper end is the estimate, deliberately: this number // is what a waiting requester is quoted, and quoting 30s to someone who then waits 60s is the @@ -161,13 +183,6 @@ async function withPasteboardWrite(run: () => Promise): Promise { return await turn; } -/** - * why: a driver that refused to start has no instance to ask, so the `simlock ` names - * it answers to are a constant, and `DriverRejection` carries them so `simlock simctl` can - * say why it is unavailable rather than reading as a missing SDK. - */ -export const IOS_PASSTHROUGH_TOOLS: readonly string[] = ["simctl", "devicectl"]; - const IOS_DEFAULT_MODELS: Readonly>> = { phone: ["iPhone 17", "iPhone 16", "iPhone 15", "iPhone 14", "iPhone 13"], tablet: [ @@ -186,24 +201,6 @@ const IOS_DEFAULT_MODELS: Readonly = new ["Apple Vision", "vision"], ]); -interface RuntimeAsset { - readonly path: string; - readonly version: string; - readonly build: string; -} - interface ResolvedIosSpec { readonly deviceType: DeviceType; readonly runtime: Runtime; @@ -454,7 +438,7 @@ export class IosSimctlDriver implements Driver { } async resolveSpec(request: ExactDeviceRequest): Promise { - this.#requireIosPlatform(request.platform); + requireIosPlatform(request.platform); if (request.imageTag !== undefined) { throw new UnsupportedRequestOptionError("ios", "imageTag"); } @@ -929,7 +913,7 @@ export class IosSimctlDriver implements Driver { spec: VirtualSpec, options?: { readonly source?: string }, ): Promise { - this.#requireIosPlatform(spec.platform); + requireIosPlatform(spec.platform); const resolved = await this.#resolvedSpec(spec); const seed = options?.source === undefined @@ -1747,124 +1731,13 @@ export class IosSimctlDriver implements Driver { readonly passthroughTools = IOS_PASSTHROUGH_TOOLS; - async passthrough( + passthrough( tool: string, device: VirtualDriverDevice | PhysicalDriverDevice | undefined, args: readonly string[], context?: PassthroughContext, ): Promise { - if (tool === "devicectl") return this.#devicectlPassthrough(device, args, context); - if (tool !== "simctl") { - throw new PassthroughRefusedError( - tool, - `Refusing \`${tool}\`: the iOS driver does not wrap it.`, - ); - } - if (context?.platform !== undefined && context.platform !== this.platform) { - throw new PassthroughRefusedError( - tool, - `Refusing \`simctl\`: it only serves an iOS simulator, and this lease is on the ${context.platform} platform.`, - ); - } - if (device !== undefined && isPhysicalDriverDevice(device)) { - throw new PassthroughRefusedError( - tool, - "Refusing `simctl`: it serves simulators, and this lease holds a physical device. Use `devicectl`.", - ); - } - this.#assertProxyable(args); - const scopedArgs = device === undefined ? args : fillSimctlDevice(args, device.deviceId); - // why: No environment: simctl ignores every variable that could carry the device set. - return await Promise.resolve({ - args: this.#scopedSimctlArgv(scopedArgs), - command: "xcrun", - env: {}, - }); - } - - #devicectlPassthrough( - device: VirtualDriverDevice | PhysicalDriverDevice | undefined, - args: readonly string[], - context: PassthroughContext | undefined, - ): PassthroughCommand { - const tool = "devicectl"; - if (device === undefined) { - throw new PassthroughRefusedError( - tool, - "Refusing `simlock devicectl`: Simlock has no local devicectl wrapper. Run `xcrun devicectl` directly, or use `device.exec` on a lease.", - ); - } - if (context?.platform !== undefined && context.platform !== this.platform) { - throw new PassthroughRefusedError( - tool, - `Refusing \`devicectl\`: it only serves a physical iOS device, and this lease is on the ${context.platform} platform.`, - ); - } - if (!isPhysicalDriverDevice(device)) { - throw new PassthroughRefusedError( - tool, - "Refusing `devicectl`: it only serves a physical device, and this lease holds a simulator. Use `simctl`.", - ); - } - return devicectlExecCommand(device.deviceId, args); - } - - #assertProxyable(args: readonly string[]): void { - const [verb, ...operands] = this.#subcommand(args); - if (verb === undefined) return; - if (REFUSED_SIMCTL_VERBS.has(verb)) { - this.#refuse( - verb, - `it changes a device's lifecycle behind Simlock's registry, which would report the device as drifted on the next reconcile. ${RECLAIM_INSTEAD}`, - ); - } - if (verb === "shutdown" && operands.includes(SHUTDOWN_ALL_TARGET)) { - this.#refuse( - `shutdown ${SHUTDOWN_ALL_TARGET}`, - `it stops every device in Simlock's set at once -- every agent's, not just yours -- and each interrupted lease spends its recovery budget rebooting, so one that runs out ends as \`lease_lost\`. Shutting a single device down by udid is still allowed. ${RECLAIM_INSTEAD}`, - ); - } - // why: A bare `simctl` reaches this too, so refusing it takes no capability away. The - // wrapper is advertised as the safe path, and being the convenient route to a - // multi-gigabyte deletion of a runtime Xcode shares is not that. - if ( - verb === "runtime" && - operands.find((operand) => !operand.startsWith("-")) === REFUSED_RUNTIME_OPERATION - ) { - this.#refuse( - `runtime ${REFUSED_RUNTIME_OPERATION}`, - "it deletes a runtime shared with Xcode and other tools. To remove a runtime Simlock added, use `simlock component remove ios `; delete any other through Xcode.", - ); - } - } - - #refuse(command: string, guidance: string): never { - throw new PassthroughRefusedError( - "simctl", - `Refusing \`simlock simctl ${command}\`: ${guidance}`, - ); - } - - #subcommand(args: readonly string[]): readonly string[] { - for (const [index, argument] of args.entries()) { - if (!argument.startsWith("-")) return args.slice(index); - const flag = /^-+([^=]*)/.exec(argument)?.[1]; - if (flag !== undefined && CALLER_SUPPLIED_SCOPE_FLAGS.has(flag)) { - throw new PassthroughRefusedError( - "simctl", - `Refusing \`simlock simctl ${argument}\`: \`simlock simctl\` supplies the device set itself, and a caller-supplied \`--${flag}\` would point simctl somewhere Simlock does not manage. Drop the flag -- the command is already scoped -- or run \`xcrun simctl\` directly if you mean to leave Simlock's set.`, - ); - } - // why: Not `--set`/`--profiles` by name, but simctl has no other legitimate global here -- - // refused on sight rather than skipped over as presumed harmless, the same way an - // unrecognized adb global is (`allowedGlobalArity`). Skipping it is exactly what let a - // caller-supplied `--set`/`--profiles` hide behind an unrecognized flag's *value*. - throw new PassthroughRefusedError( - "simctl", - `Refusing \`simlock simctl ${argument}\`: \`simlock simctl\` supplies the device set itself, and does not recognize any caller-supplied argument ahead of the subcommand. Drop the flag -- the command is already scoped -- or run \`xcrun simctl\` directly if you mean to leave Simlock's set.`, - ); - } - return []; + return passthroughCommand(tool, device, args, context, (a) => this.#scopedSimctlArgv(a)); } #deviceDirectory(udid: string): string { @@ -2112,73 +1985,6 @@ export class IosSimctlDriver implements Driver { ); } } - - #requireIosPlatform(platform: string): void { - if (platform !== this.platform) { - throw new DriverCrashError(`iOS simctl driver cannot handle ${platform} device requests`); - } - } -} - -function missingRuntime( - osVersion: string, - message: string, - component: string, -): RuntimeMissingError { - return belowDownloadFloor(component) - ? new IosDownloadFloorError(component) - : new IosRuntimeMissingError(osVersion, message, component); -} - -function belowDownloadFloor(component: string): boolean { - return component !== LATEST_COMPONENT && isTooOldToDownload(component); -} - -class IosRuntimeMissingError extends RuntimeMissingError { - constructor(osVersion: string, message: string, component: string) { - super("ios", osVersion, { component }); - this.message = message; - } -} - -class IosUnknownModelError extends UnknownModelError { - constructor(model: string) { - super("ios", model); - this.message = `Unknown ios model: ${model}; a newer Xcode version may add this device`; - } -} - -class IosVersionOutOfRangeError extends RuntimeMissingError { - constructor(model: string, requested: string, deviceType: DeviceType) { - super("ios", requested, { downloadable: false }); - const range = formatVersionRange(deviceType.minRuntimeVersion, deviceType.maxRuntimeVersion); - this.message = `${model} supports iOS ${range}; iOS ${requested} is out of range`; - } -} - -class IosRuntimeUnpairedError extends RuntimeMissingError { - constructor(model: string, requested: string) { - super("ios", requested, { downloadable: false }); - this.message = `iOS ${requested} is installed but does not support ${model}`; - } -} - -class IosDownloadFloorError extends RuntimeMissingError { - constructor(requested: string) { - super("ios", requested, { downloadable: false }); - this.message = - `iOS ${requested} predates Xcode's automatic download support (introduced for iOS ` + - `16.0 and newer); install it manually via Xcode`; - } -} - -class IosRuntimeUnavailableForDownloadError extends RuntimeMissingError { - constructor(requested: string) { - super("ios", requested, { downloadable: false }); - this.message = - `iOS ${requested} is not currently offered for download by Apple (likely superseded by a ` + - `newer point release); request a different --os, or omit --os for the newest available runtime`; - } } // why: the input is offered exactly when the screen capture is, because both are the one program. @@ -2260,313 +2066,6 @@ function configuredDeviceRoot(options: IosSimctlDriverOptions): string { return configured ?? defaultIosDeviceRoot(options.simlockHome); } -function runtimeMountsAsset(runtime: Runtime, asset: RuntimeAsset): boolean { - return runtime.build === undefined - ? runtime.version === asset.version - : runtime.build === asset.build; -} - -/** - * why: The build and marketing version out of an asset bundle's `Info.plist`. Read as text rather - * than through a plist parser: these are two flat string values in a file this driver must - * never write, and shelling out to `plutil` once per bundle would make a `doctor` run pay for - * every runtime ever downloaded. Both keys are read from `MobileAssetProperties` onwards, so a - * same-named key in the surrounding envelope cannot be mistaken for the asset's own. - */ -function parseRuntimeAsset(path: string, plist: string): RuntimeAsset | undefined { - const propertiesAt = plist.indexOf("MobileAssetProperties"); - const properties = propertiesAt === -1 ? plist : plist.slice(propertiesAt); - const build = /Build<\/key>\s*([^<]+)<\/string>/.exec(properties)?.[1]; - const version = /SimulatorVersion<\/key>\s*([^<]+)<\/string>/.exec(properties)?.[1]; - - return build === undefined || version === undefined ? undefined : { build, path, version }; -} - -interface RuntimeImage { - readonly identifier: string; - readonly runtimeIdentifier: string; - readonly build?: string; - readonly platformIdentifier?: string; - readonly version?: string; - readonly sizeBytes?: number; - readonly state?: string; -} - -function isBeingDeleted(image: RuntimeImage): boolean { - return image.state?.toLowerCase() === "deleting"; -} - -const IOS_SIMULATOR_PLATFORM = "com.apple.platform.iphonesimulator"; - -function parseRuntimeImages(value: unknown): readonly RuntimeImage[] { - if (!isRecord(value)) { - throw new DriverCrashError("Invalid simctl runtime list JSON: expected an object"); - } - return Object.values(value).flatMap((entry): RuntimeImage[] => - isRecord(entry) && - typeof entry.identifier === "string" && - typeof entry.runtimeIdentifier === "string" - ? [ - { - identifier: entry.identifier, - runtimeIdentifier: entry.runtimeIdentifier, - ...optionalString("build", entry.build), - ...optionalString("platformIdentifier", entry.platformIdentifier), - ...optionalString("version", entry.version), - ...optionalString("state", entry.state), - ...(isByteCount(entry.sizeBytes) ? { sizeBytes: entry.sizeBytes } : {}), - }, - ] - : [], - ); -} - -function optionalString( - key: Key, - value: unknown, -): Partial> { - return typeof value === "string" ? ({ [key]: value } as Record) : {}; -} - -function isByteCount(value: unknown): value is number { - return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; -} - -function runtimeReceipt(runtime: Runtime, images: readonly RuntimeImage[]): ComponentReceipt { - const image = images.find( - (candidate) => - candidate.runtimeIdentifier === runtime.identifier && - (runtime.build === undefined || candidate.build === runtime.build), - ); - return image === undefined - ? { build: runtime.build ?? "", runtime: runtime.identifier } - : imageReceipt(image); -} - -function imageReceipt(image: RuntimeImage): ComponentReceipt { - return { build: image.build ?? "", image: image.identifier }; -} - -function isIosImage(image: RuntimeImage): boolean { - return image.platformIdentifier === IOS_SIMULATOR_PLATFORM; -} - -type ListedImage = DriverComponent & { - readonly image: RuntimeImage; - readonly unusedDevices: number; -}; - -function leftoverDownloadsSentence( - labels: readonly string[], - bundles: number, - sizeBytes: number | undefined, -): string { - const runtimes = labels.length > 1; - const downloads = bundles > 1; - const size = - sizeBytes === undefined - ? "" - : ` (${(sizeBytes / 1024 ** 3).toFixed(1)} GiB${downloads ? " in all" : ""})`; - return ( - `iOS ${labels.join(", ")} ${runtimes ? "are" : "is"} no longer installed, but ` + - `${runtimes ? "their" : "its"} ${downloads ? "downloads" : "download"}${size} ` + - `${downloads ? "are" : "is"} still in ${IOS_RUNTIME_ASSET_ROOT}; ` + - `\`simctl runtime delete\` does not reclaim ${downloads ? "them" : "it"} and Simlock does ` + - "not delete files there -- remove the platform in Xcode's Settings -> Platforms to get " + - "the space back" - ); -} - -function unavailableDevicesResidue(unused: number): string | undefined { - if (unused === 0) return undefined; - const subject = - unused === 1 - ? "1 never-used simulator in the default device set is" - : `${String(unused)} never-used simulators in the default device set are`; - return ( - `${subject} now unavailable; Simlock does not delete simulators there -- ` + - "`xcrun simctl delete unavailable` clears them" - ); -} - -interface DefaultSetDevices { - readonly used: number; - readonly unused: number; -} - -function devicesPerRuntime( - value: unknown, - owned: ReadonlySet, -): ReadonlyMap { - if (!isRecord(value) || !isRecord(value.devices)) { - throw new DriverCrashError("Invalid simctl device list JSON"); - } - return new Map( - Object.entries(value.devices).map(([runtime, devices]): [string, DefaultSetDevices] => { - const entries: readonly unknown[] = (Array.isArray(devices) ? devices : []).filter( - (device: unknown) => !(isRecord(device) && owned.has(String(device.udid))), - ); - const used = entries.filter( - (device) => !isRecord(device) || device.lastUsedAt !== undefined, - ).length; - return [runtime, { unused: entries.length - used, used }]; - }), - ); -} - -function installedComponent(runtime: Runtime, images: readonly RuntimeImage[]): InstalledComponent { - return { receipt: runtimeReceipt(runtime, images), version: runtime.version }; -} - -function withBareMajorHint(component: string, error: unknown): unknown { - if (component === LATEST_COMPONENT || component.includes(".")) return error; - if (!(error instanceof DriverCrashError)) return error; - return new DriverCrashError( - `Could not download a default iOS runtime (tried ${component}): ${error.message}; ` + - `pass --os to request an exact release`, - ); -} - -function runtimeForComponent(catalog: SimctlCatalog, component: string): Runtime | undefined { - const available = catalog.runtimes.filter((runtime) => runtime.isAvailable); - if (component === LATEST_COMPONENT) return newestRuntime(available); - if (!component.includes(".")) { - return newestRuntime( - available.filter((runtime) => runtime.version.split(".")[0] === component), - ); - } - return findInstalledRuntime(catalog, component); -} - -function findInstalledRuntime(catalog: SimctlCatalog, version: string): Runtime | undefined { - return catalog.runtimes.find((runtime) => runtime.isAvailable && runtime.version === version); -} - -function findDeviceType(catalog: SimctlCatalog, model: string): DeviceType | undefined { - const wanted = model.toLocaleLowerCase(); - return catalog.deviceTypes.find((candidate) => candidate.name.toLocaleLowerCase() === wanted); -} - -function pairedInstalledRuntimes( - catalog: SimctlCatalog, - deviceType: DeviceType, -): readonly Runtime[] { - return catalog.runtimes.filter( - (runtime) => - runtime.isAvailable && - runtime.supportedDeviceTypeIds.has(deviceType.identifier) && - isVersionInRange(runtime.version, deviceType), - ); -} - -function findPairedRuntime( - catalog: SimctlCatalog, - deviceType: DeviceType, - version: string, -): Runtime | undefined { - return pairedInstalledRuntimes(catalog, deviceType).find( - (runtime) => runtime.version === version, - ); -} - -function pairedVersions(catalog: SimctlCatalog, deviceType: DeviceType | undefined): string[] { - if (deviceType === undefined) return []; - return [ - ...new Set(pairedInstalledRuntimes(catalog, deviceType).map((runtime) => runtime.version)), - ]; -} - -function decodeVersionTriple(encoded: number): readonly [number, number, number] { - return [(encoded >> 16) & 0xff, (encoded >> 8) & 0xff, encoded & 0xff]; -} - -function formatDecodedVersion(encoded: number): string { - const [major, minor, patch] = decodeVersionTriple(encoded); - return patch === 0 ? `${major}.${minor}` : `${major}.${minor}.${patch}`; -} - -function isUnboundedMax(encoded: number): boolean { - return encoded >= UNBOUNDED_VERSION; -} - -function formatVersionRange(min: number, max: number): string { - const minLabel = formatDecodedVersion(min); - return isUnboundedMax(max) ? `${minLabel}+` : `${minLabel}-${formatDecodedVersion(max)}`; -} - -function versionTriple(version: string): readonly [number, number, number] { - const parts = version.split(".").map(versionPart); - return [parts[0] ?? 0, parts[1] ?? 0, parts[2] ?? 0]; -} - -function versionOrdinal(triple: readonly [number, number, number]): number { - return triple[0] * 1_000_000 + triple[1] * 1_000 + triple[2]; -} - -function isVersionInRange(version: string, deviceType: DeviceType): boolean { - const ordinal = versionOrdinal(versionTriple(version)); - return ( - ordinal >= versionOrdinal(decodeVersionTriple(deviceType.minRuntimeVersion)) && - ordinal <= versionOrdinal(decodeVersionTriple(deviceType.maxRuntimeVersion)) - ); -} - -function isTooOldToDownload(version: string): boolean { - return versionOrdinal(versionTriple(version)) < versionOrdinal(IOS_DOWNLOAD_FLOOR); -} - -function majorVersionString(encoded: number): string { - return String(decodeVersionTriple(encoded)[0]); -} - -function newestRuntime(runtimes: readonly Runtime[]): Runtime | undefined { - return [...runtimes].sort((left, right) => compareVersions(left.version, right.version)).at(-1); -} - -function compareVersions(left: string, right: string): number { - const leftParts = left.split(".").map(versionPart); - const rightParts = right.split(".").map(versionPart); - const partCount = Math.max(leftParts.length, rightParts.length); - - for (let index = 0; index < partCount; index += 1) { - const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); - if (difference !== 0) { - return difference; - } - } - - return left.localeCompare(right); -} - -function versionPart(part: string): number { - const parsed = Number.parseInt(part, 10); - return Number.isNaN(parsed) ? 0 : parsed; -} - -function iosDriverData(device: DriverDevice): IosDriverData { - const value = device.driverData; - if ( - !isRecord(value) || - typeof value.udid !== "string" || - typeof value.deviceTypeId !== "string" || - typeof value.runtimeId !== "string" || - typeof value.name !== "string" - ) { - throw new DriverCrashError(`Invalid iOS driver data for device ${device.deviceId}`); - } - - return { - deviceTypeId: value.deviceTypeId, - name: value.name, - runtimeId: value.runtimeId, - udid: value.udid, - // why: Both are optional: absent or wrong-typed is ignored rather than rejected. A `full` key - // an older daemon stamped here is not read: the spec's mode is the only record of it (ADR 0007 §7). - ...(typeof value.slimSignature === "string" ? { slimSignature: value.slimSignature } : {}), - ...(typeof value.slimMarkToken === "string" ? { slimMarkToken: value.slimMarkToken } : {}), - }; -} - export { iosRuntimeVersionFromId }; export function supportsPersistentSlim(version: readonly [number, number] | undefined): boolean { @@ -2698,3 +2197,9 @@ export { iosDeviceDataTable } from "./device-data.js"; export { iosSourceDataOwner } from "./source-data.js"; export { iosPrerequisites } from "./prerequisites.js"; export { labelsFor, SLIM_CATEGORIES, slimSignature } from "./slim-labels.js"; + +export { IOS_PASSTHROUGH_TOOLS } from "./passthrough-rules.js"; +export { REFUSED_SIMCTL_VERBS } from "./passthrough-rules.js"; +export { CALLER_SUPPLIED_SCOPE_FLAGS } from "./passthrough-rules.js"; +export { SHUTDOWN_ALL_TARGET } from "./passthrough-rules.js"; +export { REFUSED_RUNTIME_OPERATION } from "./passthrough-rules.js"; diff --git a/src/drivers/ios/passthrough-rules.ts b/src/drivers/ios/passthrough-rules.ts new file mode 100644 index 000000000..613ae6e63 --- /dev/null +++ b/src/drivers/ios/passthrough-rules.ts @@ -0,0 +1,156 @@ +import { + type PhysicalDriverDevice, + type VirtualDriverDevice, + isPhysicalDriverDevice, + type PassthroughCommand, + type PassthroughContext, + PassthroughRefusedError, +} from "../../core/index.js"; +import { devicectlExecCommand } from "./devicectl-exec.js"; +import { fillSimctlDevice } from "./simctl-device.js"; + +/** + * why: a driver that refused to start has no instance to ask, so the `simlock ` names + * it answers to are a constant, and `DriverRejection` carries them so `simlock simctl` can + * say why it is unavailable rather than reading as a missing SDK. + */ +export const IOS_PASSTHROUGH_TOOLS: readonly string[] = ["simctl", "devicectl"]; + +/** + * why: Verbs `simlock simctl` will not proxy. Every one of them changes a device's lifecycle, + * which the registry -- not `simctl` -- is the record of: a device created here has no + * registry entry and reads as an orphan, and one erased or deleted under a live lease + * reads as tampering on the next reconcile. Injecting `--set` for them would hand back + * exactly the capability the device set exists to take away (ADR 0001, decision 7). + */ +export const REFUSED_SIMCTL_VERBS = new Set(["create", "erase", "delete"]); + +export const CALLER_SUPPLIED_SCOPE_FLAGS = new Set(["set", "profiles"]); + +export const SHUTDOWN_ALL_TARGET = "all"; + +export const REFUSED_RUNTIME_OPERATION = "delete"; + +const RECLAIM_INSTEAD = + "Use `simlock release` (which deletes the device for you) or `simlock cleanup` instead."; + +export async function passthroughCommand( + tool: string, + device: VirtualDriverDevice | PhysicalDriverDevice | undefined, + args: readonly string[], + context: PassthroughContext | undefined, + scopedArgv: (args: readonly string[]) => string[], +): Promise { + if (tool === "devicectl") return devicectlPassthrough(device, args, context); + if (tool !== "simctl") { + throw new PassthroughRefusedError( + tool, + `Refusing \`${tool}\`: the iOS driver does not wrap it.`, + ); + } + if (context?.platform !== undefined && context.platform !== "ios") { + throw new PassthroughRefusedError( + tool, + `Refusing \`simctl\`: it only serves an iOS simulator, and this lease is on the ${context.platform} platform.`, + ); + } + if (device !== undefined && isPhysicalDriverDevice(device)) { + throw new PassthroughRefusedError( + tool, + "Refusing `simctl`: it serves simulators, and this lease holds a physical device. Use `devicectl`.", + ); + } + assertProxyable(args); + const scopedArgs = device === undefined ? args : fillSimctlDevice(args, device.deviceId); + // why: No environment: simctl ignores every variable that could carry the device set. + return await Promise.resolve({ + args: scopedArgv(scopedArgs), + command: "xcrun", + env: {}, + }); +} + +function devicectlPassthrough( + device: VirtualDriverDevice | PhysicalDriverDevice | undefined, + args: readonly string[], + context: PassthroughContext | undefined, +): PassthroughCommand { + const tool = "devicectl"; + if (device === undefined) { + throw new PassthroughRefusedError( + tool, + "Refusing `simlock devicectl`: Simlock has no local devicectl wrapper. Run `xcrun devicectl` directly, or use `device.exec` on a lease.", + ); + } + if (context?.platform !== undefined && context.platform !== "ios") { + throw new PassthroughRefusedError( + tool, + `Refusing \`devicectl\`: it only serves a physical iOS device, and this lease is on the ${context.platform} platform.`, + ); + } + if (!isPhysicalDriverDevice(device)) { + throw new PassthroughRefusedError( + tool, + "Refusing `devicectl`: it only serves a physical device, and this lease holds a simulator. Use `simctl`.", + ); + } + return devicectlExecCommand(device.deviceId, args); +} + +function assertProxyable(args: readonly string[]): void { + const [verb, ...operands] = subcommand(args); + if (verb === undefined) return; + if (REFUSED_SIMCTL_VERBS.has(verb)) { + refuse( + verb, + `it changes a device's lifecycle behind Simlock's registry, which would report the device as drifted on the next reconcile. ${RECLAIM_INSTEAD}`, + ); + } + if (verb === "shutdown" && operands.includes(SHUTDOWN_ALL_TARGET)) { + refuse( + `shutdown ${SHUTDOWN_ALL_TARGET}`, + `it stops every device in Simlock's set at once -- every agent's, not just yours -- and each interrupted lease spends its recovery budget rebooting, so one that runs out ends as \`lease_lost\`. Shutting a single device down by udid is still allowed. ${RECLAIM_INSTEAD}`, + ); + } + // why: A bare `simctl` reaches this too, so refusing it takes no capability away. The + // wrapper is advertised as the safe path, and being the convenient route to a + // multi-gigabyte deletion of a runtime Xcode shares is not that. + if ( + verb === "runtime" && + operands.find((operand) => !operand.startsWith("-")) === REFUSED_RUNTIME_OPERATION + ) { + refuse( + `runtime ${REFUSED_RUNTIME_OPERATION}`, + "it deletes a runtime shared with Xcode and other tools. To remove a runtime Simlock added, use `simlock component remove ios `; delete any other through Xcode.", + ); + } +} + +function refuse(command: string, guidance: string): never { + throw new PassthroughRefusedError( + "simctl", + `Refusing \`simlock simctl ${command}\`: ${guidance}`, + ); +} + +function subcommand(args: readonly string[]): readonly string[] { + for (const [index, argument] of args.entries()) { + if (!argument.startsWith("-")) return args.slice(index); + const flag = /^-+([^=]*)/.exec(argument)?.[1]; + if (flag !== undefined && CALLER_SUPPLIED_SCOPE_FLAGS.has(flag)) { + throw new PassthroughRefusedError( + "simctl", + `Refusing \`simlock simctl ${argument}\`: \`simlock simctl\` supplies the device set itself, and a caller-supplied \`--${flag}\` would point simctl somewhere Simlock does not manage. Drop the flag -- the command is already scoped -- or run \`xcrun simctl\` directly if you mean to leave Simlock's set.`, + ); + } + // why: Not `--set`/`--profiles` by name, but simctl has no other legitimate global here -- + // refused on sight rather than skipped over as presumed harmless, the same way an + // unrecognized adb global is (`allowedGlobalArity`). Skipping it is exactly what let a + // caller-supplied `--set`/`--profiles` hide behind an unrecognized flag's *value*. + throw new PassthroughRefusedError( + "simctl", + `Refusing \`simlock simctl ${argument}\`: \`simlock simctl\` supplies the device set itself, and does not recognize any caller-supplied argument ahead of the subcommand. Drop the flag -- the command is already scoped -- or run \`xcrun simctl\` directly if you mean to leave Simlock's set.`, + ); + } + return []; +} diff --git a/src/drivers/ios/runtime-images.ts b/src/drivers/ios/runtime-images.ts new file mode 100644 index 000000000..b0b9dd2ef --- /dev/null +++ b/src/drivers/ios/runtime-images.ts @@ -0,0 +1,200 @@ +import { + type ComponentReceipt, + type DriverComponent, + DriverCrashError, + type InstalledComponent, +} from "../../core/index.js"; +import { isRecord, type Runtime } from "./simctl-parse.js"; +import { LATEST_COMPONENT } from "./runtime-versions.js"; + +// why: Conservative estimate for a simulator runtime download+install (~7 GB observed, rounded up +// with headroom) -- reserved by `ComponentInstaller` before `xcodebuild -downloadPlatform` ever +// starts, so a full disk fails fast instead of filling up mid-download. +export const IOS_RUNTIME_MIN_FREE_BYTES = 8 * 1024 ** 3; + +// why: Where macOS's own asset daemon keeps every simulator runtime it has ever downloaded, one +// `.asset` bundle per build. `simctl runtime delete` only unregisters a runtime from +// CoreSimulator; the bundle it was mounted from stays here, marked never-collected, spending +// the same free space `IOS_RUNTIME_MIN_FREE_BYTES` measures. Nothing Simlock may do can +// reclaim it (issue #79), so the driver only ever reads this path. +export const IOS_RUNTIME_ASSET_ROOT = + "/System/Library/AssetsV2/com_apple_MobileAsset_iOSSimulatorRuntime"; + +export interface RuntimeAsset { + readonly path: string; + readonly version: string; + readonly build: string; +} + +export function runtimeMountsAsset(runtime: Runtime, asset: RuntimeAsset): boolean { + return runtime.build === undefined + ? runtime.version === asset.version + : runtime.build === asset.build; +} + +/** + * why: The build and marketing version out of an asset bundle's `Info.plist`. Read as text rather + * than through a plist parser: these are two flat string values in a file this driver must + * never write, and shelling out to `plutil` once per bundle would make a `doctor` run pay for + * every runtime ever downloaded. Both keys are read from `MobileAssetProperties` onwards, so a + * same-named key in the surrounding envelope cannot be mistaken for the asset's own. + */ +export function parseRuntimeAsset(path: string, plist: string): RuntimeAsset | undefined { + const propertiesAt = plist.indexOf("MobileAssetProperties"); + const properties = propertiesAt === -1 ? plist : plist.slice(propertiesAt); + const build = /Build<\/key>\s*([^<]+)<\/string>/.exec(properties)?.[1]; + const version = /SimulatorVersion<\/key>\s*([^<]+)<\/string>/.exec(properties)?.[1]; + + return build === undefined || version === undefined ? undefined : { build, path, version }; +} + +export interface RuntimeImage { + readonly identifier: string; + readonly runtimeIdentifier: string; + readonly build?: string; + readonly platformIdentifier?: string; + readonly version?: string; + readonly sizeBytes?: number; + readonly state?: string; +} + +export function isBeingDeleted(image: RuntimeImage): boolean { + return image.state?.toLowerCase() === "deleting"; +} + +const IOS_SIMULATOR_PLATFORM = "com.apple.platform.iphonesimulator"; + +export function parseRuntimeImages(value: unknown): readonly RuntimeImage[] { + if (!isRecord(value)) { + throw new DriverCrashError("Invalid simctl runtime list JSON: expected an object"); + } + return Object.values(value).flatMap((entry): RuntimeImage[] => + isRecord(entry) && + typeof entry.identifier === "string" && + typeof entry.runtimeIdentifier === "string" + ? [ + { + identifier: entry.identifier, + runtimeIdentifier: entry.runtimeIdentifier, + ...optionalString("build", entry.build), + ...optionalString("platformIdentifier", entry.platformIdentifier), + ...optionalString("version", entry.version), + ...optionalString("state", entry.state), + ...(isByteCount(entry.sizeBytes) ? { sizeBytes: entry.sizeBytes } : {}), + }, + ] + : [], + ); +} + +function optionalString( + key: Key, + value: unknown, +): Partial> { + return typeof value === "string" ? ({ [key]: value } as Record) : {}; +} + +function isByteCount(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +export function runtimeReceipt( + runtime: Runtime, + images: readonly RuntimeImage[], +): ComponentReceipt { + const image = images.find( + (candidate) => + candidate.runtimeIdentifier === runtime.identifier && + (runtime.build === undefined || candidate.build === runtime.build), + ); + return image === undefined + ? { build: runtime.build ?? "", runtime: runtime.identifier } + : imageReceipt(image); +} + +export function imageReceipt(image: RuntimeImage): ComponentReceipt { + return { build: image.build ?? "", image: image.identifier }; +} + +export function isIosImage(image: RuntimeImage): boolean { + return image.platformIdentifier === IOS_SIMULATOR_PLATFORM; +} + +export type ListedImage = DriverComponent & { + readonly image: RuntimeImage; + readonly unusedDevices: number; +}; + +export function leftoverDownloadsSentence( + labels: readonly string[], + bundles: number, + sizeBytes: number | undefined, +): string { + const runtimes = labels.length > 1; + const downloads = bundles > 1; + const size = + sizeBytes === undefined + ? "" + : ` (${(sizeBytes / 1024 ** 3).toFixed(1)} GiB${downloads ? " in all" : ""})`; + return ( + `iOS ${labels.join(", ")} ${runtimes ? "are" : "is"} no longer installed, but ` + + `${runtimes ? "their" : "its"} ${downloads ? "downloads" : "download"}${size} ` + + `${downloads ? "are" : "is"} still in ${IOS_RUNTIME_ASSET_ROOT}; ` + + `\`simctl runtime delete\` does not reclaim ${downloads ? "them" : "it"} and Simlock does ` + + "not delete files there -- remove the platform in Xcode's Settings -> Platforms to get " + + "the space back" + ); +} + +export function unavailableDevicesResidue(unused: number): string | undefined { + if (unused === 0) return undefined; + const subject = + unused === 1 + ? "1 never-used simulator in the default device set is" + : `${String(unused)} never-used simulators in the default device set are`; + return ( + `${subject} now unavailable; Simlock does not delete simulators there -- ` + + "`xcrun simctl delete unavailable` clears them" + ); +} + +export interface DefaultSetDevices { + readonly used: number; + readonly unused: number; +} + +export function devicesPerRuntime( + value: unknown, + owned: ReadonlySet, +): ReadonlyMap { + if (!isRecord(value) || !isRecord(value.devices)) { + throw new DriverCrashError("Invalid simctl device list JSON"); + } + return new Map( + Object.entries(value.devices).map(([runtime, devices]): [string, DefaultSetDevices] => { + const entries: readonly unknown[] = (Array.isArray(devices) ? devices : []).filter( + (device: unknown) => !(isRecord(device) && owned.has(String(device.udid))), + ); + const used = entries.filter( + (device) => !isRecord(device) || device.lastUsedAt !== undefined, + ).length; + return [runtime, { unused: entries.length - used, used }]; + }), + ); +} + +export function installedComponent( + runtime: Runtime, + images: readonly RuntimeImage[], +): InstalledComponent { + return { receipt: runtimeReceipt(runtime, images), version: runtime.version }; +} + +export function withBareMajorHint(component: string, error: unknown): unknown { + if (component === LATEST_COMPONENT || component.includes(".")) return error; + if (!(error instanceof DriverCrashError)) return error; + return new DriverCrashError( + `Could not download a default iOS runtime (tried ${component}): ${error.message}; ` + + `pass --os to request an exact release`, + ); +} diff --git a/src/drivers/ios/runtime-versions.ts b/src/drivers/ios/runtime-versions.ts new file mode 100644 index 000000000..60c6298e9 --- /dev/null +++ b/src/drivers/ios/runtime-versions.ts @@ -0,0 +1,196 @@ +import { RuntimeMissingError, UnknownModelError } from "../../core/index.js"; +import { + type DeviceType, + type Runtime, + type SimctlCatalog, + UNBOUNDED_VERSION, +} from "./simctl-parse.js"; + +export const LATEST_COMPONENT = "latest"; + +// why: `xcodebuild -downloadPlatform iOS -buildVersion` only reaches back to iOS 16.0 (Xcode +// 16.1+); older runtimes must be installed through Xcode itself. +const IOS_DOWNLOAD_FLOOR: readonly [number, number, number] = [16, 0, 0]; + +export function missingRuntime( + osVersion: string, + message: string, + component: string, +): RuntimeMissingError { + return belowDownloadFloor(component) + ? new IosDownloadFloorError(component) + : new IosRuntimeMissingError(osVersion, message, component); +} + +export function belowDownloadFloor(component: string): boolean { + return component !== LATEST_COMPONENT && isTooOldToDownload(component); +} + +class IosRuntimeMissingError extends RuntimeMissingError { + constructor(osVersion: string, message: string, component: string) { + super("ios", osVersion, { component }); + this.message = message; + } +} + +export class IosUnknownModelError extends UnknownModelError { + constructor(model: string) { + super("ios", model); + this.message = `Unknown ios model: ${model}; a newer Xcode version may add this device`; + } +} + +export class IosVersionOutOfRangeError extends RuntimeMissingError { + constructor(model: string, requested: string, deviceType: DeviceType) { + super("ios", requested, { downloadable: false }); + const range = formatVersionRange(deviceType.minRuntimeVersion, deviceType.maxRuntimeVersion); + this.message = `${model} supports iOS ${range}; iOS ${requested} is out of range`; + } +} + +export class IosRuntimeUnpairedError extends RuntimeMissingError { + constructor(model: string, requested: string) { + super("ios", requested, { downloadable: false }); + this.message = `iOS ${requested} is installed but does not support ${model}`; + } +} + +export class IosDownloadFloorError extends RuntimeMissingError { + constructor(requested: string) { + super("ios", requested, { downloadable: false }); + this.message = + `iOS ${requested} predates Xcode's automatic download support (introduced for iOS ` + + `16.0 and newer); install it manually via Xcode`; + } +} + +export class IosRuntimeUnavailableForDownloadError extends RuntimeMissingError { + constructor(requested: string) { + super("ios", requested, { downloadable: false }); + this.message = + `iOS ${requested} is not currently offered for download by Apple (likely superseded by a ` + + `newer point release); request a different --os, or omit --os for the newest available runtime`; + } +} + +export function runtimeForComponent( + catalog: SimctlCatalog, + component: string, +): Runtime | undefined { + const available = catalog.runtimes.filter((runtime) => runtime.isAvailable); + if (component === LATEST_COMPONENT) return newestRuntime(available); + if (!component.includes(".")) { + return newestRuntime( + available.filter((runtime) => runtime.version.split(".")[0] === component), + ); + } + return findInstalledRuntime(catalog, component); +} + +export function findInstalledRuntime(catalog: SimctlCatalog, version: string): Runtime | undefined { + return catalog.runtimes.find((runtime) => runtime.isAvailable && runtime.version === version); +} + +export function findDeviceType(catalog: SimctlCatalog, model: string): DeviceType | undefined { + const wanted = model.toLocaleLowerCase(); + return catalog.deviceTypes.find((candidate) => candidate.name.toLocaleLowerCase() === wanted); +} + +export function pairedInstalledRuntimes( + catalog: SimctlCatalog, + deviceType: DeviceType, +): readonly Runtime[] { + return catalog.runtimes.filter( + (runtime) => + runtime.isAvailable && + runtime.supportedDeviceTypeIds.has(deviceType.identifier) && + isVersionInRange(runtime.version, deviceType), + ); +} + +export function findPairedRuntime( + catalog: SimctlCatalog, + deviceType: DeviceType, + version: string, +): Runtime | undefined { + return pairedInstalledRuntimes(catalog, deviceType).find( + (runtime) => runtime.version === version, + ); +} + +export function pairedVersions( + catalog: SimctlCatalog, + deviceType: DeviceType | undefined, +): string[] { + if (deviceType === undefined) return []; + return [ + ...new Set(pairedInstalledRuntimes(catalog, deviceType).map((runtime) => runtime.version)), + ]; +} + +function decodeVersionTriple(encoded: number): readonly [number, number, number] { + return [(encoded >> 16) & 0xff, (encoded >> 8) & 0xff, encoded & 0xff]; +} + +function formatDecodedVersion(encoded: number): string { + const [major, minor, patch] = decodeVersionTriple(encoded); + return patch === 0 ? `${major}.${minor}` : `${major}.${minor}.${patch}`; +} + +export function isUnboundedMax(encoded: number): boolean { + return encoded >= UNBOUNDED_VERSION; +} + +function formatVersionRange(min: number, max: number): string { + const minLabel = formatDecodedVersion(min); + return isUnboundedMax(max) ? `${minLabel}+` : `${minLabel}-${formatDecodedVersion(max)}`; +} + +function versionTriple(version: string): readonly [number, number, number] { + const parts = version.split(".").map(versionPart); + return [parts[0] ?? 0, parts[1] ?? 0, parts[2] ?? 0]; +} + +function versionOrdinal(triple: readonly [number, number, number]): number { + return triple[0] * 1_000_000 + triple[1] * 1_000 + triple[2]; +} + +export function isVersionInRange(version: string, deviceType: DeviceType): boolean { + const ordinal = versionOrdinal(versionTriple(version)); + return ( + ordinal >= versionOrdinal(decodeVersionTriple(deviceType.minRuntimeVersion)) && + ordinal <= versionOrdinal(decodeVersionTriple(deviceType.maxRuntimeVersion)) + ); +} + +function isTooOldToDownload(version: string): boolean { + return versionOrdinal(versionTriple(version)) < versionOrdinal(IOS_DOWNLOAD_FLOOR); +} + +export function majorVersionString(encoded: number): string { + return String(decodeVersionTriple(encoded)[0]); +} + +export function newestRuntime(runtimes: readonly Runtime[]): Runtime | undefined { + return [...runtimes].sort((left, right) => compareVersions(left.version, right.version)).at(-1); +} + +export function compareVersions(left: string, right: string): number { + const leftParts = left.split(".").map(versionPart); + const rightParts = right.split(".").map(versionPart); + const partCount = Math.max(leftParts.length, rightParts.length); + + for (let index = 0; index < partCount; index += 1) { + const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); + if (difference !== 0) { + return difference; + } + } + + return left.localeCompare(right); +} + +function versionPart(part: string): number { + const parsed = Number.parseInt(part, 10); + return Number.isNaN(parsed) ? 0 : parsed; +} diff --git a/src/drivers/ios/simulator-data.ts b/src/drivers/ios/simulator-data.ts new file mode 100644 index 000000000..89acb6fe3 --- /dev/null +++ b/src/drivers/ios/simulator-data.ts @@ -0,0 +1,39 @@ +import { type DriverDevice, DriverCrashError } from "../../core/index.js"; +import { isRecord } from "./simctl-parse.js"; + +export interface IosDriverData { + readonly deviceTypeId: string; + readonly name: string; + readonly runtimeId: string; + readonly udid: string; +} + +export function iosDriverData(device: DriverDevice): IosDriverData { + const value = device.driverData; + if ( + !isRecord(value) || + typeof value.udid !== "string" || + typeof value.deviceTypeId !== "string" || + typeof value.runtimeId !== "string" || + typeof value.name !== "string" + ) { + throw new DriverCrashError(`Invalid iOS driver data for device ${device.deviceId}`); + } + + return { + deviceTypeId: value.deviceTypeId, + name: value.name, + runtimeId: value.runtimeId, + udid: value.udid, + // why: Both are optional: absent or wrong-typed is ignored rather than rejected. A `full` key + // an older daemon stamped here is not read: the spec's mode is the only record of it (ADR 0007 §7). + ...(typeof value.slimSignature === "string" ? { slimSignature: value.slimSignature } : {}), + ...(typeof value.slimMarkToken === "string" ? { slimMarkToken: value.slimMarkToken } : {}), + }; +} + +export function requireIosPlatform(platform: string): void { + if (platform !== "ios") { + throw new DriverCrashError(`iOS simctl driver cannot handle ${platform} device requests`); + } +} From d26674308631cd4f3db612e6e987d806098e1733 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:13:36 +0100 Subject: [PATCH 13/83] refactor(leasing): split lease-acquisition-coordinator.ts into parts --- src/core/device-provisioner.test.ts | 2 +- src/leasing/acquisition-admission.ts | 135 ++ src/leasing/acquisition-errors.ts | 42 + src/leasing/acquisition-gates.ts | 43 + src/leasing/acquisition-ports.ts | 89 ++ src/leasing/acquisition-spec-resolver.ts | 350 ++++ src/leasing/acquisition-waiter.ts | 82 + src/leasing/acquisition-workflow.ts | 696 ++++++++ src/leasing/lease-acquisition-coordinator.ts | 1497 +----------------- src/leasing/waiter-outcomes.ts | 143 ++ src/source-hygiene.test.ts | 1 - 11 files changed, 1652 insertions(+), 1428 deletions(-) create mode 100644 src/leasing/acquisition-admission.ts create mode 100644 src/leasing/acquisition-errors.ts create mode 100644 src/leasing/acquisition-gates.ts create mode 100644 src/leasing/acquisition-ports.ts create mode 100644 src/leasing/acquisition-spec-resolver.ts create mode 100644 src/leasing/acquisition-waiter.ts create mode 100644 src/leasing/acquisition-workflow.ts create mode 100644 src/leasing/waiter-outcomes.ts diff --git a/src/core/device-provisioner.test.ts b/src/core/device-provisioner.test.ts index 77b987dcf..4fadff3d4 100644 --- a/src/core/device-provisioner.test.ts +++ b/src/core/device-provisioner.test.ts @@ -455,7 +455,7 @@ describe("DeviceProvisioner and a component removal (ADR 0010 §8)", () => { // The driver call itself. [join("core", "device-provisioner.ts")]: 1, // `provisioner.provision(...)`: the lease path asking this module, not a driver. - [join("leasing", "lease-acquisition-coordinator.ts")]: 1, + [join("leasing", "acquisition-workflow.ts")]: 1, // The same, for a device the warm pool creates for a target. [join("core", "warm-pool", "converger.ts")]: 1, // why: not a second creator: the Android driver's own `provision` hands the call to its seeds diff --git a/src/leasing/acquisition-admission.ts b/src/leasing/acquisition-admission.ts new file mode 100644 index 000000000..4b6a31e90 --- /dev/null +++ b/src/leasing/acquisition-admission.ts @@ -0,0 +1,135 @@ +import { leaseEndedError } from "../contract/index.js"; +import { type DeviceRequest } from "../core/index.js"; +import { type LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; +import { emitLeaseRefusal } from "./lease-refusal.js"; +import { + LeaseIdTakenError, + type LeaseRequestOptions, + requesterLimitDecision, +} from "./wait-queue.js"; + +export type AcquisitionAdmissionOptions = Pick< + LeaseAcquisitionCoordinatorOptions, + "eventBus" | "maxPerRequester" | "queue" | "registry" | "requests" +>; + +/** why: the admission checks that refuse a request before it queues; each emits its refusal and then throws, and the class holds no state. */ +export class AcquisitionAdmission { + constructor(private readonly options: AcquisitionAdmissionOptions) {} + + refuseIfRequesterAtLimit( + request: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { devices, leases } = this.options.registry.snapshot; + const held = leases + .filter((lease) => lease.requesterId === options.requesterId) + .map((lease) => { + const platform = devices.find((device) => device.id === lease.deviceId)?.spec.platform; + return { + grantedAt: lease.grantedAt, + id: lease.id, + ...(platform === undefined ? {} : { platform }), + }; + }); + const waiting = this.options.queue.pendingPlatforms(options.requesterId); + // why: a probe's held leases were counted by the gateway, fleet-wide; only its waiting + // request is checked here (ADR 0023 §2). + const refusal = requesterLimitDecision({ + held, + maxPerRequester: this.options.maxPerRequester, + platform: request.platform, + probe: options.fleetRequestId !== undefined, + requesterId: options.requesterId, + waiting, + }); + if (refusal === undefined) return; + emitLeaseRefusal( + this.options.eventBus, + { + requestId, + requester: options.requesterId, + requestSpec: request, + reason: "already-leased", + fleetRequestId: options.fleetRequestId, + }, + "lease-acquisition-coordinator", + ); + throw refusal; + } + + /** + * ADR 0020: the one clash check on a host, inside the same admission section as the + * one-lease-per-requester check and after it. An ID is held by an active lease and by a + * request still waiting, so a restart, which fails every waiting request, frees it. + */ + refuseIfLeaseIdTaken( + request: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + if ( + !this.options.registry.snapshot.leases.some((lease) => lease.id === leaseId) && + !this.options.requests.holdsLeaseId(leaseId) + ) { + return; + } + emitLeaseRefusal( + this.options.eventBus, + { + requestId, + requester: options.requesterId, + requestSpec: request, + reason: "lease-id-taken", + fleetRequestId: options.fleetRequestId, + }, + "lease-acquisition-coordinator", + ); + throw new LeaseIdTakenError(leaseId); + } + + /** why: ADR 0074 §4: a chosen id whose lease ended within `lease.endedIdRetentionMs` is refused, after the taken-id check so a live id stays `LEASE_ID_TAKEN`. */ + refuseIfLeaseIdEnded( + request: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + const endedAt = this.options.registry.endedLeaseId(leaseId); + if (endedAt === undefined) return; + emitLeaseRefusal( + this.options.eventBus, + { + requestId, + requester: options.requesterId, + requestSpec: request, + reason: "lease-ended", + fleetRequestId: options.fleetRequestId, + }, + "lease-acquisition-coordinator", + ); + throw leaseEndedError(leaseId, endedAt); + } + + refuseForUpdate( + requestId: string, + request: DeviceRequest, + options: Pick, + ): void { + emitLeaseRefusal( + this.options.eventBus, + { + requestId, + requester: options.requesterId, + requestSpec: request, + reason: "updating", + fleetRequestId: options.fleetRequestId, + }, + "lease-acquisition-coordinator", + ); + } +} diff --git a/src/leasing/acquisition-errors.ts b/src/leasing/acquisition-errors.ts new file mode 100644 index 000000000..9b8621710 --- /dev/null +++ b/src/leasing/acquisition-errors.ts @@ -0,0 +1,42 @@ +import { noPhysicalMatch, type PhysicalRequest, UnknownModelError } from "../core/index.js"; +import { type WaitingReason } from "./acquisition-planner.js"; + +export class NoCapacityError extends Error { + constructor(readonly reason?: WaitingReason) { + super("No device capacity is currently available"); + this.name = "NoCapacityError"; + } +} + +export class ProvisionFailedError extends Error { + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause), { cause }); + this.name = "ProvisionFailedError"; + } +} + +/** why: mapped to `DAEMON_UPDATING`; the message is what a refused agent reads. */ +export class DaemonUpdatingError extends Error { + constructor() { + super("this daemon is updating and takes no new leases"); + this.name = "DaemonUpdatingError"; + } +} + +export class NukeCancelledError extends Error { + constructor() { + super("Request cancelled by nuke"); + this.name = "NukeCancelledError"; + } +} + +export class PhysicalUnknownModelError extends UnknownModelError { + constructor(request: PhysicalRequest) { + super(request.platform, request.model, request.model === undefined ? request.class : undefined); + this.message = noPhysicalMatch(request).message; + } +} + +export function asError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/src/leasing/acquisition-gates.ts b/src/leasing/acquisition-gates.ts new file mode 100644 index 000000000..347bbbfd4 --- /dev/null +++ b/src/leasing/acquisition-gates.ts @@ -0,0 +1,43 @@ +/** why: the three flags that gate admission and planning, each changed by one named method, so no other part writes them. */ +export class AcquisitionGates { + #admissionClosed = false; + // why: apart from #admissionClosed, which also stops the warm pool; an update refuses leases and leaves the pool running. + #updateHold = false; + // why: set only by a stop (`settleAcquisitions`), never cleared; unlike maintenance, which a nuke ends, nothing reopens planning after it. + #stopPlanning = false; + #maintenanceDepth = 0; + + get admissionClosed(): boolean { + return this.#admissionClosed; + } + + get updateHold(): boolean { + return this.#updateHold; + } + + get stopPlanning(): boolean { + return this.#stopPlanning; + } + + enterMaintenance(): void { + this.#maintenanceDepth += 1; + this.#admissionClosed = true; + } + + leaveMaintenance(): void { + this.#maintenanceDepth = Math.max(0, this.#maintenanceDepth - 1); + this.#admissionClosed = this.#maintenanceDepth > 0; + } + + holdForUpdate(): void { + this.#updateHold = true; + } + + releaseUpdateHold(): void { + this.#updateHold = false; + } + + stopPlanningForGood(): void { + this.#stopPlanning = true; + } +} diff --git a/src/leasing/acquisition-ports.ts b/src/leasing/acquisition-ports.ts new file mode 100644 index 000000000..5abc36814 --- /dev/null +++ b/src/leasing/acquisition-ports.ts @@ -0,0 +1,89 @@ +import { type EventBus } from "../bus/index.js"; +import { + type BaselineService, + type CatalogReader, + type ComponentInstaller, + type Core, + type DeviceMode, + type DeviceOperationClaims, + type DeviceProvisioner, + type DeviceRecord, + type DriverCatalog, + type LeaseGrant, + type LeaseRecord, + type ManagedDeviceLifecycle, + type MaxPerRequester, + type ModelPreferences, + type Platform, + type SerializedDecision, + type VirtualDevice, +} from "../core/index.js"; +import { type IdGenerator, type Logger } from "../ports/index.js"; +import { type AcquisitionPlanner } from "./acquisition-planner.js"; +import { type LeaseLifecycle } from "./lease-lifecycle.js"; +import { type LeaseRequestBook } from "./lease-request-book.js"; +import { type WaitQueue } from "./wait-queue.js"; + +export interface LeaseAcquisitionRegistry { + readonly snapshot: { + readonly devices: readonly DeviceRecord[]; + readonly virtualDevices: readonly VirtualDevice[]; + readonly leases: readonly LeaseRecord[]; + }; + endedLeaseId(leaseId: string): number | undefined; +} + +export type AcquisitionClaims = Pick; +export type AcquisitionQuarantine = Pick; +export type AcquisitionDecision = Pick; +export type AcquisitionDrivers = Pick; +export type AcquisitionPlannerPort = Pick; +export type AcquisitionQueue = Pick< + WaitQueue, + | "create" + | "cancelAll" + | "depth" + | "enqueue" + | "findPendingWaiter" + | "pendingPlatforms" + | "head" + | "isQueued" + | "markNew" + | "markProcessing" + | "notifyProgress" + | "pending" + | "physicalDepth" + | "reject" + | "resolve" +>; + +export interface LeaseAcquisitionCoordinatorOptions { + readonly baselines: Pick; + readonly catalog: Pick; + readonly claims: AcquisitionClaims; + readonly components: Pick; + readonly decisions: AcquisitionDecision; + readonly defaultModes: Readonly>>; + readonly drivers: AcquisitionDrivers; + readonly eventBus: Pick; + /** Mints a request's id before admission checks, so a request refused there still has one. */ + readonly idGenerator: IdGenerator; + readonly leases: Pick; + readonly lifecycle: Pick< + ManagedDeviceLifecycle, + "bootForLease" | "destroy" | "dispose" | "rebootForLease" | "shutdown" + >; + /** + * The model names to try for each class, per platform, operator's list first (ADR 0015 §4), + * built at the composition root. This coordinator holds no model name of its own. + */ + readonly modelPreferences: ModelPreferences; + readonly planner: AcquisitionPlannerPort; + readonly provisioner: Pick; + readonly queue: AcquisitionQueue; + readonly quarantine: AcquisitionQuarantine; + readonly registry: LeaseAcquisitionRegistry; + readonly logger?: Logger; + readonly requests: Pick, "admit" | "holdsLeaseId" | "replay">; + readonly maxPerRequester?: MaxPerRequester; +} diff --git a/src/leasing/acquisition-spec-resolver.ts b/src/leasing/acquisition-spec-resolver.ts new file mode 100644 index 000000000..3124cc75d --- /dev/null +++ b/src/leasing/acquisition-spec-resolver.ts @@ -0,0 +1,350 @@ +import { type OsRange, requestedClass } from "../contract/index.js"; +import { + BadDeviceRequestError, + baselineMismatch, + BaselineMismatchError, + BaselineStaleError, + classCandidates, + type CurrentBaseline, + type DeviceClass, + type DeviceMode, + type DeviceRequest, + type DeviceRequirement, + type Driver, + type DriverCatalogEntry, + type ExactDeviceRequest, + exactRequirement, + findCatalogModel, + modelClass, + type Platform, + RuntimeMissingError, + type TargetResolution, + UnknownBaselineError, + UnknownModelError, + type VirtualSpec, +} from "../core/index.js"; +import { asError } from "./acquisition-errors.js"; +import { type LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; +import { + downloadingProgress, + type DownloadingProgress, + fullSpec, + matchingRuntimes, + newestRuntime, + requestedRange, + requiredOs, + sameDownloadingProgress, +} from "./acquisition-requests.js"; +import { type LeaseRequestOptions } from "./wait-queue.js"; + +export type AcquisitionSpecResolverOptions = Pick< + LeaseAcquisitionCoordinatorOptions, + "baselines" | "catalog" | "components" | "defaultModes" | "drivers" | "modelPreferences" +>; + +/** why: turns a request into the exact spec a driver resolves, and a baseline request into that baseline's spec, so the root and the workflow never resolve a spec themselves. */ +export class AcquisitionSpecResolver { + constructor(private readonly options: AcquisitionSpecResolverOptions) {} + + defaultMode(platform: Platform): DeviceMode { + return this.options.defaultModes[platform] ?? "full"; + } + + async resolveSpec( + driver: Driver, + request: DeviceRequest, + options: LeaseRequestOptions, + onProgress: (progress: DownloadingProgress) => void, + ): Promise<{ + readonly spec: VirtualSpec; + readonly requirement: DeviceRequirement; + readonly classOf: ((model: string) => DeviceClass | undefined) | undefined; + }> { + if (request.baseline !== undefined) { + return this.#resolveBaseline(driver, request.baseline, request); + } + // The one place a request with no mode gets the worker's default (ADR 0007 §2). + const mode = request.mode ?? this.defaultMode(request.platform); + if (request.except !== undefined && mode !== "slim") { + throw new BadDeviceRequestError( + "except needs a slim lease (this worker's default mode is full)", + ); + } + const range = requestedRange(request); + const target = + range === undefined + ? (this.#exactTarget(request, mode) ?? (await this.#resolveClass(request, mode))) + : await this.#resolveRanged(request, mode, range); + // A range was settled against the catalog above, so it never installs (ADR 0015 §5). + const resolved = await this.#resolveOrInstall(onProgress, driver, target.exact, { + ...options, + allowDownload: options.allowDownload === true && range === undefined, + }); + const spec = checkedSpec(resolved, request, mode); + return { + classOf: target.classOf, + requirement: target.requirement ?? exactRequirement(spec), + spec, + }; + } + + /** why: a request naming a baseline takes the baseline's own platform, model, OS, mode and image tag; no download is ever attempted. */ + async #resolveBaseline( + driver: Driver, + name: string, + request: DeviceRequest, + ): Promise<{ + readonly spec: VirtualSpec; + readonly requirement: DeviceRequirement; + readonly classOf: undefined; + }> { + const current = this.options.baselines.current(name); + if (current === undefined) throw new UnknownBaselineError(name); + const entry = (await this.options.catalog.listCatalog(current.platform))[0]; + const mismatch = baselineMismatch(current, request, entry ?? { modelAliases: {}, models: [] }); + if (mismatch !== undefined) { + throw new BaselineMismatchError(name, mismatch.field, mismatch.message); + } + if (current.staleReason !== undefined) throw new BaselineStaleError(name, current.staleReason); + const spec = baselineSpec(current); + if (request.except !== undefined && spec.mode !== "slim") { + throw new BadDeviceRequestError(`except needs a slim lease (baseline "${name}" is full)`); + } + if (request.except !== undefined) { + // why: the driver judges the names (ADR 0066 §2); its answer is dropped, the baseline's spec stands. + await driver.resolveSpec({ + except: request.except, + mode: "slim", + model: spec.model, + osVersion: spec.osVersion, + platform: spec.platform, + ...(spec.imageTag === undefined ? {} : { imageTag: spec.imageTag }), + }); + } + return { classOf: undefined, requirement: exactRequirement(spec), spec }; + } + + /** + * The exact request an exact-model request already is, answered without awaiting anything so + * its path takes no extra turn of the event loop; `undefined` for one that names no model. + */ + #exactTarget(request: DeviceRequest, mode: DeviceMode): ResolvedTarget | undefined { + const { class: _class, model, ...rest } = request; + return model === undefined ? undefined : { exact: { ...rest, mode, model } }; + } + + /** + * Turns a request that names no model, or names an OS range, into the exact one a driver + * resolves (ADR 0015 §5). A class, named or meant by naming nothing, becomes the first model + * on its preference list that this host's catalog lists, classes as that class, and pairs with + * an installed runtime (of the requested image tag, and in the requested range, when it names + * them); an exact model with a range must be listed, or the request fails as + * `UnknownModelError`. With a range the runtime is the newest paired one in it, passed on as an + * exact version. The catalog is read, and nothing is downloaded. A class with no listed model + * fails as `UnknownModelError`, and a model or class with no pairing as a `RuntimeMissingError` + * no download can fix. + */ + async #resolveClass( + request: DeviceRequest, + mode: DeviceMode, + range?: OsRange, + ): Promise { + const deviceClass = requestedClass({ class: request.class }); + const entry = (await this.options.catalog.listCatalog(request.platform))[0]; + if (entry === undefined) throw new UnknownModelError(request.platform, undefined, deviceClass); + const candidates = classCandidates( + entry, + deviceClass, + this.options.modelPreferences[request.platform]?.[deviceClass] ?? [], + ); + if (candidates.length === 0) { + throw new UnknownModelError(request.platform, undefined, deviceClass); + } + const chosen = candidates.find( + (candidate) => matchingRuntimes(entry, candidate, request, range).length > 0, + ); + if (chosen === undefined) { + throw new RuntimeMissingError(request.platform, request.osVersion ?? "default"); + } + return { + ...this.#rangedTarget( + entry, + chosen, + { class: deviceClass, kind: "class" }, + request, + mode, + range, + ), + classOf: (candidate) => modelClass(entry, candidate), + }; + } + + /** A request that names a model and an OS range: the model must be listed (ADR 0015 §5). */ + async #resolveRanged( + request: DeviceRequest, + mode: DeviceMode, + range: OsRange, + ): Promise { + if (request.model === undefined) return this.#resolveClass(request, mode, range); + const entry = (await this.options.catalog.listCatalog(request.platform))[0]; + if (entry === undefined) throw new UnknownModelError(request.platform, request.model); + const listed = findCatalogModel(entry, request.model); + if (listed === undefined) throw new UnknownModelError(request.platform, request.model); + return this.#rangedTarget( + entry, + listed, + { kind: "model", model: listed }, + request, + mode, + range, + ); + } + + /** The target for `model` once the request's runtime constraints are settled against `entry`. */ + #rangedTarget( + entry: DriverCatalogEntry, + model: string, + target: DeviceRequirement["target"], + request: DeviceRequest, + mode: DeviceMode, + range: OsRange | undefined, + ): ResolvedTarget { + const { class: _class, model: _model, ...rest } = request; + return { + exact: { + ...rest, + mode, + model, + ...(range === undefined ? {} : { osVersion: newestRuntime(entry, model, request, range) }), + }, + requirement: { + imageTag: request.imageTag, + osVersion: requiredOs(entry, request, range), + platform: request.platform, + target, + }, + }; + } + + /** + * Resolves the request; when its runtime is missing, a download could supply it, and the + * request may download, asks the installer for it and resolves once more. The installer skips + * the install when, by the time this call reaches the front of its queue, `resolveSpec` no + * longer fails with `RuntimeMissingError` -- another install made the runtime available. + * Without `allowDownload` the first failure stands and the installer is never called. + * + * The installer's reports reach the requester as the `downloading` stage, through the queue + * like every other stage. A report equal to the last one sent is skipped, so a percentage is + * sent once per whole number. + */ + async #resolveOrInstall( + onProgress: (progress: DownloadingProgress) => void, + driver: Driver, + request: ExactDeviceRequest, + options: LeaseRequestOptions, + ): Promise { + try { + return await driver.resolveSpec(request); + } catch (error: unknown) { + if ( + !(error instanceof RuntimeMissingError) || + !error.downloadable || + error.component === undefined || + options.allowDownload !== true + ) { + throw error; + } + const { component } = error; + let lastSent: DownloadingProgress | undefined; + await this.options.components.install({ + component, + onProgress: (progress) => { + const next = downloadingProgress(component, progress); + if (lastSent !== undefined && sameDownloadingProgress(lastSent, next)) return; + lastSent = next; + onProgress(next); + }, + platform: request.platform, + requesterId: options.requesterId, + stillNeeded: async () => { + try { + await driver.resolveSpec(request); + return false; + } catch (stillMissing: unknown) { + return stillMissing instanceof RuntimeMissingError; + } + }, + }); + return driver.resolveSpec(request); + } + } + + /** + * Resolves a request to the spec it would be granted a device of, with downloads off, or says + * why there is none. A read: nothing is queued, reserved or installed. The warm pool resolves + * its targets here, so a target and a request become a spec in one place. + */ + async resolve(request: DeviceRequest): Promise { + let driver: Driver; + try { + driver = this.options.drivers.get(request.platform); + } catch (error: unknown) { + return { message: asError(error).message, refusal: "no-driver" }; + } + try { + const { spec } = await this.resolveSpec( + driver, + request, + { allowDownload: false, ownerId: "warm-pool", requesterId: "warm-pool" }, + () => undefined, + ); + return { spec }; + } catch (error: unknown) { + return { message: asError(error).message, refusal: refusalOf(error) }; + } + } +} + +/** why: the exact spec of a baseline's current version, which a request naming the baseline stands for. */ +export function baselineSpec(current: CurrentBaseline): VirtualSpec { + return { + baseline: { name: current.name, version: current.version }, + model: current.model, + osVersion: current.osVersion, + physical: false, + platform: current.platform, + ...(current.mode === "slim" ? { mode: "slim" as const } : {}), + ...(current.imageTag === undefined ? {} : { imageTag: current.imageTag }), + }; +} + +/** + * The spec a driver resolved, held to what the request asked. The image tag is the request's, or + * none: a driver that returns another, or one the request did not name, would plan the device + * into a pool the request did not ask for. Full is a guarantee (ADR 0007 §5): a slim spec is + * accepted only for a slim request, so a driver that returns the wrong thing still cannot put a + * full request on a slim device. + */ +function checkedSpec(resolved: VirtualSpec, request: DeviceRequest, mode: DeviceMode): VirtualSpec { + if (resolved.imageTag !== request.imageTag) { + throw new Error( + `The ${request.platform} driver resolved image tag ${String(resolved.imageTag)} ` + + `for a request naming ${String(request.imageTag)}`, + ); + } + return mode === "slim" ? resolved : fullSpec(resolved); +} + +/** What a target's refusal is called: the kinds a pool can act on, and any other as is. */ +function refusalOf(error: unknown): Exclude["refusal"] { + if (error instanceof RuntimeMissingError) return "runtime-missing"; + if (error instanceof UnknownModelError) return "unknown-model"; + return "unresolvable"; +} + +/** What `#resolveClass` settles on; an exact request has only `exact`. */ +interface ResolvedTarget { + readonly exact: ExactDeviceRequest; + readonly requirement?: DeviceRequirement; + readonly classOf?: (model: string) => DeviceClass | undefined; +} diff --git a/src/leasing/acquisition-waiter.ts b/src/leasing/acquisition-waiter.ts new file mode 100644 index 000000000..d1c1e73db --- /dev/null +++ b/src/leasing/acquisition-waiter.ts @@ -0,0 +1,82 @@ +import { type EventMap } from "../bus/index.js"; +import { + type DeviceClass, + type DeviceRequirement, + type Driver, + type LeaseTiming, + type VirtualSpec, +} from "../core/index.js"; +import { type AcquisitionPlan, type WaitingReason } from "./acquisition-planner.js"; +import { type Waiter } from "./wait-queue.js"; + +export interface AcquisitionWaiter extends Waiter { + failures: number; + classOf?: ((model: string) => DeviceClass | undefined) | undefined; + requirement?: DeviceRequirement | undefined; + spec?: VirtualSpec; + timing: LeaseTiming; +} + +export type GrantingPlanKind = + | "grant-ready" + | "grant-physical" + | "boot-shutdown" + | "reboot-ready" + | "provision"; + +export const GRANT_SOURCE: Readonly> = + { + "boot-shutdown": "booted", + "grant-physical": "warm", + "grant-ready": "warm", + provision: "provisioned", + "reboot-ready": "booted", + }; + +export type OperationPlan = Exclude< + AcquisitionPlan, + { readonly kind: "grant-ready" | "grant-physical" | "wait" | "no-capacity" } +>; + +export const noTiming: LeaseTiming = { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, +}; + +/** why: an `except` list, or a driver that must boot a ready device again before it can be granted, sends the device through the claimed reboot. */ +export function rebootsBeforeGrant(waiter: AcquisitionWaiter, driver: Driver): boolean { + return exceptOption(waiter).except !== undefined || driver.rebootReadyBeforeGrant === true; +} + +/** why: a request's `except` reaches the driver and the device only when the resolved spec is slim; on a runtime that cannot be slimmed the lease is granted full with no `except` (ADR 0066 §1). */ +export function exceptOption(waiter: AcquisitionWaiter): { readonly except?: readonly string[] } { + const except = waiter.request.except; + return except === undefined || waiter.spec?.mode !== "slim" ? {} : { except }; +} + +/** + * why: a plan handed out the capacity or device claim its operation needs; dropping the plan has + * to give it back, or the resource stays held with nobody left to release it. + */ +export function releasePlanResources(plan: OperationPlan): void { + if (plan.kind === "provision") plan.reservation.release(); + else if (plan.kind === "boot-shutdown") { + plan.capacityReservation.release(); + plan.claim.release(); + } else plan.claim.release(); +} + +export function blockedReason(plan: AcquisitionPlan): WaitingReason { + return "reason" in plan ? plan.reason : "no-free-physical"; +} + +export function operationPlan(plan: AcquisitionPlan): OperationPlan | undefined { + return plan.kind === "grant-ready" || + plan.kind === "grant-physical" || + plan.kind === "wait" || + plan.kind === "no-capacity" + ? undefined + : plan; +} diff --git a/src/leasing/acquisition-workflow.ts b/src/leasing/acquisition-workflow.ts new file mode 100644 index 000000000..eca3f0dd9 --- /dev/null +++ b/src/leasing/acquisition-workflow.ts @@ -0,0 +1,696 @@ +import { + BootTimeoutError, + type CapacityReservation, + ComponentBeingRemovedError, + type DeviceOperationClaim, + exactRequirement, + fitsPhysical, + type PhysicalDevice, + type PhysicalRequest, + readPhysicalPresence, + type ReadyDeviceHandoff, + UnknownBaselineError, + type VirtualDevice, +} from "../core/index.js"; +import { type Logger } from "../ports/index.js"; +import { type AcquisitionAdmission } from "./acquisition-admission.js"; +import { + DaemonUpdatingError, + NukeCancelledError, + PhysicalUnknownModelError, + ProvisionFailedError, +} from "./acquisition-errors.js"; +import { type AcquisitionGates } from "./acquisition-gates.js"; +import { + type AcquisitionPlan, + type PhysicalPlanInput, + type WaitingReason, +} from "./acquisition-planner.js"; +import { type LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; +import { physicalRequestOf } from "./acquisition-requests.js"; +import { baselineSpec } from "./acquisition-spec-resolver.js"; +import { estimatedBootTiming, estimatedTiming, grantReadyTiming } from "./acquisition-timing.js"; +import { + type AcquisitionWaiter, + blockedReason, + exceptOption, + type GrantingPlanKind, + type OperationPlan, + operationPlan, + rebootsBeforeGrant, + releasePlanResources, +} from "./acquisition-waiter.js"; +import { lineOf } from "./wait-queue.js"; +import { type WaiterOutcomes } from "./waiter-outcomes.js"; + +export type AcquisitionWorkflowOptions = Pick< + LeaseAcquisitionCoordinatorOptions, + | "baselines" + | "claims" + | "decisions" + | "drivers" + | "lifecycle" + | "planner" + | "provisioner" + | "queue" + | "quarantine" + | "registry" +> & { + readonly admission: AcquisitionAdmission; + readonly gates: AcquisitionGates; + readonly log: Logger; + readonly outcomes: WaiterOutcomes; +}; + +/** why: the lease workflows that drive a waiter from plan to grant, one claimed device step at a time; they reach the outcomes, gates and admission through the options, and own only the in-flight sets. */ +export class AcquisitionWorkflow { + readonly #activeWorkflows = new Set>(); + readonly #driving = new WeakSet(); + readonly #planAgain = new WeakSet(); + + constructor(private readonly options: AcquisitionWorkflowOptions) {} + + async waitForWorkflows(): Promise { + while (this.#activeWorkflows.size > 0) { + await Promise.allSettled(this.#activeWorkflows); + } + } + + // why: every waiter passes `#nextPlan` before a provision starts, in whatever state `markNew` or `#defer` left it, so the update hold refuses here (#825 Decisions). + refuseUnderHold(waiter: AcquisitionWaiter): boolean { + if (!this.options.gates.updateHold) return false; + if (this.options.queue.reject(waiter, new DaemonUpdatingError())) { + this.options.admission.refuseForUpdate(waiter.id, waiter.request, waiter.options); + } + return true; + } + + async drive(waiter: AcquisitionWaiter): Promise { + if (this.#driving.has(waiter)) return; + this.#driving.add(waiter); + try { + if (waiter.request.physical === true) { + await this.drivePhysical(waiter); + return; + } + const action = await this.options.decisions.run(async () => this.decide(waiter)); + await this.perform(waiter, action); + } finally { + this.#driving.delete(waiter); + // why: the wake inside the drive that discarded a superseded copy found the waiter still driving, so nothing else would plan it again. + if (this.#planAgain.delete(waiter)) this.wakeQueue(); + } + } + + async drivePhysical(waiter: AcquisitionWaiter): Promise { + const absent = new Set(); + for (;;) { + const plan = await this.options.decisions.run(async () => + this.decidePhysical(waiter, absent), + ); + if (plan === undefined) return; + if (!(await this.grantPhysical(waiter, plan, absent))) return; + } + } + + decidePhysical( + waiter: AcquisitionWaiter, + absent: ReadonlySet, + ): Extract | undefined { + const request = physicalRequestOf(waiter.request); + const enrolled = this.options.registry.snapshot.devices; + if ( + !enrolled.some( + (device) => device.physical && device.state !== "deleted" && fitsPhysical(device, request), + ) + ) { + this.options.outcomes.reject( + waiter, + new PhysicalUnknownModelError(request), + "unresolvable-spec", + ); + return undefined; + } + const plan = this.options.planner.planPhysical({ + absent, + heldByOlder: this.heldByOlder(waiter), + noWait: waiter.options.noWait ?? false, + owner: waiter.id, + request, + snapshot: this.options.registry.snapshot, + }); + if (plan.kind !== "grant-physical") { + this.options.outcomes.defer(waiter, blockedReason(plan)); + return undefined; + } + if (!this.options.queue.markProcessing(waiter)) { + plan.claim.release(); + return undefined; + } + return plan; + } + + heldByOlder(waiter: AcquisitionWaiter): PhysicalPlanInput["heldByOlder"] { + const older: PhysicalRequest[] = []; + for (const other of this.options.queue.pending()) { + if (other === waiter) break; + if (lineOf(other.request) === "physical" && other.state === "queued") { + older.push(physicalRequestOf(other.request)); + } + } + return (device: PhysicalDevice) => older.some((request) => fitsPhysical(device, request)); + } + + async grantPhysical( + waiter: AcquisitionWaiter, + plan: Extract, + absent: Set, + ): Promise { + const { claim, device } = plan; + let present = false; + try { + const reads = await readPhysicalPresence(this.options.drivers.get(device.spec.platform), [ + device.driverDeviceId, + ]); + present = reads.get(device.driverDeviceId) === true; + } catch (error: unknown) { + this.options.outcomes.logFailure( + "reading a physical device's presence failed", + waiter, + device, + "presence", + error, + ); + } + let replan = false; + await this.options.decisions.run(async () => { + try { + if (!this.options.queue.markProcessing(waiter)) return; + const current = this.options.registry.snapshot.devices.find( + (candidate) => candidate.id === device.id, + ); + if (!present || !this.options.claims.isActive(claim) || current?.state !== "ready") { + absent.add(device.id); + replan = true; + return; + } + await this.options.outcomes.grant(waiter, current, plan.kind); + } catch (error: unknown) { + this.options.outcomes.logFailure( + "granting a physical device failed", + waiter, + device, + "grant", + error, + ); + this.options.outcomes.defer(waiter, "no-free-physical"); + } finally { + claim.release(); + } + }); + if (!replan) this.wakeQueue(); + return replan; + } + + async perform(waiter: AcquisitionWaiter, action: OperationPlan | undefined): Promise { + if (action === undefined) return; + if (action.kind === "provision") { + const spec = waiter.spec; + if (spec === undefined) { + action.reservation.release(); + return; + } + waiter.timing = estimatedTiming(this.options.drivers.get(spec.platform), spec); + await this.provision(waiter, action.reservation); + return; + } + if (action.kind === "boot-shutdown") { + waiter.timing = await estimatedBootTiming( + this.options.drivers.get(action.device.spec.platform), + action.device, + ); + await this.bootShutdown(waiter, action.device, action.capacityReservation, action.claim); + return; + } + if (action.kind === "reboot-ready") { + waiter.timing = await estimatedBootTiming( + this.options.drivers.get(action.device.spec.platform), + action.device, + ); + await this.rebootReady(waiter, action.device, action.claim); + return; + } + if (action.kind === "evict-running") { + await this.evictRunning(waiter, action.device, action.claim, action.reason); + return; + } + await this.evictManaged(waiter, action.device, action.claim, action.reason); + } + + async provision(waiter: AcquisitionWaiter, reservation: CapacityReservation): Promise { + const spec = waiter.spec; + if (spec === undefined) { + reservation.release(); + return; + } + let handoff: ReadyDeviceHandoff; + try { + handoff = await this.options.provisioner.provision(spec, { + claim: { kind: "boot", owner: waiter.id }, + ...exceptOption(waiter), + onProgress: (progress) => this.options.queue.notifyProgress(waiter, progress), + reservation, + }); + } catch (error: unknown) { + if (error instanceof BootTimeoutError && exceptOption(waiter).except === undefined) { + await this.options.decisions.run(async () => { + if (waiter.state !== "rejected") + this.options.outcomes.reject(waiter, error, "boot-timeout"); + }); + this.wakeQueue(); + return; + } + // why: a component being removed (ADR 0010 §8) or a baseline removed since the request resolved it will not come back, so no device is created and the request ends as `RUNTIME_MISSING` or `UNKNOWN_BASELINE` rather than retrying. + if (error instanceof ComponentBeingRemovedError || error instanceof UnknownBaselineError) { + await this.options.decisions.run(async () => { + if (waiter.state !== "rejected") + this.options.outcomes.reject(waiter, error, "unresolvable-spec"); + }); + this.wakeQueue(); + return; + } + await this.countFailure(waiter, error); + return; + } + await this.grantHandoff(waiter, handoff, "provision"); + } + + async decide(waiter: AcquisitionWaiter): Promise { + if (this.options.gates.admissionClosed && waiter.state !== "rejected") { + this.options.outcomes.reject(waiter, new NukeCancelledError(), "killed"); + return undefined; + } + if (this.options.gates.stopPlanning) return undefined; + const plan = this.nextPlan(waiter); + if (plan === undefined) return undefined; + if (plan.kind === "grant-ready") return this.decideGrantReady(waiter, plan); + const operation = operationPlan(plan); + if (operation === undefined) { + this.options.outcomes.defer(waiter, blockedReason(plan)); + return undefined; + } + // why: a refusal means the queue has just rejected this waiter at its deadline, so the device + // work this plan stands for would run for a requester who was already answered. The plan holds + // a capacity reservation or a device claim, so it gives those back here rather than at a + // performer that will never be reached. + if (!this.options.queue.markProcessing(waiter)) { + releasePlanResources(operation); + return undefined; + } + return operation; + } + + async decideGrantReady( + waiter: AcquisitionWaiter, + plan: Extract, + ): Promise { + waiter.timing = grantReadyTiming( + this.options.drivers.get(plan.device.spec.platform), + plan.device.spec, + ); + // why: the queue's own rule is that a timeout firing mid-attempt does not reject + // (`wait-queue.ts`), and it tells an attempt is in flight only from `processing`. Without + // this mark the waiter is still `queued` while the lease is written, so a timeout that + // falls due in that window rejects the caller over a lease that is being stored. + if (!this.options.queue.markProcessing(waiter)) return undefined; + try { + await this.options.outcomes.grant(waiter, plan.device, plan.kind); + // why: the next request in line may already have a ready device waiting for it, and no + // other event will wake it. + this.wakeQueue(); + } catch (error: unknown) { + this.options.outcomes.logFailure( + "granting a ready device failed", + waiter, + plan.device, + "grant", + error, + ); + this.options.outcomes.defer(waiter, "waiting-for-device"); + } + return undefined; + } + + nextPlan(waiter: AcquisitionWaiter): AcquisitionPlan | undefined { + if (waiter.state === "rejected" || waiter.state === "granted" || waiter.spec === undefined) { + return undefined; + } + + if (this.refuseUnderHold(waiter)) return undefined; + this.followBaseline(waiter); + const head = this.options.queue.head; + if (head !== undefined && head !== waiter) { + const reason = "behind-earlier-request"; + return waiter.options.noWait ? { kind: "no-capacity", reason } : { kind: "wait", reason }; + } + return this.options.planner.plan({ + noWait: waiter.options.noWait ?? false, + owner: waiter.id, + snapshot: this.options.registry.snapshot, + spec: waiter.spec, + requirement: waiter.requirement, + classOf: waiter.classOf, + rebootReady: rebootsBeforeGrant(waiter, this.options.drivers.get(waiter.spec.platform)), + }); + } + + /** why: a save between two decisions makes the baseline's newest version the one a waiting request gets (ADR 0061 §5), so each decision plans against the version current at that moment. */ + followBaseline(waiter: AcquisitionWaiter): void { + const named = waiter.spec?.baseline; + if (named === undefined) return; + const current = this.options.baselines.current(named.name); + if (current === undefined || current.version === named.version) return; + waiter.spec = baselineSpec(current); + waiter.requirement = exactRequirement(waiter.spec); + } + + async evictRunning( + waiter: AcquisitionWaiter, + device: VirtualDevice, + claim: DeviceOperationClaim, + reason: WaitingReason, + ): Promise { + try { + const shutdown = await this.options.lifecycle.shutdown( + device, + "warm-pool-active-demand", + "eviction", + claim, + ); + if (shutdown === undefined) + throw new Error(`Eviction target is no longer safe: ${device.id}`); + } catch (error: unknown) { + this.options.outcomes.logFailure( + "shutting down an eviction target failed", + waiter, + device, + "shutdown", + error, + ); + await this.options.decisions.run(async () => { + claim.release(); + this.options.outcomes.defer(waiter, reason); + }); + return; + } + await this.continueAfterEviction(waiter, claim); + } + + async evictManaged( + waiter: AcquisitionWaiter, + device: VirtualDevice, + claim: DeviceOperationClaim, + reason: WaitingReason, + ): Promise { + try { + const deleted = await this.options.lifecycle.dispose( + device, + "warm-pool-active-demand", + "eviction", + claim, + ); + if (deleted === undefined) throw new Error(`Eviction target is no longer safe: ${device.id}`); + } catch (error: unknown) { + this.options.outcomes.logFailure( + "deleting an eviction target failed", + waiter, + device, + "delete", + error, + ); + await this.options.decisions.run(async () => { + claim.release(); + this.options.outcomes.defer(waiter, reason); + }); + return; + } + await this.continueAfterEviction(waiter, claim); + } + + async continueAfterEviction( + waiter: AcquisitionWaiter, + claim: DeviceOperationClaim, + ): Promise { + const next = await this.options.decisions.run(async () => { + claim.release(); + this.options.queue.markNew(waiter); + return this.decide(waiter); + }); + await this.perform(waiter, next); + } + + async rebootReady( + waiter: AcquisitionWaiter, + device: VirtualDevice, + claim: DeviceOperationClaim, + ): Promise { + let handoff: ReadyDeviceHandoff; + try { + this.options.queue.notifyProgress(waiter, { + stage: "booting", + etaMs: waiter.timing.estimatedBootMs, + }); + const except = exceptOption(waiter).except ?? []; + const ready = await this.options.lifecycle.rebootForLease(device, claim, except); + if (ready === undefined) throw new Error(`Reboot target is no longer safe: ${device.id}`); + handoff = ready; + } catch (error: unknown) { + this.options.outcomes.logFailure( + "rebooting a warm device for a waiter failed", + waiter, + device, + "reboot", + error, + ); + await this.destroyFailedBoot(waiter, device, "shutdown"); + await this.countFailure(waiter, error); + return; + } + await this.grantHandoff(waiter, handoff, "reboot-ready"); + } + + /** why: a device that failed to boot is destroyed, or quarantined when the destroy fails too, before the waiter's slot is freed (see below). */ + async destroyFailedBoot( + waiter: AcquisitionWaiter, + device: VirtualDevice, + state: VirtualDevice["state"], + ): Promise { + let destroyed = true; + try { + await this.options.lifecycle.destroy(device, "lease-engine", "cleanup"); + } catch (destroyError: unknown) { + this.options.outcomes.logFailure( + "destroying a device that failed to boot failed", + waiter, + device, + "destroy", + destroyError, + ); + destroyed = false; + } + // why: a device the destroy could not retire may still be running, so it is handed to + // quarantine before the waiter's reservation is freed: `quarantined` is the one state that + // retries it, keeps it out of every plan and still counts it as running. Freeing the slot + // first would leave a window where the next planner boots a device that may be running. + if (!destroyed) { + try { + const quarantined = await this.options.quarantine.enterFromStalledTransition( + device.id, + state, + ); + if (!quarantined) { + this.options.log.warn( + "a device that failed to be destroyed was not quarantined: something else owns it now", + { deviceId: device.id, requesterId: waiter.options.requesterId, step: "quarantine" }, + ); + } + } catch (quarantineError: unknown) { + this.options.outcomes.logFailure( + "quarantining a device that failed to boot and to be destroyed failed", + waiter, + device, + "quarantine", + quarantineError, + ); + } + } + } + + /** + * why: a failed boot of an except lease counts like a failed provision (ADR 0066 §3): the first + * plans again, the second rejects `PROVISION_FAILED`, one count for provision, boot and reboot. + */ + async countFailure(waiter: AcquisitionWaiter, error: unknown): Promise { + const retry = await this.options.decisions.run(async () => { + if (waiter.state === "rejected") return false; + waiter.failures += 1; + if (waiter.failures === 1) { + this.options.queue.markNew(waiter); + return true; + } + // why: ADR 0021 §1 declines a probe instead of rejecting it, and its caller (the gateway) reads the code. + this.options.outcomes.reject( + waiter, + new ProvisionFailedError(error), + waiter.options.fleetRequestId === undefined ? "unresolvable-spec" : "no-wait", + ); + return false; + }); + if (retry) { + this.#driving.delete(waiter); + await this.drive(waiter); + } else { + this.wakeQueue(); + } + } + + async bootShutdown( + waiter: AcquisitionWaiter, + device: VirtualDevice, + capacityReservation: CapacityReservation, + claim: DeviceOperationClaim, + ): Promise { + let handoff: ReadyDeviceHandoff; + try { + this.options.queue.notifyProgress(waiter, { + stage: "booting", + etaMs: waiter.timing.estimatedBootMs, + }); + const ready = await this.options.lifecycle.bootForLease( + device, + claim, + exceptOption(waiter).except, + ); + if (ready === undefined) throw new Error(`Boot target is no longer safe: ${device.id}`); + handoff = ready; + } catch (error: unknown) { + this.options.outcomes.logFailure( + "booting a shut-down device for a waiter failed", + waiter, + device, + "boot", + error, + ); + await this.destroyFailedBoot(waiter, device, device.state); + if (exceptOption(waiter).except !== undefined) { + await this.options.decisions.run(() => capacityReservation.release()); + await this.countFailure(waiter, error); + return; + } + await this.options.decisions.run(async () => { + capacityReservation.release(); + if (waiter.state !== "rejected") { + this.options.outcomes.reject( + waiter, + BootTimeoutError.from(device.id, error), + "boot-timeout", + ); + } + }); + this.wakeQueue(); + return; + } + await this.grantHandoff(waiter, handoff, "boot-shutdown", capacityReservation); + } + + /** + * Wakes the queue after a grant as well as after a waiter that was rejected meanwhile: a + * request that planned while this device was still being readied may have queued only + * because this waiter held the head of the queue or because the capacity reservation + * released here still counted. + */ + async grantHandoff( + waiter: AcquisitionWaiter, + handoff: ReadyDeviceHandoff, + kind: GrantingPlanKind, + capacityReservation?: CapacityReservation, + ): Promise { + let superseded = false; + await this.options.decisions.run(async () => { + try { + if (waiter.state === "rejected") return; + if (this.isSuperseded(handoff.device)) { + superseded = true; + return; + } + await this.options.outcomes.grant(waiter, handoff.device, kind); + } catch (error: unknown) { + // why: a grant that failed to store leaves no lease behind (`#commit` mutates memory + // only after the file write), so the waiter is re-queued for a fresh grant like every + // sibling step; without this catch the error dies in `#track` and the waiter is never + // planned again -- no timer, no wake, and its request stays open for the process. + this.options.outcomes.logFailure( + kind === "provision" + ? "granting a provisioned device failed" + : "granting a booted device failed", + waiter, + handoff.device, + "grant", + error, + ); + this.options.outcomes.defer(waiter, "waiting-for-device"); + } finally { + capacityReservation?.release(); + handoff.claim.release(); + } + }); + if (superseded) await this.discardSuperseded(waiter, handoff); + this.wakeQueue(); + } + + /** why: a copy of a version that a save replaced while it was being made is never granted. */ + isSuperseded(device: VirtualDevice): boolean { + const made = device.spec.baseline; + if (made === undefined) return false; + return this.options.baselines.current(made.name)?.version !== made.version; + } + + /** why: the superseded copy is deleted outside the gate, and its waiter plans again against the newest version. */ + async discardSuperseded(waiter: AcquisitionWaiter, handoff: ReadyDeviceHandoff): Promise { + try { + await this.options.lifecycle.dispose(handoff.device, "baseline-replaced", "cleanup"); + } catch (error: unknown) { + this.options.outcomes.logFailure( + "deleting a copy of a replaced baseline failed", + waiter, + handoff.device, + "delete", + error, + ); + } + await this.options.decisions.run(() => + this.options.outcomes.defer(waiter, "waiting-for-device"), + ); + this.#planAgain.add(waiter); + } + + wakeQueue(): void { + void this.options.decisions.run(async () => { + const next = this.options.queue.head as AcquisitionWaiter | undefined; + if (next !== undefined && next.state === "queued") this.track(this.drive(next)); + for (const waiter of this.options.queue.pending() as readonly AcquisitionWaiter[]) { + if (lineOf(waiter.request) === "physical") { + this.track(this.drive(waiter)); + } + } + }); + } + + track(workflow: Promise): void { + this.#activeWorkflows.add(workflow); + void workflow.then( + () => this.#activeWorkflows.delete(workflow), + () => this.#activeWorkflows.delete(workflow), + ); + } +} diff --git a/src/leasing/lease-acquisition-coordinator.ts b/src/leasing/lease-acquisition-coordinator.ts index b1db6fc00..c6dd072e0 100644 --- a/src/leasing/lease-acquisition-coordinator.ts +++ b/src/leasing/lease-acquisition-coordinator.ts @@ -1,269 +1,67 @@ -import type { EventBus, EventMap } from "../bus/index.js"; -import { leaseEndedError, type OsRange, requestedClass } from "../contract/index.js"; -import { type IdGenerator, type Logger, NoopLogger } from "../ports/index.js"; import { - downloadingProgress, - type DownloadingProgress, - fullSpec, - matchingRuntimes, - newestRuntime, - physicalRequestOf, - requestedRange, - requiredOs, - sameDownloadingProgress, -} from "./acquisition-requests.js"; -import { - estimatedBootTiming, - estimatedTiming, - grantReadyTiming, - releaseReclaimEstimateMs, -} from "./acquisition-timing.js"; -import { - type CapacityReservation, - type DeviceOperationClaim, - type DeviceOperationClaims, - type DeviceProvisioner, - classCandidates, - findCatalogModel, - modelClass, - type DeviceClass, + type AcquisitionMaintenance, type DeviceMode, - type DeviceRecord, - fitsPhysical, - noPhysicalMatch, - type PhysicalDevice, - type PhysicalRequest, - readPhysicalPresence, - type VirtualDevice, - type DeviceRequirement, + type DeviceRequest, type DeviceSpec, - type VirtualSpec, + type Driver, exactRequirement, type LeaseGrant, - type LeaseRecord, - type MaxPerRequester, - type LeaseTiming, newLeaseRequestId, type Platform, - sameSpec, specMode, - ComponentBeingRemovedError, - type ComponentInstaller, - BootTimeoutError, - type DeviceRequest, - type Driver, - type DriverCatalogEntry, - type ExactDeviceRequest, - RuntimeMissingError, - UnknownModelError, - type DriverCatalog, - type ModelPreferences, - type CatalogReader, - type BaselineService, - type CurrentBaseline, - UnknownBaselineError, - BaselineMismatchError, - BadDeviceRequestError, - BaselineStaleError, - baselineMismatch, - type AcquisitionMaintenance, - type Core, - type ManagedDeviceLifecycle, - type ReadyDeviceHandoff, - toDriverDevice, - type SerializedDecision, - stableError, type TargetResolution, type WaitingDemand, } from "../core/index.js"; -import { - type AcquisitionPlan, - type AcquisitionPlanner, - type PhysicalPlanInput, - type WaitingReason, -} from "./acquisition-planner.js"; +import { type Logger, NoopLogger } from "../ports/index.js"; +import { AcquisitionAdmission } from "./acquisition-admission.js"; +import { asError, DaemonUpdatingError, NukeCancelledError } from "./acquisition-errors.js"; +import { AcquisitionGates } from "./acquisition-gates.js"; +import { type LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; +import { AcquisitionSpecResolver } from "./acquisition-spec-resolver.js"; +import { type AcquisitionWaiter, noTiming } from "./acquisition-waiter.js"; +import { AcquisitionWorkflow } from "./acquisition-workflow.js"; import { emitLeaseRefusal } from "./lease-refusal.js"; -import { type LeaseRequestBook } from "./lease-request-book.js"; -import { type LeaseLifecycle } from "./lease-lifecycle.js"; -import { - LeaseIdTakenError, - lineOf, - type LeaseRequestOptions, - RequestCancelledError, - requesterLimitDecision, - type Waiter, - type WaitQueue, -} from "./wait-queue.js"; +import { type LeaseRequestOptions, RequestCancelledError } from "./wait-queue.js"; +import { WaiterOutcomes } from "./waiter-outcomes.js"; export type { LeaseRequestOptions } from "./wait-queue.js"; - -export class NoCapacityError extends Error { - constructor(readonly reason?: WaitingReason) { - super("No device capacity is currently available"); - this.name = "NoCapacityError"; - } -} - -export class ProvisionFailedError extends Error { - constructor(cause: unknown) { - super(cause instanceof Error ? cause.message : String(cause), { cause }); - this.name = "ProvisionFailedError"; - } -} - -/** why: mapped to `DAEMON_UPDATING`; the message is what a refused agent reads. */ -export class DaemonUpdatingError extends Error { - constructor() { - super("this daemon is updating and takes no new leases"); - this.name = "DaemonUpdatingError"; - } -} - -class NukeCancelledError extends Error { - constructor() { - super("Request cancelled by nuke"); - this.name = "NukeCancelledError"; - } -} - -export interface LeaseAcquisitionRegistry { - readonly snapshot: { - readonly devices: readonly DeviceRecord[]; - readonly virtualDevices: readonly VirtualDevice[]; - readonly leases: readonly LeaseRecord[]; - }; - endedLeaseId(leaseId: string): number | undefined; -} - -export type AcquisitionClaims = Pick; -export type AcquisitionQuarantine = Pick; -export type AcquisitionDecision = Pick; -export type AcquisitionDrivers = Pick; -export type AcquisitionPlannerPort = Pick; -export type AcquisitionQueue = Pick< - WaitQueue, - | "create" - | "cancelAll" - | "depth" - | "enqueue" - | "findPendingWaiter" - | "pendingPlatforms" - | "head" - | "isQueued" - | "markNew" - | "markProcessing" - | "notifyProgress" - | "pending" - | "physicalDepth" - | "reject" - | "resolve" ->; - -/** why: the exact spec of a baseline's current version, which a request naming the baseline stands for. */ -function baselineSpec(current: CurrentBaseline): VirtualSpec { - return { - baseline: { name: current.name, version: current.version }, - model: current.model, - osVersion: current.osVersion, - physical: false, - platform: current.platform, - ...(current.mode === "slim" ? { mode: "slim" as const } : {}), - ...(current.imageTag === undefined ? {} : { imageTag: current.imageTag }), - }; -} - -export interface LeaseAcquisitionCoordinatorOptions { - readonly baselines: Pick; - readonly catalog: Pick; - readonly claims: AcquisitionClaims; - readonly components: Pick; - readonly decisions: AcquisitionDecision; - readonly defaultModes: Readonly>>; - readonly drivers: AcquisitionDrivers; - readonly eventBus: Pick; - /** Mints a request's id before admission checks, so a request refused there still has one. */ - readonly idGenerator: IdGenerator; - readonly leases: Pick; - readonly lifecycle: Pick< - ManagedDeviceLifecycle, - "bootForLease" | "destroy" | "dispose" | "rebootForLease" | "shutdown" - >; - /** - * The model names to try for each class, per platform, operator's list first (ADR 0015 §4), - * built at the composition root. This coordinator holds no model name of its own. - */ - readonly modelPreferences: ModelPreferences; - readonly planner: AcquisitionPlannerPort; - readonly provisioner: Pick; - readonly queue: AcquisitionQueue; - readonly quarantine: AcquisitionQuarantine; - readonly registry: LeaseAcquisitionRegistry; - readonly logger?: Logger; - readonly requests: Pick, "admit" | "holdsLeaseId" | "replay">; - readonly maxPerRequester?: MaxPerRequester; -} - -interface AcquisitionWaiter extends Waiter { - failures: number; - classOf?: ((model: string) => DeviceClass | undefined) | undefined; - requirement?: DeviceRequirement | undefined; - spec?: VirtualSpec; - timing: LeaseTiming; -} - -type GrantingPlanKind = - | "grant-ready" - | "grant-physical" - | "boot-shutdown" - | "reboot-ready" - | "provision"; - -const GRANT_SOURCE: Readonly> = { - "boot-shutdown": "booted", - "grant-physical": "warm", - "grant-ready": "warm", - provision: "provisioned", - "reboot-ready": "booted", -}; - -type OperationPlan = Exclude< - AcquisitionPlan, - { readonly kind: "grant-ready" | "grant-physical" | "wait" | "no-capacity" } ->; - -class PhysicalUnknownModelError extends UnknownModelError { - constructor(request: PhysicalRequest) { - super(request.platform, request.model, request.model === undefined ? request.class : undefined); - this.message = noPhysicalMatch(request).message; - } -} - -const noTiming: LeaseTiming = { - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, -}; - -/** why: an `except` list, or a driver that must boot a ready device again before it can be granted, sends the device through the claimed reboot. */ -function rebootsBeforeGrant(waiter: AcquisitionWaiter, driver: Driver): boolean { - return exceptOption(waiter).except !== undefined || driver.rebootReadyBeforeGrant === true; -} +export { + DaemonUpdatingError, + NoCapacityError, + ProvisionFailedError, +} from "./acquisition-errors.js"; +export type { + AcquisitionClaims, + AcquisitionDecision, + AcquisitionDrivers, + AcquisitionPlannerPort, + AcquisitionQueue, + AcquisitionQuarantine, + LeaseAcquisitionCoordinatorOptions, + LeaseAcquisitionRegistry, +} from "./acquisition-ports.js"; export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { - readonly #activeWorkflows = new Set>(); - readonly #driving = new WeakSet(); - readonly #planAgain = new WeakSet(); - #admissionClosed = false; - // why: apart from #admissionClosed, which also stops the warm pool; an update refuses leases and leaves the pool running. - #updateHold = false; - // why: set only by a stop (`settleAcquisitions`), never cleared; unlike maintenance, which a nuke ends, nothing reopens planning after it. - #stopPlanning = false; - #maintenanceDepth = 0; readonly #logger: Logger; + readonly #gates: AcquisitionGates; + readonly #specs: AcquisitionSpecResolver; + readonly #admission: AcquisitionAdmission; + readonly #outcomes: WaiterOutcomes; + readonly #workflow: AcquisitionWorkflow; constructor(private readonly options: LeaseAcquisitionCoordinatorOptions) { this.#logger = options.logger?.child("lease-acquisition-coordinator") ?? new NoopLogger(); + this.#gates = new AcquisitionGates(); + this.#specs = new AcquisitionSpecResolver(options); + this.#admission = new AcquisitionAdmission(options); + this.#outcomes = new WaiterOutcomes({ ...options, log: this.#logger }); + this.#workflow = new AcquisitionWorkflow({ + ...options, + admission: this.#admission, + gates: this.#gates, + log: this.#logger, + outcomes: this.#outcomes, + }); } get queueDepth(): number { @@ -280,7 +78,7 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { } defaultMode(platform: Platform): DeviceMode { - return this.options.defaultModes[platform] ?? "full"; + return this.#specs.defaultMode(platform); } /** @@ -304,11 +102,11 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { const replay = this.options.requests.replay(request, options); if (replay !== undefined) return { replay }; const requestId = newLeaseRequestId(this.options.idGenerator); - if (this.#updateHold) { - this.#refuseForUpdate(requestId, request, options); + if (this.#gates.updateHold) { + this.#admission.refuseForUpdate(requestId, request, options); throw new DaemonUpdatingError(); } - if (this.#admissionClosed) { + if (this.#gates.admissionClosed) { emitLeaseRefusal( this.options.eventBus, { @@ -322,9 +120,9 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { ); throw new NukeCancelledError(); } - this.#refuseIfRequesterAtLimit(request, options, requestId); - this.#refuseIfLeaseIdTaken(request, options, requestId); - this.#refuseIfLeaseIdEnded(request, options, requestId); + this.#admission.refuseIfRequesterAtLimit(request, options, requestId); + this.#admission.refuseIfLeaseIdTaken(request, options, requestId); + this.#admission.refuseIfLeaseIdEnded(request, options, requestId); const { id, started: accepted } = await this.options.requests.admit( request, options, @@ -352,113 +150,14 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { if ("replay" in admitted) return admitted.replay; const { waiter } = admitted; - this.#track(this.#resolveAndDrive(waiter, request, options)); + this.#workflow.track(this.#resolveAndDrive(waiter, request, options)); return waiter.promise; } - #refuseIfRequesterAtLimit( - request: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): void { - const { devices, leases } = this.options.registry.snapshot; - const held = leases - .filter((lease) => lease.requesterId === options.requesterId) - .map((lease) => { - const platform = devices.find((device) => device.id === lease.deviceId)?.spec.platform; - return { - grantedAt: lease.grantedAt, - id: lease.id, - ...(platform === undefined ? {} : { platform }), - }; - }); - const waiting = this.options.queue.pendingPlatforms(options.requesterId); - // why: a probe's held leases were counted by the gateway, fleet-wide; only its waiting - // request is checked here (ADR 0023 §2). - const refusal = requesterLimitDecision({ - held, - maxPerRequester: this.options.maxPerRequester, - platform: request.platform, - probe: options.fleetRequestId !== undefined, - requesterId: options.requesterId, - waiting, - }); - if (refusal === undefined) return; - emitLeaseRefusal( - this.options.eventBus, - { - requestId, - requester: options.requesterId, - requestSpec: request, - reason: "already-leased", - fleetRequestId: options.fleetRequestId, - }, - "lease-acquisition-coordinator", - ); - throw refusal; - } - - /** - * ADR 0020: the one clash check on a host, inside the same admission section as the - * one-lease-per-requester check and after it. An ID is held by an active lease and by a - * request still waiting, so a restart, which fails every waiting request, frees it. - */ - #refuseIfLeaseIdTaken( - request: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): void { - const { leaseId } = options; - if (leaseId === undefined) return; - if ( - !this.options.registry.snapshot.leases.some((lease) => lease.id === leaseId) && - !this.options.requests.holdsLeaseId(leaseId) - ) { - return; - } - emitLeaseRefusal( - this.options.eventBus, - { - requestId, - requester: options.requesterId, - requestSpec: request, - reason: "lease-id-taken", - fleetRequestId: options.fleetRequestId, - }, - "lease-acquisition-coordinator", - ); - throw new LeaseIdTakenError(leaseId); - } - - /** why: ADR 0074 §4: a chosen id whose lease ended within `lease.endedIdRetentionMs` is refused, after the taken-id check so a live id stays `LEASE_ID_TAKEN`. */ - #refuseIfLeaseIdEnded( - request: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): void { - const { leaseId } = options; - if (leaseId === undefined) return; - const endedAt = this.options.registry.endedLeaseId(leaseId); - if (endedAt === undefined) return; - emitLeaseRefusal( - this.options.eventBus, - { - requestId, - requester: options.requesterId, - requestSpec: request, - reason: "lease-ended", - fleetRequestId: options.fleetRequestId, - }, - "lease-acquisition-coordinator", - ); - throw leaseEndedError(leaseId, endedAt); - } - /** Closes acquisition admission, settles all demand, and drains driver work. */ async beginMaintenance(): Promise { await this.options.decisions.run(async () => { - this.#maintenanceDepth += 1; - this.#admissionClosed = true; + this.#gates.enterMaintenance(); for (const waiter of this.options.queue.cancelAll(() => new NukeCancelledError())) { emitLeaseRefusal( this.options.eventBus, @@ -473,74 +172,40 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { ); } }); - await this.#waitForWorkflows(); + await this.#workflow.waitForWorkflows(); } /** * why: a stop waits for every request mid-provision or mid-boot, so the driver it then disposes is not under a boot still running (#1200). It also stops planning for good, so a failed boot waking the queue does not start another device the stop would wait for; a queued request ends with the daemon. */ async settleAcquisitions(): Promise { - this.#stopPlanning = true; - await this.#waitForWorkflows(); - } - - async #waitForWorkflows(): Promise { - while (this.#activeWorkflows.size > 0) { - await Promise.allSettled(this.#activeWorkflows); - } + this.#gates.stopPlanningForGood(); + await this.#workflow.waitForWorkflows(); } async holdForUpdate(): Promise { await this.options.decisions.run(async () => { - this.#updateHold = true; + this.#gates.holdForUpdate(); // why: a request already provisioning is granted (#825 Decisions), so only the ones not yet provisioning, `new` (still resolving its spec) or queued, are refused. for (const waiter of this.options.queue.pending() as readonly AcquisitionWaiter[]) { if (waiter.state !== "queued" && waiter.state !== "new") continue; if (this.options.queue.reject(waiter, new DaemonUpdatingError())) { - this.#refuseForUpdate(waiter.id, waiter.request, waiter.options); + this.#admission.refuseForUpdate(waiter.id, waiter.request, waiter.options); } } }); } - // why: every waiter passes `#nextPlan` before a provision starts, in whatever state `markNew` or `#defer` left it, so the update hold refuses here (#825 Decisions). - #refuseUnderHold(waiter: AcquisitionWaiter): boolean { - if (!this.#updateHold) return false; - if (this.options.queue.reject(waiter, new DaemonUpdatingError())) { - this.#refuseForUpdate(waiter.id, waiter.request, waiter.options); - } - return true; - } - async releaseUpdateHold(): Promise { await this.options.decisions.run(async () => { - this.#updateHold = false; + this.#gates.releaseUpdateHold(); }); } - #refuseForUpdate( - requestId: string, - request: DeviceRequest, - options: Pick, - ): void { - emitLeaseRefusal( - this.options.eventBus, - { - requestId, - requester: options.requesterId, - requestSpec: request, - reason: "updating", - fleetRequestId: options.fleetRequestId, - }, - "lease-acquisition-coordinator", - ); - } - /** Reopens acquisition admission after an administrative maintenance operation. */ async endMaintenance(): Promise { await this.options.decisions.run(async () => { - this.#maintenanceDepth = Math.max(0, this.#maintenanceDepth - 1); - this.#admissionClosed = this.#maintenanceDepth > 0; + this.#gates.leaveMaintenance(); }); } @@ -554,16 +219,16 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { driver = this.options.drivers.get(request.platform); } catch (error: unknown) { await this.options.decisions.run(async () => { - this.#reject(waiter, asError(error), "unresolvable-spec"); + this.#outcomes.reject(waiter, asError(error), "unresolvable-spec"); }); return; } if (request.physical === true) { - await this.#drive(waiter); + await this.#workflow.drive(waiter); return; } try { - const resolved = await this.#resolveSpec(driver, request, options, (progress) => + const resolved = await this.#specs.resolveSpec(driver, request, options, (progress) => this.options.queue.notifyProgress(waiter, progress), ); waiter.spec = resolved.spec; @@ -571,241 +236,12 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { waiter.classOf = resolved.classOf; } catch (error: unknown) { await this.options.decisions.run(async () => { - this.#reject(waiter, asError(error), "unresolvable-spec"); + this.#outcomes.reject(waiter, asError(error), "unresolvable-spec"); }); return; } - await this.#drive(waiter); - } - - async #resolveSpec( - driver: Driver, - request: DeviceRequest, - options: LeaseRequestOptions, - onProgress: (progress: DownloadingProgress) => void, - ): Promise<{ - readonly spec: VirtualSpec; - readonly requirement: DeviceRequirement; - readonly classOf: ((model: string) => DeviceClass | undefined) | undefined; - }> { - if (request.baseline !== undefined) { - return this.#resolveBaseline(driver, request.baseline, request); - } - // The one place a request with no mode gets the worker's default (ADR 0007 §2). - const mode = request.mode ?? this.defaultMode(request.platform); - if (request.except !== undefined && mode !== "slim") { - throw new BadDeviceRequestError( - "except needs a slim lease (this worker's default mode is full)", - ); - } - const range = requestedRange(request); - const target = - range === undefined - ? (this.#exactTarget(request, mode) ?? (await this.#resolveClass(request, mode))) - : await this.#resolveRanged(request, mode, range); - // A range was settled against the catalog above, so it never installs (ADR 0015 §5). - const resolved = await this.#resolveOrInstall(onProgress, driver, target.exact, { - ...options, - allowDownload: options.allowDownload === true && range === undefined, - }); - const spec = checkedSpec(resolved, request, mode); - return { - classOf: target.classOf, - requirement: target.requirement ?? exactRequirement(spec), - spec, - }; - } - - /** why: a request naming a baseline takes the baseline's own platform, model, OS, mode and image tag; no download is ever attempted. */ - async #resolveBaseline( - driver: Driver, - name: string, - request: DeviceRequest, - ): Promise<{ - readonly spec: VirtualSpec; - readonly requirement: DeviceRequirement; - readonly classOf: undefined; - }> { - const current = this.options.baselines.current(name); - if (current === undefined) throw new UnknownBaselineError(name); - const entry = (await this.options.catalog.listCatalog(current.platform))[0]; - const mismatch = baselineMismatch(current, request, entry ?? { modelAliases: {}, models: [] }); - if (mismatch !== undefined) { - throw new BaselineMismatchError(name, mismatch.field, mismatch.message); - } - if (current.staleReason !== undefined) throw new BaselineStaleError(name, current.staleReason); - const spec = baselineSpec(current); - if (request.except !== undefined && spec.mode !== "slim") { - throw new BadDeviceRequestError(`except needs a slim lease (baseline "${name}" is full)`); - } - if (request.except !== undefined) { - // why: the driver judges the names (ADR 0066 §2); its answer is dropped, the baseline's spec stands. - await driver.resolveSpec({ - except: request.except, - mode: "slim", - model: spec.model, - osVersion: spec.osVersion, - platform: spec.platform, - ...(spec.imageTag === undefined ? {} : { imageTag: spec.imageTag }), - }); - } - return { classOf: undefined, requirement: exactRequirement(spec), spec }; - } - - /** - * The exact request an exact-model request already is, answered without awaiting anything so - * its path takes no extra turn of the event loop; `undefined` for one that names no model. - */ - #exactTarget(request: DeviceRequest, mode: DeviceMode): ResolvedTarget | undefined { - const { class: _class, model, ...rest } = request; - return model === undefined ? undefined : { exact: { ...rest, mode, model } }; - } - - /** - * Turns a request that names no model, or names an OS range, into the exact one a driver - * resolves (ADR 0015 §5). A class, named or meant by naming nothing, becomes the first model - * on its preference list that this host's catalog lists, classes as that class, and pairs with - * an installed runtime (of the requested image tag, and in the requested range, when it names - * them); an exact model with a range must be listed, or the request fails as - * `UnknownModelError`. With a range the runtime is the newest paired one in it, passed on as an - * exact version. The catalog is read, and nothing is downloaded. A class with no listed model - * fails as `UnknownModelError`, and a model or class with no pairing as a `RuntimeMissingError` - * no download can fix. - */ - async #resolveClass( - request: DeviceRequest, - mode: DeviceMode, - range?: OsRange, - ): Promise { - const deviceClass = requestedClass({ class: request.class }); - const entry = (await this.options.catalog.listCatalog(request.platform))[0]; - if (entry === undefined) throw new UnknownModelError(request.platform, undefined, deviceClass); - const candidates = classCandidates( - entry, - deviceClass, - this.options.modelPreferences[request.platform]?.[deviceClass] ?? [], - ); - if (candidates.length === 0) { - throw new UnknownModelError(request.platform, undefined, deviceClass); - } - const chosen = candidates.find( - (candidate) => matchingRuntimes(entry, candidate, request, range).length > 0, - ); - if (chosen === undefined) { - throw new RuntimeMissingError(request.platform, request.osVersion ?? "default"); - } - return { - ...this.#rangedTarget( - entry, - chosen, - { class: deviceClass, kind: "class" }, - request, - mode, - range, - ), - classOf: (candidate) => modelClass(entry, candidate), - }; - } - - /** A request that names a model and an OS range: the model must be listed (ADR 0015 §5). */ - async #resolveRanged( - request: DeviceRequest, - mode: DeviceMode, - range: OsRange, - ): Promise { - if (request.model === undefined) return this.#resolveClass(request, mode, range); - const entry = (await this.options.catalog.listCatalog(request.platform))[0]; - if (entry === undefined) throw new UnknownModelError(request.platform, request.model); - const listed = findCatalogModel(entry, request.model); - if (listed === undefined) throw new UnknownModelError(request.platform, request.model); - return this.#rangedTarget( - entry, - listed, - { kind: "model", model: listed }, - request, - mode, - range, - ); - } - - /** The target for `model` once the request's runtime constraints are settled against `entry`. */ - #rangedTarget( - entry: DriverCatalogEntry, - model: string, - target: DeviceRequirement["target"], - request: DeviceRequest, - mode: DeviceMode, - range: OsRange | undefined, - ): ResolvedTarget { - const { class: _class, model: _model, ...rest } = request; - return { - exact: { - ...rest, - mode, - model, - ...(range === undefined ? {} : { osVersion: newestRuntime(entry, model, request, range) }), - }, - requirement: { - imageTag: request.imageTag, - osVersion: requiredOs(entry, request, range), - platform: request.platform, - target, - }, - }; - } - - /** - * Resolves the request; when its runtime is missing, a download could supply it, and the - * request may download, asks the installer for it and resolves once more. The installer skips - * the install when, by the time this call reaches the front of its queue, `resolveSpec` no - * longer fails with `RuntimeMissingError` -- another install made the runtime available. - * Without `allowDownload` the first failure stands and the installer is never called. - * - * The installer's reports reach the requester as the `downloading` stage, through the queue - * like every other stage. A report equal to the last one sent is skipped, so a percentage is - * sent once per whole number. - */ - async #resolveOrInstall( - onProgress: (progress: DownloadingProgress) => void, - driver: Driver, - request: ExactDeviceRequest, - options: LeaseRequestOptions, - ): Promise { - try { - return await driver.resolveSpec(request); - } catch (error: unknown) { - if ( - !(error instanceof RuntimeMissingError) || - !error.downloadable || - error.component === undefined || - options.allowDownload !== true - ) { - throw error; - } - const { component } = error; - let lastSent: DownloadingProgress | undefined; - await this.options.components.install({ - component, - onProgress: (progress) => { - const next = downloadingProgress(component, progress); - if (lastSent !== undefined && sameDownloadingProgress(lastSent, next)) return; - lastSent = next; - onProgress(next); - }, - platform: request.platform, - requesterId: options.requesterId, - stillNeeded: async () => { - try { - await driver.resolveSpec(request); - return false; - } catch (stillMissing: unknown) { - return stillMissing instanceof RuntimeMissingError; - } - }, - }); - return driver.resolveSpec(request); - } + await this.#workflow.drive(waiter); } /** @@ -827,9 +263,9 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { return "not-cancellable"; } const holdsClaim = waiter.state === "processing"; - this.#reject(waiter, new RequestCancelledError(waiter.id), "cancelled"); + this.#outcomes.reject(waiter, new RequestCancelledError(waiter.id), "cancelled"); // why: a waiter mid presence read holds a grant claim, and its own wake follows the release. - if (!holdsClaim) this.#wakeQueue(); + if (!holdsClaim) this.#workflow.wakeQueue(); return "cancelled"; }); } @@ -851,735 +287,16 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { ); } - /** - * Resolves a request to the spec it would be granted a device of, with downloads off, or says - * why there is none. A read: nothing is queued, reserved or installed. The warm pool resolves - * its targets here, so a target and a request become a spec in one place. - */ - async resolve(request: DeviceRequest): Promise { - let driver: Driver; - try { - driver = this.options.drivers.get(request.platform); - } catch (error: unknown) { - return { message: asError(error).message, refusal: "no-driver" }; - } - try { - const { spec } = await this.#resolveSpec( - driver, - request, - { allowDownload: false, ownerId: "warm-pool", requesterId: "warm-pool" }, - () => undefined, - ); - return { spec }; - } catch (error: unknown) { - return { message: asError(error).message, refusal: refusalOf(error) }; - } + resolve(request: DeviceRequest): Promise { + return this.#specs.resolve(request); } get maintenanceActive(): boolean { - return this.#admissionClosed; + return this.#gates.admissionClosed; } kick(): void { - this.#wakeQueue(); - } - - async #drive(waiter: AcquisitionWaiter): Promise { - if (this.#driving.has(waiter)) return; - this.#driving.add(waiter); - try { - if (waiter.request.physical === true) { - await this.#drivePhysical(waiter); - return; - } - const action = await this.options.decisions.run(async () => this.#decide(waiter)); - await this.#perform(waiter, action); - } finally { - this.#driving.delete(waiter); - // why: the wake inside the drive that discarded a superseded copy found the waiter still driving, so nothing else would plan it again. - if (this.#planAgain.delete(waiter)) this.#wakeQueue(); - } - } - - async #drivePhysical(waiter: AcquisitionWaiter): Promise { - const absent = new Set(); - for (;;) { - const plan = await this.options.decisions.run(async () => - this.#decidePhysical(waiter, absent), - ); - if (plan === undefined) return; - if (!(await this.#grantPhysical(waiter, plan, absent))) return; - } - } - - #decidePhysical( - waiter: AcquisitionWaiter, - absent: ReadonlySet, - ): Extract | undefined { - const request = physicalRequestOf(waiter.request); - const enrolled = this.options.registry.snapshot.devices; - if ( - !enrolled.some( - (device) => device.physical && device.state !== "deleted" && fitsPhysical(device, request), - ) - ) { - this.#reject(waiter, new PhysicalUnknownModelError(request), "unresolvable-spec"); - return undefined; - } - const plan = this.options.planner.planPhysical({ - absent, - heldByOlder: this.#heldByOlder(waiter), - noWait: waiter.options.noWait ?? false, - owner: waiter.id, - request, - snapshot: this.options.registry.snapshot, - }); - if (plan.kind !== "grant-physical") { - this.#defer(waiter, blockedReason(plan)); - return undefined; - } - if (!this.options.queue.markProcessing(waiter)) { - plan.claim.release(); - return undefined; - } - return plan; - } - - #heldByOlder(waiter: AcquisitionWaiter): PhysicalPlanInput["heldByOlder"] { - const older: PhysicalRequest[] = []; - for (const other of this.options.queue.pending()) { - if (other === waiter) break; - if (lineOf(other.request) === "physical" && other.state === "queued") { - older.push(physicalRequestOf(other.request)); - } - } - return (device: PhysicalDevice) => older.some((request) => fitsPhysical(device, request)); - } - - async #grantPhysical( - waiter: AcquisitionWaiter, - plan: Extract, - absent: Set, - ): Promise { - const { claim, device } = plan; - let present = false; - try { - const reads = await readPhysicalPresence(this.options.drivers.get(device.spec.platform), [ - device.driverDeviceId, - ]); - present = reads.get(device.driverDeviceId) === true; - } catch (error: unknown) { - this.#logFailure( - "reading a physical device's presence failed", - waiter, - device, - "presence", - error, - ); - } - let replan = false; - await this.options.decisions.run(async () => { - try { - if (!this.options.queue.markProcessing(waiter)) return; - const current = this.options.registry.snapshot.devices.find( - (candidate) => candidate.id === device.id, - ); - if (!present || !this.options.claims.isActive(claim) || current?.state !== "ready") { - absent.add(device.id); - replan = true; - return; - } - await this.#grant(waiter, current, plan.kind); - } catch (error: unknown) { - this.#logFailure("granting a physical device failed", waiter, device, "grant", error); - this.#defer(waiter, "no-free-physical"); - } finally { - claim.release(); - } - }); - if (!replan) this.#wakeQueue(); - return replan; - } - - async #perform(waiter: AcquisitionWaiter, action: OperationPlan | undefined): Promise { - if (action === undefined) return; - if (action.kind === "provision") { - const spec = waiter.spec; - if (spec === undefined) { - action.reservation.release(); - return; - } - waiter.timing = estimatedTiming(this.options.drivers.get(spec.platform), spec); - await this.#provision(waiter, action.reservation); - return; - } - if (action.kind === "boot-shutdown") { - waiter.timing = await estimatedBootTiming( - this.options.drivers.get(action.device.spec.platform), - action.device, - ); - await this.#bootShutdown(waiter, action.device, action.capacityReservation, action.claim); - return; - } - if (action.kind === "reboot-ready") { - waiter.timing = await estimatedBootTiming( - this.options.drivers.get(action.device.spec.platform), - action.device, - ); - await this.#rebootReady(waiter, action.device, action.claim); - return; - } - if (action.kind === "evict-running") { - await this.#evictRunning(waiter, action.device, action.claim, action.reason); - return; - } - await this.#evictManaged(waiter, action.device, action.claim, action.reason); - } - - async #provision(waiter: AcquisitionWaiter, reservation: CapacityReservation): Promise { - const spec = waiter.spec; - if (spec === undefined) { - reservation.release(); - return; - } - let handoff: ReadyDeviceHandoff; - try { - handoff = await this.options.provisioner.provision(spec, { - claim: { kind: "boot", owner: waiter.id }, - ...exceptOption(waiter), - onProgress: (progress) => this.options.queue.notifyProgress(waiter, progress), - reservation, - }); - } catch (error: unknown) { - if (error instanceof BootTimeoutError && exceptOption(waiter).except === undefined) { - await this.options.decisions.run(async () => { - if (waiter.state !== "rejected") this.#reject(waiter, error, "boot-timeout"); - }); - this.#wakeQueue(); - return; - } - // why: a component being removed (ADR 0010 §8) or a baseline removed since the request resolved it will not come back, so no device is created and the request ends as `RUNTIME_MISSING` or `UNKNOWN_BASELINE` rather than retrying. - if (error instanceof ComponentBeingRemovedError || error instanceof UnknownBaselineError) { - await this.options.decisions.run(async () => { - if (waiter.state !== "rejected") this.#reject(waiter, error, "unresolvable-spec"); - }); - this.#wakeQueue(); - return; - } - await this.#countFailure(waiter, error); - return; - } - await this.#grantHandoff(waiter, handoff, "provision"); - } - - async #decide(waiter: AcquisitionWaiter): Promise { - if (this.#admissionClosed && waiter.state !== "rejected") { - this.#reject(waiter, new NukeCancelledError(), "killed"); - return undefined; - } - if (this.#stopPlanning) return undefined; - const plan = this.#nextPlan(waiter); - if (plan === undefined) return undefined; - if (plan.kind === "grant-ready") return this.#decideGrantReady(waiter, plan); - const operation = operationPlan(plan); - if (operation === undefined) { - this.#defer(waiter, blockedReason(plan)); - return undefined; - } - // why: a refusal means the queue has just rejected this waiter at its deadline, so the device - // work this plan stands for would run for a requester who was already answered. The plan holds - // a capacity reservation or a device claim, so it gives those back here rather than at a - // performer that will never be reached. - if (!this.options.queue.markProcessing(waiter)) { - releasePlanResources(operation); - return undefined; - } - return operation; - } - - async #decideGrantReady( - waiter: AcquisitionWaiter, - plan: Extract, - ): Promise { - waiter.timing = grantReadyTiming( - this.options.drivers.get(plan.device.spec.platform), - plan.device.spec, - ); - // why: the queue's own rule is that a timeout firing mid-attempt does not reject - // (`wait-queue.ts`), and it tells an attempt is in flight only from `processing`. Without - // this mark the waiter is still `queued` while the lease is written, so a timeout that - // falls due in that window rejects the caller over a lease that is being stored. - if (!this.options.queue.markProcessing(waiter)) return undefined; - try { - await this.#grant(waiter, plan.device, plan.kind); - // why: the next request in line may already have a ready device waiting for it, and no - // other event will wake it. - this.#wakeQueue(); - } catch (error: unknown) { - this.#logFailure("granting a ready device failed", waiter, plan.device, "grant", error); - this.#defer(waiter, "waiting-for-device"); - } - return undefined; - } - - #nextPlan(waiter: AcquisitionWaiter): AcquisitionPlan | undefined { - if (waiter.state === "rejected" || waiter.state === "granted" || waiter.spec === undefined) { - return undefined; - } - - if (this.#refuseUnderHold(waiter)) return undefined; - this.#followBaseline(waiter); - const head = this.options.queue.head; - if (head !== undefined && head !== waiter) { - const reason = "behind-earlier-request"; - return waiter.options.noWait ? { kind: "no-capacity", reason } : { kind: "wait", reason }; - } - return this.options.planner.plan({ - noWait: waiter.options.noWait ?? false, - owner: waiter.id, - snapshot: this.options.registry.snapshot, - spec: waiter.spec, - requirement: waiter.requirement, - classOf: waiter.classOf, - rebootReady: rebootsBeforeGrant(waiter, this.options.drivers.get(waiter.spec.platform)), - }); - } - - /** why: a save between two decisions makes the baseline's newest version the one a waiting request gets (ADR 0061 §5), so each decision plans against the version current at that moment. */ - #followBaseline(waiter: AcquisitionWaiter): void { - const named = waiter.spec?.baseline; - if (named === undefined) return; - const current = this.options.baselines.current(named.name); - if (current === undefined || current.version === named.version) return; - waiter.spec = baselineSpec(current); - waiter.requirement = exactRequirement(waiter.spec); - } - - async #grant( - waiter: AcquisitionWaiter, - device: DeviceRecord, - kind: GrantingPlanKind, - ): Promise { - // Built before the write: the grant a repeat of this request answers is stored with the lease. - const granted = await this.options.leases.grant({ - deviceId: device.id, - environment: this.options.drivers - .get(device.spec.platform) - .leaseEnvironment(toDriverDevice(device)), - ownerId: waiter.options.ownerId, - requestId: waiter.id, - requesterId: waiter.options.requesterId, - source: GRANT_SOURCE[kind], - timing: waiter.timing, - ...(device.mode === "slim" ? exceptOption(waiter) : {}), - ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), - ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), - ...(waiter.options.fleetRequestId === undefined - ? {} - : { fleetRequestId: waiter.options.fleetRequestId }), - }); - this.options.queue.resolve(waiter, granted); - } - - async #evictRunning( - waiter: AcquisitionWaiter, - device: VirtualDevice, - claim: DeviceOperationClaim, - reason: WaitingReason, - ): Promise { - try { - const shutdown = await this.options.lifecycle.shutdown( - device, - "warm-pool-active-demand", - "eviction", - claim, - ); - if (shutdown === undefined) - throw new Error(`Eviction target is no longer safe: ${device.id}`); - } catch (error: unknown) { - this.#logFailure( - "shutting down an eviction target failed", - waiter, - device, - "shutdown", - error, - ); - await this.options.decisions.run(async () => { - claim.release(); - this.#defer(waiter, reason); - }); - return; - } - await this.#continueAfterEviction(waiter, claim); - } - - async #evictManaged( - waiter: AcquisitionWaiter, - device: VirtualDevice, - claim: DeviceOperationClaim, - reason: WaitingReason, - ): Promise { - try { - const deleted = await this.options.lifecycle.dispose( - device, - "warm-pool-active-demand", - "eviction", - claim, - ); - if (deleted === undefined) throw new Error(`Eviction target is no longer safe: ${device.id}`); - } catch (error: unknown) { - this.#logFailure("deleting an eviction target failed", waiter, device, "delete", error); - await this.options.decisions.run(async () => { - claim.release(); - this.#defer(waiter, reason); - }); - return; - } - await this.#continueAfterEviction(waiter, claim); - } - - async #continueAfterEviction( - waiter: AcquisitionWaiter, - claim: DeviceOperationClaim, - ): Promise { - const next = await this.options.decisions.run(async () => { - claim.release(); - this.options.queue.markNew(waiter); - return this.#decide(waiter); - }); - await this.#perform(waiter, next); - } - - /** One `warn` line for a device step that failed in the background and is handled here by - * deferring the waiter or destroying the device. The waiter only ever sees a generic outcome. */ - #logFailure( - message: string, - waiter: AcquisitionWaiter, - device: DeviceRecord, - step: string, - error: unknown, - ): void { - this.#logger.warn(message, { - deviceId: device.id, - requesterId: waiter.options.requesterId, - step, - error: stableError(error), - }); - } - - async #rebootReady( - waiter: AcquisitionWaiter, - device: VirtualDevice, - claim: DeviceOperationClaim, - ): Promise { - let handoff: ReadyDeviceHandoff; - try { - this.options.queue.notifyProgress(waiter, { - stage: "booting", - etaMs: waiter.timing.estimatedBootMs, - }); - const except = exceptOption(waiter).except ?? []; - const ready = await this.options.lifecycle.rebootForLease(device, claim, except); - if (ready === undefined) throw new Error(`Reboot target is no longer safe: ${device.id}`); - handoff = ready; - } catch (error: unknown) { - this.#logFailure( - "rebooting a warm device for a waiter failed", - waiter, - device, - "reboot", - error, - ); - await this.#destroyFailedBoot(waiter, device, "shutdown"); - await this.#countFailure(waiter, error); - return; - } - await this.#grantHandoff(waiter, handoff, "reboot-ready"); - } - - /** why: a device that failed to boot is destroyed, or quarantined when the destroy fails too, before the waiter's slot is freed (see below). */ - async #destroyFailedBoot( - waiter: AcquisitionWaiter, - device: VirtualDevice, - state: VirtualDevice["state"], - ): Promise { - let destroyed = true; - try { - await this.options.lifecycle.destroy(device, "lease-engine", "cleanup"); - } catch (destroyError: unknown) { - this.#logFailure( - "destroying a device that failed to boot failed", - waiter, - device, - "destroy", - destroyError, - ); - destroyed = false; - } - // why: a device the destroy could not retire may still be running, so it is handed to - // quarantine before the waiter's reservation is freed: `quarantined` is the one state that - // retries it, keeps it out of every plan and still counts it as running. Freeing the slot - // first would leave a window where the next planner boots a device that may be running. - if (!destroyed) { - try { - const quarantined = await this.options.quarantine.enterFromStalledTransition( - device.id, - state, - ); - if (!quarantined) { - this.#logger.warn( - "a device that failed to be destroyed was not quarantined: something else owns it now", - { deviceId: device.id, requesterId: waiter.options.requesterId, step: "quarantine" }, - ); - } - } catch (quarantineError: unknown) { - this.#logFailure( - "quarantining a device that failed to boot and to be destroyed failed", - waiter, - device, - "quarantine", - quarantineError, - ); - } - } - } - - /** - * why: a failed boot of an except lease counts like a failed provision (ADR 0066 §3): the first - * plans again, the second rejects `PROVISION_FAILED`, one count for provision, boot and reboot. - */ - async #countFailure(waiter: AcquisitionWaiter, error: unknown): Promise { - const retry = await this.options.decisions.run(async () => { - if (waiter.state === "rejected") return false; - waiter.failures += 1; - if (waiter.failures === 1) { - this.options.queue.markNew(waiter); - return true; - } - // why: ADR 0021 §1 declines a probe instead of rejecting it, and its caller (the gateway) reads the code. - this.#reject( - waiter, - new ProvisionFailedError(error), - waiter.options.fleetRequestId === undefined ? "unresolvable-spec" : "no-wait", - ); - return false; - }); - if (retry) { - this.#driving.delete(waiter); - await this.#drive(waiter); - } else { - this.#wakeQueue(); - } - } - - async #bootShutdown( - waiter: AcquisitionWaiter, - device: VirtualDevice, - capacityReservation: CapacityReservation, - claim: DeviceOperationClaim, - ): Promise { - let handoff: ReadyDeviceHandoff; - try { - this.options.queue.notifyProgress(waiter, { - stage: "booting", - etaMs: waiter.timing.estimatedBootMs, - }); - const ready = await this.options.lifecycle.bootForLease( - device, - claim, - exceptOption(waiter).except, - ); - if (ready === undefined) throw new Error(`Boot target is no longer safe: ${device.id}`); - handoff = ready; - } catch (error: unknown) { - this.#logFailure( - "booting a shut-down device for a waiter failed", - waiter, - device, - "boot", - error, - ); - await this.#destroyFailedBoot(waiter, device, device.state); - if (exceptOption(waiter).except !== undefined) { - await this.options.decisions.run(() => capacityReservation.release()); - await this.#countFailure(waiter, error); - return; - } - await this.options.decisions.run(async () => { - capacityReservation.release(); - if (waiter.state !== "rejected") { - this.#reject(waiter, BootTimeoutError.from(device.id, error), "boot-timeout"); - } - }); - this.#wakeQueue(); - return; - } - await this.#grantHandoff(waiter, handoff, "boot-shutdown", capacityReservation); - } - - #defer(waiter: AcquisitionWaiter, reason: WaitingReason): void { - if (waiter.options.noWait) { - this.#reject(waiter, new NoCapacityError(reason), "no-wait"); - return; - } - this.#enqueue(waiter, reason); - this.#notifyReclaimWait(waiter); - } - - /** - * A queued waiter whose spec matches a device already being reclaimed is waiting on that - * reclaim, not on nothing in particular: an iOS erase holds the only matching device for - * tens of seconds, and reporting only `queued` leaves the requester with a position and no - * sense of how long. Purely informational -- the plan is untouched. When the reclaim - * commits, the waiter is planned again like any other: an Android device comes back - * `ready` and is granted at once, an iOS one comes back `shutdown` and is granted - * through a `boot-shutdown` plan, a full boot after the erase. Nothing is reported - * when no matching device is reclaiming, so this never invents a stage out of an idle wait. - */ - #notifyReclaimWait(waiter: AcquisitionWaiter): void { - const spec = waiter.spec; - if (spec === undefined) return; - const reclaiming = this.options.registry.snapshot.devices.some( - (device) => device.state === "reclaiming" && sameSpec(device.spec, spec), - ); - if (!reclaiming) return; - this.options.queue.notifyProgress(waiter, { - etaMs: releaseReclaimEstimateMs(this.options.drivers.get(spec.platform), spec), - stage: "reclaiming", - }); - } - - /** - * Wakes the queue after a grant as well as after a waiter that was rejected meanwhile: a - * request that planned while this device was still being readied may have queued only - * because this waiter held the head of the queue or because the capacity reservation - * released here still counted. - */ - async #grantHandoff( - waiter: AcquisitionWaiter, - handoff: ReadyDeviceHandoff, - kind: GrantingPlanKind, - capacityReservation?: CapacityReservation, - ): Promise { - let superseded = false; - await this.options.decisions.run(async () => { - try { - if (waiter.state === "rejected") return; - if (this.#isSuperseded(handoff.device)) { - superseded = true; - return; - } - await this.#grant(waiter, handoff.device, kind); - } catch (error: unknown) { - // why: a grant that failed to store leaves no lease behind (`#commit` mutates memory - // only after the file write), so the waiter is re-queued for a fresh grant like every - // sibling step; without this catch the error dies in `#track` and the waiter is never - // planned again -- no timer, no wake, and its request stays open for the process. - this.#logFailure( - kind === "provision" - ? "granting a provisioned device failed" - : "granting a booted device failed", - waiter, - handoff.device, - "grant", - error, - ); - this.#defer(waiter, "waiting-for-device"); - } finally { - capacityReservation?.release(); - handoff.claim.release(); - } - }); - if (superseded) await this.#discardSuperseded(waiter, handoff); - this.#wakeQueue(); - } - - /** why: a copy of a version that a save replaced while it was being made is never granted. */ - #isSuperseded(device: VirtualDevice): boolean { - const made = device.spec.baseline; - if (made === undefined) return false; - return this.options.baselines.current(made.name)?.version !== made.version; - } - - /** why: the superseded copy is deleted outside the gate, and its waiter plans again against the newest version. */ - async #discardSuperseded(waiter: AcquisitionWaiter, handoff: ReadyDeviceHandoff): Promise { - try { - await this.options.lifecycle.dispose(handoff.device, "baseline-replaced", "cleanup"); - } catch (error: unknown) { - this.#logFailure( - "deleting a copy of a replaced baseline failed", - waiter, - handoff.device, - "delete", - error, - ); - } - await this.options.decisions.run(() => this.#defer(waiter, "waiting-for-device")); - this.#planAgain.add(waiter); - } - - #enqueue(waiter: AcquisitionWaiter, reason: WaitingReason): void { - const alreadyQueued = this.options.queue.isQueued(waiter); - if (this.options.queue.enqueue(waiter, reason) && !alreadyQueued) { - this.options.eventBus.emit( - "lease.queued", - { - queuePosition: - lineOf(waiter.request) === "physical" - ? this.options.queue.physicalDepth - : this.options.queue.depth, - requestId: waiter.id, - }, - "lease-acquisition-coordinator", - ); - } - } - - #reject( - waiter: AcquisitionWaiter, - error: Error, - reason: - | "timeout" - | "no-wait" - | "unresolvable-spec" - | "already-leased" - | "boot-timeout" - | "killed" - | "cancelled", - ): void { - if (this.options.queue.reject(waiter, error)) { - emitLeaseRefusal( - this.options.eventBus, - { - requestId: waiter.id, - requester: waiter.options.requesterId, - requestSpec: waiter.request, - reason, - fleetRequestId: waiter.options.fleetRequestId, - }, - "lease-acquisition-coordinator", - ); - } - } - - #wakeQueue(): void { - void this.options.decisions.run(async () => { - const next = this.options.queue.head as AcquisitionWaiter | undefined; - if (next !== undefined && next.state === "queued") this.#track(this.#drive(next)); - for (const waiter of this.options.queue.pending() as readonly AcquisitionWaiter[]) { - if (lineOf(waiter.request) === "physical") { - this.#track(this.#drive(waiter)); - } - } - }); - } - - #track(workflow: Promise): void { - this.#activeWorkflows.add(workflow); - void workflow.then( - () => this.#activeWorkflows.delete(workflow), - () => this.#activeWorkflows.delete(workflow), - ); + this.#workflow.wakeQueue(); } #newWaiter(request: DeviceRequest, options: LeaseRequestOptions, id: string): AcquisitionWaiter { @@ -1589,75 +306,3 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { }); } } - -/** why: a request's `except` reaches the driver and the device only when the resolved spec is slim; on a runtime that cannot be slimmed the lease is granted full with no `except` (ADR 0066 §1). */ -function exceptOption(waiter: AcquisitionWaiter): { readonly except?: readonly string[] } { - const except = waiter.request.except; - return except === undefined || waiter.spec?.mode !== "slim" ? {} : { except }; -} - -function asError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); -} - -/** - * why: a plan handed out the capacity or device claim its operation needs; dropping the plan has - * to give it back, or the resource stays held with nobody left to release it. - */ -function releasePlanResources(plan: OperationPlan): void { - if (plan.kind === "provision") plan.reservation.release(); - else if (plan.kind === "boot-shutdown") { - plan.capacityReservation.release(); - plan.claim.release(); - } else plan.claim.release(); -} - -function blockedReason(plan: AcquisitionPlan): WaitingReason { - return "reason" in plan ? plan.reason : "no-free-physical"; -} - -function operationPlan(plan: AcquisitionPlan): OperationPlan | undefined { - return plan.kind === "grant-ready" || - plan.kind === "grant-physical" || - plan.kind === "wait" || - plan.kind === "no-capacity" - ? undefined - : plan; -} - -/** - * The spec a driver resolved, held to what the request asked. The image tag is the request's, or - * none: a driver that returns another, or one the request did not name, would plan the device - * into a pool the request did not ask for. Full is a guarantee (ADR 0007 §5): a slim spec is - * accepted only for a slim request, so a driver that returns the wrong thing still cannot put a - * full request on a slim device. - */ -function checkedSpec(resolved: VirtualSpec, request: DeviceRequest, mode: DeviceMode): VirtualSpec { - if (resolved.imageTag !== request.imageTag) { - throw new Error( - `The ${request.platform} driver resolved image tag ${String(resolved.imageTag)} ` + - `for a request naming ${String(request.imageTag)}`, - ); - } - return mode === "slim" ? resolved : fullSpec(resolved); -} - -/** What a target's refusal is called: the kinds a pool can act on, and any other as is. */ -function refusalOf(error: unknown): Exclude["refusal"] { - if (error instanceof RuntimeMissingError) return "runtime-missing"; - if (error instanceof UnknownModelError) return "unknown-model"; - return "unresolvable"; -} - -/** What `#resolveClass` settles on; an exact request has only `exact`. */ -interface ResolvedTarget { - readonly exact: ExactDeviceRequest; - readonly requirement?: DeviceRequirement; - readonly classOf?: (model: string) => DeviceClass | undefined; -} - -/** - * The installed runtimes a model pairs with that the request accepts: the one it names, or those - * in its range, or any when it names none; and only runtimes that have an image of the requested - * tag when it names one (the rule the gateway's `matchRequest` applies). - */ diff --git a/src/leasing/waiter-outcomes.ts b/src/leasing/waiter-outcomes.ts new file mode 100644 index 000000000..04cef9dbc --- /dev/null +++ b/src/leasing/waiter-outcomes.ts @@ -0,0 +1,143 @@ +import { type DeviceRecord, sameSpec, stableError, toDriverDevice } from "../core/index.js"; +import { type Logger } from "../ports/index.js"; +import { NoCapacityError } from "./acquisition-errors.js"; +import { type WaitingReason } from "./acquisition-planner.js"; +import { type LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; +import { releaseReclaimEstimateMs } from "./acquisition-timing.js"; +import { + type AcquisitionWaiter, + exceptOption, + GRANT_SOURCE, + type GrantingPlanKind, +} from "./acquisition-waiter.js"; +import { emitLeaseRefusal } from "./lease-refusal.js"; +import { lineOf } from "./wait-queue.js"; + +export type WaiterOutcomesOptions = Pick< + LeaseAcquisitionCoordinatorOptions, + "drivers" | "eventBus" | "leases" | "queue" | "registry" +> & { readonly log: Logger }; + +/** why: every way a waiter waits, is refused or is granted goes through one method here, so each emits its one event in one place. */ +export class WaiterOutcomes { + constructor(private readonly options: WaiterOutcomesOptions) {} + + async grant( + waiter: AcquisitionWaiter, + device: DeviceRecord, + kind: GrantingPlanKind, + ): Promise { + // Built before the write: the grant a repeat of this request answers is stored with the lease. + const granted = await this.options.leases.grant({ + deviceId: device.id, + environment: this.options.drivers + .get(device.spec.platform) + .leaseEnvironment(toDriverDevice(device)), + ownerId: waiter.options.ownerId, + requestId: waiter.id, + requesterId: waiter.options.requesterId, + source: GRANT_SOURCE[kind], + timing: waiter.timing, + ...(device.mode === "slim" ? exceptOption(waiter) : {}), + ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), + ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), + ...(waiter.options.fleetRequestId === undefined + ? {} + : { fleetRequestId: waiter.options.fleetRequestId }), + }); + this.options.queue.resolve(waiter, granted); + } + + /** One `warn` line for a device step that failed in the background and is handled here by + * deferring the waiter or destroying the device. The waiter only ever sees a generic outcome. */ + logFailure( + message: string, + waiter: AcquisitionWaiter, + device: DeviceRecord, + step: string, + error: unknown, + ): void { + this.options.log.warn(message, { + deviceId: device.id, + requesterId: waiter.options.requesterId, + step, + error: stableError(error), + }); + } + + defer(waiter: AcquisitionWaiter, reason: WaitingReason): void { + if (waiter.options.noWait) { + this.reject(waiter, new NoCapacityError(reason), "no-wait"); + return; + } + this.enqueue(waiter, reason); + this.notifyReclaimWait(waiter); + } + + /** + * A queued waiter whose spec matches a device already being reclaimed is waiting on that + * reclaim, not on nothing in particular: an iOS erase holds the only matching device for + * tens of seconds, and reporting only `queued` leaves the requester with a position and no + * sense of how long. Purely informational -- the plan is untouched. When the reclaim + * commits, the waiter is planned again like any other: an Android device comes back + * `ready` and is granted at once, an iOS one comes back `shutdown` and is granted + * through a `boot-shutdown` plan, a full boot after the erase. Nothing is reported + * when no matching device is reclaiming, so this never invents a stage out of an idle wait. + */ + notifyReclaimWait(waiter: AcquisitionWaiter): void { + const spec = waiter.spec; + if (spec === undefined) return; + const reclaiming = this.options.registry.snapshot.devices.some( + (device) => device.state === "reclaiming" && sameSpec(device.spec, spec), + ); + if (!reclaiming) return; + this.options.queue.notifyProgress(waiter, { + etaMs: releaseReclaimEstimateMs(this.options.drivers.get(spec.platform), spec), + stage: "reclaiming", + }); + } + + enqueue(waiter: AcquisitionWaiter, reason: WaitingReason): void { + const alreadyQueued = this.options.queue.isQueued(waiter); + if (this.options.queue.enqueue(waiter, reason) && !alreadyQueued) { + this.options.eventBus.emit( + "lease.queued", + { + queuePosition: + lineOf(waiter.request) === "physical" + ? this.options.queue.physicalDepth + : this.options.queue.depth, + requestId: waiter.id, + }, + "lease-acquisition-coordinator", + ); + } + } + + reject( + waiter: AcquisitionWaiter, + error: Error, + reason: + | "timeout" + | "no-wait" + | "unresolvable-spec" + | "already-leased" + | "boot-timeout" + | "killed" + | "cancelled", + ): void { + if (this.options.queue.reject(waiter, error)) { + emitLeaseRefusal( + this.options.eventBus, + { + requestId: waiter.id, + requester: waiter.options.requesterId, + requestSpec: waiter.request, + reason, + fleetRequestId: waiter.options.fleetRequestId, + }, + "lease-acquisition-coordinator", + ); + } + } +} diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index 53cb2b89d..a1fda9b32 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -31,7 +31,6 @@ const SHRINK_ONLY: Record = { "src/drivers/ios/index.test.ts": 6208, "src/drivers/ios/index.ts": 2700, "src/leasing/create-leasing.test.ts": 5389, - "src/leasing/lease-acquisition-coordinator.ts": 1663, }; const CODE_LIMIT = 1400; From 71fe75e6a185ea4b336f2af97b9e3ca7520bc9a0 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:19:18 +0100 Subject: [PATCH 14/83] refactor(leasing): keep the workflow's provision step private, as it was in the coordinator --- src/leasing/acquisition-workflow.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leasing/acquisition-workflow.ts b/src/leasing/acquisition-workflow.ts index eca3f0dd9..19d1a3051 100644 --- a/src/leasing/acquisition-workflow.ts +++ b/src/leasing/acquisition-workflow.ts @@ -221,7 +221,7 @@ export class AcquisitionWorkflow { return; } waiter.timing = estimatedTiming(this.options.drivers.get(spec.platform), spec); - await this.provision(waiter, action.reservation); + await this.#provision(waiter, action.reservation); return; } if (action.kind === "boot-shutdown") { @@ -247,7 +247,7 @@ export class AcquisitionWorkflow { await this.evictManaged(waiter, action.device, action.claim, action.reason); } - async provision(waiter: AcquisitionWaiter, reservation: CapacityReservation): Promise { + async #provision(waiter: AcquisitionWaiter, reservation: CapacityReservation): Promise { const spec = waiter.spec; if (spec === undefined) { reservation.release(); From f3740ad13d22c0f8501699c0cef098993cb42900 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:12:36 +0100 Subject: [PATCH 15/83] refactor(daemon): split server.ts into parts --- src/daemon/server-options.ts | 242 +++++++++++ src/daemon/server-requests.ts | 530 ++++++++++++++++++++++++ src/daemon/server.ts | 747 ++-------------------------------- 3 files changed, 805 insertions(+), 714 deletions(-) create mode 100644 src/daemon/server-options.ts create mode 100644 src/daemon/server-requests.ts diff --git a/src/daemon/server-options.ts b/src/daemon/server-options.ts new file mode 100644 index 000000000..93b0f94ef --- /dev/null +++ b/src/daemon/server-options.ts @@ -0,0 +1,242 @@ +import { type EventBus } from "../bus/index.js"; +import { type EventHistoryHandle } from "../history/index.js"; +import { + type ComponentInstaller, + type Config, + type Registry, + type CleanupReaper, + type Doctor, + type DriverRejection, + type HostFacts, + type Nuke, + type BaselineService, + type WarmKindForgetter, + type CapacityReader, + type WarmPoolReader, + type CatalogReader, + type PassthroughResolver, + type PhysicalAutoEnrollment, +} from "../core/index.js"; +import { + type LeaseHealthMonitor, + type PhysicalDeviceWatcher, + type DeviceModeReader, + type LeaseCommands, + type QueueControl, +} from "../leasing/index.js"; +import { type Clock, type Logger, type ProcessRunner } from "../ports/index.js"; +import { + type ProtocolRange, + type StatusLoad, + type StatusMemoryPressure, +} from "../contract/index.js"; +import { type ConnectionHost } from "./connection-host.js"; +import { type Auth, type SessionRoleResolver, type TokenOperations } from "../auth/index.js"; +import { Dispatcher, type ContractDispatcher, type DispatcherOptions } from "./dispatcher.js"; +import { type OwnerRoutedFacts } from "./owner-routed-facts.js"; +import { classifyError } from "./error-code.js"; +import { type RelayTargets, type TwoWayRelayTargets } from "./relay.js"; +import { type UpdateCoordinator } from "./update-coordinator.js"; + +/** + * why: kept in the type rather than making every field optional: a gateway passes `dispatcher` instead, + * and omitting `queue` from a worker's options stays a compile error. + */ +export interface DaemonServerEngineOptions { + readonly capabilitySources?: DispatcherOptions["capabilitySources"]; + readonly capacity: CapacityReader; + readonly warmPool: WarmPoolReader; + readonly deviceModes: DeviceModeReader; + readonly catalog: CatalogReader; + readonly components: Pick; + readonly doctor?: Doctor; + readonly eventHistory: Pick; + readonly historyPath?: string; + readonly leases: LeaseCommands; + readonly hostFacts: () => HostFacts; + readonly load?: () => StatusLoad | undefined; + readonly instanceId: string; + readonly queue: QueueControl; + readonly presence?: DispatcherOptions["presence"]; + readonly reaper: CleanupReaper; + readonly healthMonitor?: LeaseHealthMonitor | undefined; + readonly physicalDeviceWatcher?: PhysicalDeviceWatcher | undefined; + readonly autoEnrollment?: Pick | undefined; + readonly enrollment?: DispatcherOptions["enrollment"]; + readonly nuke?: Nuke; + readonly warmKinds?: WarmKindForgetter; + readonly baselines?: Pick; + readonly passthrough?: PassthroughResolver; + readonly inputs?: DispatcherOptions["inputs"]; + readonly processRunner?: ProcessRunner; + readonly leaseFiles?: DispatcherOptions["leaseFiles"]; + readonly relayTargets?: RelayTargets; + readonly relayTwoWayTargets?: TwoWayRelayTargets; + readonly execEnv?: NodeJS.ProcessEnv; + readonly registry: Registry; + readonly stalls?: DispatcherOptions["stalls"]; + readonly tokens?: TokenOperations; + readonly update?: UpdateCoordinator; +} + +export interface DaemonServerCommonOptions { + readonly clock: Clock; + readonly config: Config; + readonly defaultRequesterId: string; + /** + * why: the daemon serves without a platform whose driver refused to start (safety rule 9), so each + * refusal lands as an event in the ring buffer here. + */ + readonly driverRejections?: readonly DriverRejection[]; + readonly eventBus: EventBus; + readonly host: ConnectionHost; + readonly hostLoad?: { start(): void; dispose(): void }; + readonly memoryPressure?: { current(): StatusMemoryPressure | undefined; dispose(): void }; + readonly logger?: Logger; + readonly protocolVersion?: number | ProtocolRange; + readonly version: string; + readonly resolveRole?: SessionRoleResolver; + /** + * why: ADR §5's per-start admin secret. Undefined is a legitimate default for tests that don't + * exercise the admin handshake at all -- `start()`/`#stop()` simply skip the persist/remove + * calls below when it is absent, same style as `doctor`/`nuke`. + */ + readonly adminSecret?: Auth["adminSecret"]; + /** + * why: Startup recovery work (doctor reconciliation, running-capacity convergence) run + * *after* the socket is claimed, so reachability never depends on it. `hello` and + * `status.get` answer immediately; every other request type parks on this promise. + * A rejection here stops the daemon rather than leaving it half-open — see `start()`. + */ + readonly converge?: () => Promise; + /** + * why: Awaits lease work the subsystem is finishing off its callers' paths — today the + * backgrounded device reclaims a release hands off (see `LeaseReleaseCoordinator`). + * A graceful stop drains it so the pool is left settled; an ungraceful death leaves + * it for startup recovery. The stop itself releases nothing (ADR 0004 §3), so what this + * drains is work somebody else's release already handed off; it runs before `dispose`, so a + * quarantine retry timer armed by a reclaim settling late is still cancelled. + */ + readonly settle?: () => Promise; + /** + * why: Called synchronously the moment a stop is asked for, before anything is awaited: it closes + * to new work whatever `settle` will later wait for (the warm pool), so nothing started in the + * window while auxiliary frontends stop is something the drain then waits out (architecture + * rule 12). + */ + readonly beginStop?: () => void; + /** + * why: Releases what the daemon holds beyond its own state on shutdown, and on a start that + * fails at the claim or the `admin.token` write: timers the lease subsystem armed (quarantine + * retries), and every driver's own external resources. + * Awaited, because a driver's release is asynchronous and a stop that returned before it + * finished would report a shutdown that had not happened. + */ + readonly dispose?: () => void | Promise; + /** + * why: Stops an auxiliary frontend (today: the HTTP gateway's listener, which `worker-composition.ts` or `main.ts` starts + * from `onSocketClaimed`, partway through `start()`) before anything else in `stop()` runs, so + * no request arriving through it can ever observe a stopping engine. A no-op default when + * no auxiliary frontend is running. + */ + readonly stopAuxiliary?: () => Promise; + /** + * why: ADR 0003 §2's "an HTTP request during startup now waits like a socket request instead of + * being refused" needs the HTTP gateway actually listening *before* convergence finishes -- + * otherwise there is nothing for a request to park against, it just gets connection-refused + * the way it always did. Called synchronously right after `#readyPromise` is assigned (so any + * request `dispatch()` serves as a result of this callback already parks correctly) and after + * the socket claim and the admin secret write, but *before* awaiting convergence -- an + * auxiliary frontend started from here must not itself assume convergence is done. A no-op + * default leaves today's behaviour (no auxiliary frontend at all) unchanged. + */ + readonly onSocketClaimed?: () => void; + /** + * why: #118: a gateway's own `OwnerRoutedFacts` (`src/gateway/owner-routed-facts.ts`'s + * `GatewayOwnerRoutedFacts`), replacing the inert one gateway mode has used since #117. Worker + * mode ignores this entirely -- it builds its own `OwnerRoutedFactBus` from the engine's + * registry, exactly as before. Structurally typed (not imported) so this module need not + * depend on `src/gateway`; see that class's own doc comment for why the shapes line up with + * no cast. + */ + readonly ownerRoutedFacts?: OwnerRoutedFacts & { dispose(): void }; + /** + * why: #118: every lease this daemon currently knows about, for `lease.release-all`'s self-push + * suppression (`#handleRequest`'s `lease.release-all` case below). Worker mode reads its own + * registry directly and never needs this (`this.#engine?.registry` already answers it); a + * gateway has no registry, so it supplies its fleet lease index's own entries here instead -- + * without it, a fleet client's own `lease.release-all` would push every one of its own + * `lease-lost` facts back at itself, since nothing would ever mark them self-initiated. + * Optional so a test exercising neither engine nor gateway leases need not supply one. + */ + readonly leaseSnapshot?: () => readonly { readonly id: string; readonly ownerId: string }[]; +} + +/** + * why: ADR 0005 §32: one transport, two dispatchers. A worker passes its engine and gets the + * `Dispatcher` built for it; a gateway passes a ready-made `dispatcher` (its own handlers, over + * worker views and uplinks) and no engine at all. The union -- rather than one interface with + * everything optional -- is what keeps each mode's requirements checked at compile time. + */ +export type DaemonServerOptions = + | (DaemonServerCommonOptions & DaemonServerEngineOptions & { readonly dispatcher?: undefined }) + | (DaemonServerCommonOptions & { + readonly dispatcher: ContractDispatcher; + readonly update?: UpdateCoordinator; + }); + +export type DaemonHealth = "starting" | "running" | "failed"; + +// fallow-ignore-next-line complexity -- one conditional spread per optional dependency. +export function buildDispatcher( + options: DaemonServerCommonOptions & DaemonServerEngineOptions, + hooks: { + readonly awaitReady: () => Promise; + readonly health: () => DaemonHealth; + }, +): Dispatcher { + return new Dispatcher({ + awaitReady: hooks.awaitReady, + ...(options.capabilitySources === undefined + ? {} + : { capabilitySources: options.capabilitySources }), + capacity: options.capacity, + warmPool: options.warmPool, + catalog: options.catalog, + deviceModes: options.deviceModes, + clock: options.clock, + components: options.components, + config: options.config, + ...(options.doctor === undefined ? {} : { doctor: options.doctor }), + errorCode: classifyError, + eventBus: options.eventBus, + eventHistory: options.eventHistory, + ...(options.historyPath === undefined ? {} : { historyPath: options.historyPath }), + health: hooks.health, + hostFacts: options.hostFacts, + ...(options.load === undefined ? {} : { load: options.load }), + ...(options.memoryPressure === undefined + ? {} + : { memoryPressure: () => options.memoryPressure?.current() }), + instanceId: options.instanceId, + leases: options.leases, + ...(options.logger === undefined ? {} : { logger: options.logger }), + ...(options.enrollment === undefined ? {} : { enrollment: options.enrollment }), + ...(options.nuke === undefined ? {} : { nuke: options.nuke }), + ...(options.warmKinds === undefined ? {} : { warmKinds: options.warmKinds }), + ...(options.baselines === undefined ? {} : { baselines: options.baselines }), + ...(options.passthrough === undefined ? {} : { passthrough: options.passthrough }), + ...(options.inputs === undefined ? {} : { inputs: options.inputs }), + ...(options.processRunner === undefined ? {} : { processRunner: options.processRunner }), + ...(options.leaseFiles === undefined ? {} : { leaseFiles: options.leaseFiles }), + ...(options.execEnv === undefined ? {} : { execEnv: options.execEnv }), + queue: options.queue, + ...(options.presence === undefined ? {} : { presence: options.presence }), + reaper: options.reaper, + registry: options.registry, + ...(options.stalls === undefined ? {} : { stalls: options.stalls }), + ...(options.tokens === undefined ? {} : { tokens: options.tokens }), + ...(options.update === undefined ? {} : { update: options.update }), + version: options.version, + }); +} diff --git a/src/daemon/server-requests.ts b/src/daemon/server-requests.ts new file mode 100644 index 000000000..bde249863 --- /dev/null +++ b/src/daemon/server-requests.ts @@ -0,0 +1,530 @@ +import { type DriverRejection, NoDriverError, UnknownPassthroughToolError } from "../core/index.js"; +import { type IpcConnection, type Logger } from "../ports/index.js"; +import { type RequestFrame, type RequestId } from "../daemon-protocol/index.js"; +import { leaseRelease, relayChannelSchema } from "../contract/index.js"; +import { type LinkChannel } from "../link/index.js"; +import { type ContractDispatcher, type DispatchSession } from "./dispatcher.js"; +import { type Connection, errorMessage } from "./server-connection.js"; +import { parseInput, ProtocolError } from "./server-hello.js"; +import { ConnectionPushes } from "./server-pushes.js"; + +interface ConnectionRequestsOptions { + readonly dispatcher: ContractDispatcher; + readonly pushes: ConnectionPushes; + readonly logger: Logger; + readonly driverRejections: readonly DriverRejection[]; + readonly leaseSnapshot: () => readonly { readonly id: string; readonly ownerId: string }[]; +} + +export class ConnectionRequests { + constructor(private readonly options: ConnectionRequestsOptions) {} + + /** + * why: A `NO_DRIVER` for a platform whose driver refused to start says nothing on its own: + * it reads identically to "this host has no Xcode". Safety rule 9 promises Simlock + * reports *why* a platform is missing, so the refusal's own one-liner travels with the + * error on both paths a user meets it: leasing, and the `simlock simctl` / `simlock adb` + * wrapper, whose bare "No driver provides a adb passthrough" reads as a missing SDK and + * sends the operator off to install one instead of at the port conflict that caused it. + */ + describeError(error: unknown): string { + const message = errorMessage(error); + const rejection = this.#rejectionFor(error); + return rejection === undefined ? message : `${message} (${rejection.summary})`; + } + + #rejectionFor(error: unknown): DriverRejection | undefined { + const rejections = this.options.driverRejections; + if (error instanceof NoDriverError) { + return rejections.find((candidate) => candidate.platform === error.platform); + } + if (error instanceof UnknownPassthroughToolError) { + return rejections.find( + (candidate) => candidate.passthroughTools?.includes(error.tool) === true, + ); + } + return undefined; + } + + // fallow-ignore-next-line complexity -- one case per request type; the switch is the transport's whole surface. + async handle( + connection: Connection, + frame: RequestFrame, + relayChannel: LinkChannel | undefined, + ): Promise { + switch (frame.type) { + case "lease.request": + return this.#requestLease(connection, frame.id, frame.payload); + case "lease.release": { + const input = parseInput(leaseRelease.input, frame.payload); + connection.selfInitiatedReleases.add(input.leaseId); + try { + return await this.options.dispatcher.dispatch( + "lease.release", + frame.payload, + this.#session(connection), + ); + } finally { + connection.selfInitiatedReleases.delete(input.leaseId); + } + } + case "lease.release-all": { + // why: Every lease this principal owns is about to end, and this connection asked for it, + // so suppress the self-push for each. Read from the registry rather than from + // per-connection state, which no longer exists (ADR 0004 §3). + // + // The snapshot can skew either way against what the dispatched call actually + // releases, and both ways are acceptable: a lease that expires inside the window is + // suppressed here even though its `lease-lost` was the daemon's own doing (the caller + // asked for that lease to end and is about to be told every lease of theirs is gone), + // and one granted inside the window is not suppressed, so its holder gets the push it + // would have got for any other force-release. Neither can suppress a push for a lease + // this principal does not own, which is the property that matters. + // #118: a gateway has no `#engine`, so it falls back to `options.leaseSnapshot` (its + // fleet lease index) -- see that option's own doc comment. + const releasing = this.options + .leaseSnapshot() + .filter((lease) => connection.role === "admin" || lease.ownerId === connection.principal) + .map((lease) => lease.id); + for (const leaseId of releasing) connection.selfInitiatedReleases.add(leaseId); + try { + return await this.options.dispatcher.dispatch( + "lease.release-all", + frame.payload ?? {}, + this.#session(connection), + ); + } finally { + for (const leaseId of releasing) connection.selfInitiatedReleases.delete(leaseId); + } + } + case "lease.renew": + return this.options.dispatcher.dispatch( + "lease.renew", + frame.payload, + this.#session(connection), + ); + case "lease.cancel": + return this.options.dispatcher.dispatch( + "lease.cancel", + frame.payload ?? {}, + this.#session(connection), + ); + case "lease.get": + return this.options.dispatcher.dispatch( + "lease.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "lease.list": + return this.options.dispatcher.dispatch( + "lease.list", + frame.payload ?? {}, + this.#session(connection), + ); + case "status.get": + return this.options.dispatcher.dispatch( + "status.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "list.get": + return this.options.dispatcher.dispatch( + "list.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "catalog.get": + return this.options.dispatcher.dispatch( + "catalog.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "cleanup.run": + return this.options.dispatcher.dispatch( + "cleanup.run", + frame.payload ?? {}, + this.#session(connection), + ); + case "doctor.run": + return this.options.dispatcher.dispatch( + "doctor.run", + frame.payload ?? {}, + this.#session(connection), + ); + case "nuke.run": + return this.options.dispatcher.dispatch( + "nuke.run", + frame.payload ?? {}, + this.#session(connection), + ); + case "warm.forget": + return this.options.dispatcher.dispatch( + "warm.forget", + frame.payload ?? {}, + this.#session(connection), + ); + case "baseline.list": + return this.options.dispatcher.dispatch( + "baseline.list", + frame.payload ?? {}, + this.#session(connection), + ); + case "baseline.remove": + return this.options.dispatcher.dispatch( + "baseline.remove", + frame.payload ?? {}, + this.#session(connection), + ); + case "baseline.save": + return this.options.dispatcher.dispatch( + "baseline.save", + frame.payload ?? {}, + this.#session(connection), + ); + case "events.replay": + return this.options.dispatcher.dispatch( + "events.replay", + frame.payload ?? {}, + this.#session(connection), + ); + case "usage.get": + return this.options.dispatcher.dispatch( + "usage.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "inbox.get": + return this.options.dispatcher.dispatch( + "inbox.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "events.subscribe": + return this.options.dispatcher.dispatch( + "events.subscribe", + frame.payload ?? {}, + this.#session(connection), + ); + case "events.unsubscribe": + return this.options.dispatcher.dispatch( + "events.unsubscribe", + frame.payload ?? {}, + this.#session(connection), + ); + case "driver.passthrough": + // why: Resolution only: the daemon never runs the command. Spawning it here would attach + // a user's interactive `adb shell` to the daemon's stdio, and the CLI is the process + // that actually has a terminal. + return this.options.dispatcher.dispatch( + "driver.passthrough", + frame.payload ?? {}, + this.#session(connection), + ); + case "device.exec": + // why: Unlike `driver.passthrough` above, this one runs the command here -- so it needs the + // per-call push wiring `lease.request` needs, and gets it the same way. + return this.#execDevice(connection, frame.id, frame.payload); + case "device.rotate": + return this.options.dispatcher.dispatch( + "device.rotate", + frame.payload, + this.#session(connection), + ); + case "relay.request": + return this.#relayRequest(connection, frame.id, frame.payload, relayChannel); + case "relay.cancel": + return this.options.dispatcher.dispatch( + "relay.cancel", + frame.payload, + this.#session(connection), + ); + case "config.get": + return this.options.dispatcher.dispatch( + "config.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "identity.get": + return this.options.dispatcher.dispatch( + "identity.get", + frame.payload ?? {}, + this.#session(connection), + ); + case "token.create": + return this.options.dispatcher.dispatch( + "token.create", + frame.payload, + this.#session(connection), + ); + case "token.list": + return this.options.dispatcher.dispatch( + "token.list", + frame.payload ?? {}, + this.#session(connection), + ); + case "token.revoke": + return this.options.dispatcher.dispatch( + "token.revoke", + frame.payload, + this.#session(connection), + ); + // why: ADR 0005 §23. Listed here like any other operation: this switch is the socket's whole + // surface, and what a *given* daemon answers is the dispatcher's business, not the + // transport's -- a worker's dispatcher answers `worker.list` about itself and refuses the + // rest with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). + case "worker.list": + return this.options.dispatcher.dispatch( + "worker.list", + frame.payload ?? {}, + this.#session(connection), + ); + case "worker.drain": + return this.options.dispatcher.dispatch( + "worker.drain", + frame.payload, + this.#session(connection), + ); + case "worker.undrain": + return this.options.dispatcher.dispatch( + "worker.undrain", + frame.payload, + this.#session(connection), + ); + case "worker.remove": + return this.options.dispatcher.dispatch( + "worker.remove", + frame.payload, + this.#session(connection), + ); + case "worker.update": + return this.options.dispatcher.dispatch( + "worker.update", + frame.payload, + this.#session(connection), + ); + case "alerts.test": + return this.options.dispatcher.dispatch( + "alerts.test", + frame.payload, + this.#session(connection), + ); + case "component.install": + return this.#installComponent(connection, frame.id, "component.install", frame.payload); + case "worker.install-component": + return this.#installComponent( + connection, + frame.id, + "worker.install-component", + frame.payload, + ); + case "component.list": + return this.options.dispatcher.dispatch( + "component.list", + frame.payload ?? {}, + this.#session(connection), + ); + case "component.remove": + return this.options.dispatcher.dispatch( + "component.remove", + frame.payload, + this.#session(connection), + ); + case "device.add": + return this.options.dispatcher.dispatch( + "device.add", + frame.payload, + this.#session(connection), + ); + case "device.remove": + return this.options.dispatcher.dispatch( + "device.remove", + frame.payload, + this.#session(connection), + ); + case "daemon.update": + return this.options.dispatcher.dispatch( + "daemon.update", + frame.payload, + this.#session(connection), + ); + // why: "daemon.stop" is deliberately absent from this switch: `#dispatchLine` intercepts it + // itself, ahead of the protocol-mismatch and `#stopping` gates (ADR §6's frozen exception, + // scoped to protocol version only -- still gated on a completed handshake and the `admin` + // role there), so it never reaches `#handleRequest`. + default: + throw new ProtocolError("UNKNOWN_REQUEST", `Unknown request type: ${frame.type}`); + } + } + #session(connection: Connection): DispatchSession { + return { + isGatewayUplink: connection.isGatewayUplink, + manageEventSubscription: (subscribe) => + this.options.pushes.manageEventSubscription(connection, subscribe), + principal: connection.principal, + relays: connection.relays, + role: connection.role, + via: connection.isGatewayUplink === true ? "uplink" : "socket", + }; + } + + /** + * why: The progress-delivery wiring ADR 0003 §2 keeps in `DaemonServer` ("DaemonServer keeps + * framing, connection lifecycle, and pushes; it loses the request switch"). The actual lease + * acquisition is `Dispatcher`'s `lease.request` handler; this method's job is what is left + * around that call that depends on *this connection* rather than on the operation itself, + * which since ADR 0004 is progress routing and nothing else -- a granted lease is not + * attached to the connection that asked for it in any way. + */ + async #requestLease( + connection: Connection, + requestId: RequestId, + value: unknown, + ): Promise { + let progressSocket: IpcConnection | undefined = connection.socket; + const disposeProgress = () => { + progressSocket = undefined; + }; + connection.progressDisposers.add(disposeProgress); + // why: A connection close only silences this call's pushes (`disposeProgress`): the request + // itself keeps waiting, and a client that reconnects repeats it to get its answer. + let grant; + try { + grant = await this.options.dispatcher.dispatch("lease.request", value ?? {}, { + ...this.#session(connection), + onProgress: (progress) => { + if (progressSocket !== undefined) { + // why: Progress is advisory: one that does not fit the push contract (a `downloading` + // component the driver named past the contract's bound) or a socket that went away + // is dropped, and the request carries on to its answer. + void this.options.pushes.progress(progressSocket, requestId, progress).catch(() => { + this.options.logger.warn("A lease progress push was not sent", { + stage: progress.stage, + }); + }); + } + }, + }); + } finally { + connection.progressDisposers.delete(disposeProgress); + disposeProgress(); + } + // why: A grant that lands after its requester's connection died (or during a stop) is left + // standing on purpose: ADR 0004 §3 -- connection close means nothing to a lease, so the + // client that reconnects can renew it, and if none does it expires at its deadline like + // any other. + return grant; + } + + /** + * why: ADR 0005 §19a's output relay, and the exact counterpart of `#requestLease` above: the + * operation itself is the dispatcher's `device.exec` handler; what lives here is the part + * that depends on *this connection* -- turning each chunk into an `output` push carrying this + * call's frame id, and stopping the moment the socket goes away. + * + * The command keeps running after a disconnect rather than being killed with it. That is the + * same answer ADR 0004 §3 gives for leases (a dead connection is not a dead lease) and the + * only safe one here: a half-applied `simctl install` interrupted because a client's tunnel + * blipped is worse than one that finishes with nobody listening, and the daemon has no way to + * know which half a kill would land in. Its exit code is simply lost with the connection, and + * `exec.timeoutMs` still bounds how long it can run. + */ + async #execDevice( + connection: Connection, + requestId: RequestId, + value: unknown, + ): Promise { + let outputSocket: IpcConnection | undefined = connection.socket; + const disposeOutput = () => { + outputSocket = undefined; + }; + connection.progressDisposers.add(disposeOutput); + try { + return await this.options.dispatcher.dispatch("device.exec", value ?? {}, { + ...this.#session(connection), + // why: Returned rather than fired and forgotten, which is what applies backpressure: the + // dispatcher hands this promise to the process runner, which stops reading the child + // until the frame has actually been written (ADR 0005 §19e -- "streamed, never + // buffered" is only true end to end if a client that does not read slows the command + // down instead of filling this process). A write that fails on a socket which died + // between the check and the write (EPIPE) resolves rather than rejecting: the chunk is + // lost with the connection, the command carries on, and a rejection here would both + // wedge the child and, unhandled, take the daemon down over a client that hung up. + onOutput: async (stream, chunk) => { + if (outputSocket === undefined) return; + await this.options.pushes + .output(outputSocket, requestId, stream, chunk) + .catch(() => undefined); + }, + // why: ADR 0005 §19a: the process now exists. Fired and forgotten rather than awaited -- + // unlike `onOutput` this carries no backpressure meaning, and the dispatcher calls it + // synchronously between the spawn and the first chunk. Every peer that negotiates + // protocol 5 understands the frame (§31), so this is not a signal anyone opts out of. + onStarted: () => { + if (outputSocket === undefined) return; + void this.options.pushes.started(outputSocket, requestId).catch(() => undefined); + }, + }); + } finally { + connection.progressDisposers.delete(disposeOutput); + disposeOutput(); + } + } + + #relayRequest( + connection: Connection, + requestId: RequestId, + value: unknown, + relayChannel: LinkChannel | undefined, + ): Promise { + return this.options.dispatcher.dispatch("relay.request", value, { + ...this.#session(connection), + ...(relayChannel === undefined ? {} : { relayChannel }), + onRelayHead: (status, headers) => + this.options.pushes.relayHead(connection.socket, requestId, status, headers), + }); + } + + /** + * why: ADR 0010 §6: the counterpart of `#requestLease` for an install. Each progress update becomes + * a `component-progress` push carrying this call's frame id. A closed connection only + * silences the pushes (`writeFrame` writes nothing to a closed socket, and a write that fails + * is dropped): the install carries on for every call joined to it, and a client that + * reconnects repeats the request to join it again. + * + * A gateway's `worker.install-component` takes the same path: its progress names the worker + * it came from, and the push carries that `workerId` (ADR 0010 §7). A worker's dispatcher + * refuses it with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012 §2). + */ + #installComponent( + connection: Connection, + requestId: RequestId, + operation: "component.install" | "worker.install-component", + value: unknown, + ): Promise { + return this.options.dispatcher.dispatch(operation, value ?? {}, { + ...this.#session(connection), + onComponentProgress: (progress, workerId) => { + void this.options.pushes + .componentProgress(connection.socket, requestId, progress, workerId) + .catch(() => undefined); + }, + }); + } +} + +// why: `channel` is wire input, so its shape is checked before it names a channel; a name that fails, or one the link never saw, accepts nothing and the call fails `BAD_REQUEST` later. +export function acceptRelayChannel( + connection: Connection, + frame: RequestFrame, +): LinkChannel | undefined { + if (frame.type !== "relay.request" || connection.link === undefined) return undefined; + const payload: unknown = frame.payload; + if (typeof payload !== "object" || payload === null) return undefined; + const name = relayChannelSchema.safeParse((payload as { channel?: unknown }).channel); + if (!name.success) return undefined; + try { + return connection.link.acceptChannel(name.data); + } catch { + return undefined; + } +} diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 4bccb6154..131804499 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -1,67 +1,27 @@ import { z } from "zod"; - -import { type EventBus } from "../bus/index.js"; -import type { EventHistoryHandle } from "../history/index.js"; -import { - type ComponentInstaller, - type Config, - type Registry, - type CleanupReaper, - type Doctor, - type DriverRejection, - type HostFacts, - NoDriverError, - type Nuke, - type BaselineService, - type WarmKindForgetter, - redactConfig, - UnknownPassthroughToolError, - type CapacityReader, - type WarmPoolReader, - type CatalogReader, - type PassthroughResolver, - type PhysicalAutoEnrollment, -} from "../core/index.js"; +import { redactConfig } from "../core/index.js"; +import { type IpcConnection, type Logger, NoopLogger } from "../ports/index.js"; +import { parseRequestFrame, type RequestFrame } from "../daemon-protocol/index.js"; import { - type LeaseHealthMonitor, - type PhysicalDeviceWatcher, - type DeviceModeReader, - type LeaseCommands, - type QueueControl, -} from "../leasing/index.js"; -import type { Clock, IpcConnection, Logger, ProcessRunner } from "../ports/index.js"; -import { NoopLogger } from "../ports/index.js"; -import { parseRequestFrame, type RequestFrame, type RequestId } from "../daemon-protocol/index.js"; -import { - leaseRelease, normalizeProtocolVersion, PROTOCOL_VERSION_RANGE, - relayChannelSchema, type OperationName, type OPERATIONS, type ProtocolRange, type Role, - type StatusLoad, - type StatusMemoryPressure, } from "../contract/index.js"; -import type { LinkChannel, LinkConnection } from "../link/index.js"; -import { DaemonAlreadyRunningError, type ConnectionHost } from "./connection-host.js"; -import { - resolveAgentRole, - type Auth, - type SessionRoleResolver, - type TokenOperations, -} from "../auth/index.js"; +import { type LinkConnection } from "../link/index.js"; +import { DaemonAlreadyRunningError } from "./connection-host.js"; +import { resolveAgentRole } from "../auth/index.js"; import { Dispatcher, DispatchError, type ContractDispatcher, - type DispatcherOptions, type DispatchSession, } from "./dispatcher.js"; import { OwnerRoutedFactBus, type OwnerRoutedFacts } from "./owner-routed-facts.js"; -import { classifyError, StartupFailedError } from "./error-code.js"; -import { RelayConnection, type RelayTargets, type TwoWayRelayTargets } from "./relay.js"; +import { StartupFailedError } from "./error-code.js"; +import { RelayConnection } from "./relay.js"; import { type Connection, errorMessage, @@ -69,212 +29,20 @@ import { respondError, writeFrame, } from "./server-connection.js"; -import { errorCode, Handshake, parseInput, ProtocolError } from "./server-hello.js"; +import { errorCode, Handshake, ProtocolError } from "./server-hello.js"; import { ConnectionPushes } from "./server-pushes.js"; -import type { UpdateCoordinator } from "./update-coordinator.js"; - -/** - * why: kept in the type rather than making every field optional: a gateway passes `dispatcher` instead, - * and omitting `queue` from a worker's options stays a compile error. - */ -export interface DaemonServerEngineOptions { - readonly capabilitySources?: DispatcherOptions["capabilitySources"]; - readonly capacity: CapacityReader; - readonly warmPool: WarmPoolReader; - readonly deviceModes: DeviceModeReader; - readonly catalog: CatalogReader; - readonly components: Pick; - readonly doctor?: Doctor; - readonly eventHistory: Pick; - readonly historyPath?: string; - readonly leases: LeaseCommands; - readonly hostFacts: () => HostFacts; - readonly load?: () => StatusLoad | undefined; - readonly instanceId: string; - readonly queue: QueueControl; - readonly presence?: DispatcherOptions["presence"]; - readonly reaper: CleanupReaper; - readonly healthMonitor?: LeaseHealthMonitor | undefined; - readonly physicalDeviceWatcher?: PhysicalDeviceWatcher | undefined; - readonly autoEnrollment?: Pick | undefined; - readonly enrollment?: DispatcherOptions["enrollment"]; - readonly nuke?: Nuke; - readonly warmKinds?: WarmKindForgetter; - readonly baselines?: Pick; - readonly passthrough?: PassthroughResolver; - readonly inputs?: DispatcherOptions["inputs"]; - readonly processRunner?: ProcessRunner; - readonly leaseFiles?: DispatcherOptions["leaseFiles"]; - readonly relayTargets?: RelayTargets; - readonly relayTwoWayTargets?: TwoWayRelayTargets; - readonly execEnv?: NodeJS.ProcessEnv; - readonly registry: Registry; - readonly stalls?: DispatcherOptions["stalls"]; - readonly tokens?: TokenOperations; - readonly update?: UpdateCoordinator; -} - -export interface DaemonServerCommonOptions { - readonly clock: Clock; - readonly config: Config; - readonly defaultRequesterId: string; - /** - * why: the daemon serves without a platform whose driver refused to start (safety rule 9), so each - * refusal lands as an event in the ring buffer here. - */ - readonly driverRejections?: readonly DriverRejection[]; - readonly eventBus: EventBus; - readonly host: ConnectionHost; - readonly hostLoad?: { start(): void; dispose(): void }; - readonly memoryPressure?: { current(): StatusMemoryPressure | undefined; dispose(): void }; - readonly logger?: Logger; - readonly protocolVersion?: number | ProtocolRange; - readonly version: string; - readonly resolveRole?: SessionRoleResolver; - /** - * why: ADR §5's per-start admin secret. Undefined is a legitimate default for tests that don't - * exercise the admin handshake at all -- `start()`/`#stop()` simply skip the persist/remove - * calls below when it is absent, same style as `doctor`/`nuke`. - */ - readonly adminSecret?: Auth["adminSecret"]; - /** - * why: Startup recovery work (doctor reconciliation, running-capacity convergence) run - * *after* the socket is claimed, so reachability never depends on it. `hello` and - * `status.get` answer immediately; every other request type parks on this promise. - * A rejection here stops the daemon rather than leaving it half-open — see `start()`. - */ - readonly converge?: () => Promise; - /** - * why: Awaits lease work the subsystem is finishing off its callers' paths — today the - * backgrounded device reclaims a release hands off (see `LeaseReleaseCoordinator`). - * A graceful stop drains it so the pool is left settled; an ungraceful death leaves - * it for startup recovery. The stop itself releases nothing (ADR 0004 §3), so what this - * drains is work somebody else's release already handed off; it runs before `dispose`, so a - * quarantine retry timer armed by a reclaim settling late is still cancelled. - */ - readonly settle?: () => Promise; - /** - * why: Called synchronously the moment a stop is asked for, before anything is awaited: it closes - * to new work whatever `settle` will later wait for (the warm pool), so nothing started in the - * window while auxiliary frontends stop is something the drain then waits out (architecture - * rule 12). - */ - readonly beginStop?: () => void; - /** - * why: Releases what the daemon holds beyond its own state on shutdown, and on a start that - * fails at the claim or the `admin.token` write: timers the lease subsystem armed (quarantine - * retries), and every driver's own external resources. - * Awaited, because a driver's release is asynchronous and a stop that returned before it - * finished would report a shutdown that had not happened. - */ - readonly dispose?: () => void | Promise; - /** - * why: Stops an auxiliary frontend (today: the HTTP gateway's listener, which `worker-composition.ts` or `main.ts` starts - * from `onSocketClaimed`, partway through `start()`) before anything else in `stop()` runs, so - * no request arriving through it can ever observe a stopping engine. A no-op default when - * no auxiliary frontend is running. - */ - readonly stopAuxiliary?: () => Promise; - /** - * why: ADR 0003 §2's "an HTTP request during startup now waits like a socket request instead of - * being refused" needs the HTTP gateway actually listening *before* convergence finishes -- - * otherwise there is nothing for a request to park against, it just gets connection-refused - * the way it always did. Called synchronously right after `#readyPromise` is assigned (so any - * request `dispatch()` serves as a result of this callback already parks correctly) and after - * the socket claim and the admin secret write, but *before* awaiting convergence -- an - * auxiliary frontend started from here must not itself assume convergence is done. A no-op - * default leaves today's behaviour (no auxiliary frontend at all) unchanged. - */ - readonly onSocketClaimed?: () => void; - /** - * why: #118: a gateway's own `OwnerRoutedFacts` (`src/gateway/owner-routed-facts.ts`'s - * `GatewayOwnerRoutedFacts`), replacing the inert one gateway mode has used since #117. Worker - * mode ignores this entirely -- it builds its own `OwnerRoutedFactBus` from the engine's - * registry, exactly as before. Structurally typed (not imported) so this module need not - * depend on `src/gateway`; see that class's own doc comment for why the shapes line up with - * no cast. - */ - readonly ownerRoutedFacts?: OwnerRoutedFacts & { dispose(): void }; - /** - * why: #118: every lease this daemon currently knows about, for `lease.release-all`'s self-push - * suppression (`#handleRequest`'s `lease.release-all` case below). Worker mode reads its own - * registry directly and never needs this (`this.#engine?.registry` already answers it); a - * gateway has no registry, so it supplies its fleet lease index's own entries here instead -- - * without it, a fleet client's own `lease.release-all` would push every one of its own - * `lease-lost` facts back at itself, since nothing would ever mark them self-initiated. - * Optional so a test exercising neither engine nor gateway leases need not supply one. - */ - readonly leaseSnapshot?: () => readonly { readonly id: string; readonly ownerId: string }[]; -} - -/** - * why: ADR 0005 §32: one transport, two dispatchers. A worker passes its engine and gets the - * `Dispatcher` built for it; a gateway passes a ready-made `dispatcher` (its own handlers, over - * worker views and uplinks) and no engine at all. The union -- rather than one interface with - * everything optional -- is what keeps each mode's requirements checked at compile time. - */ -export type DaemonServerOptions = - | (DaemonServerCommonOptions & DaemonServerEngineOptions & { readonly dispatcher?: undefined }) - | (DaemonServerCommonOptions & { - readonly dispatcher: ContractDispatcher; - readonly update?: UpdateCoordinator; - }); - -type DaemonHealth = "starting" | "running" | "failed"; - -// fallow-ignore-next-line complexity -- one conditional spread per optional dependency. -function buildDispatcher( - options: DaemonServerCommonOptions & DaemonServerEngineOptions, - hooks: { - readonly awaitReady: () => Promise; - readonly health: () => DaemonHealth; - }, -): Dispatcher { - return new Dispatcher({ - awaitReady: hooks.awaitReady, - ...(options.capabilitySources === undefined - ? {} - : { capabilitySources: options.capabilitySources }), - capacity: options.capacity, - warmPool: options.warmPool, - catalog: options.catalog, - deviceModes: options.deviceModes, - clock: options.clock, - components: options.components, - config: options.config, - ...(options.doctor === undefined ? {} : { doctor: options.doctor }), - errorCode: classifyError, - eventBus: options.eventBus, - eventHistory: options.eventHistory, - ...(options.historyPath === undefined ? {} : { historyPath: options.historyPath }), - health: hooks.health, - hostFacts: options.hostFacts, - ...(options.load === undefined ? {} : { load: options.load }), - ...(options.memoryPressure === undefined - ? {} - : { memoryPressure: () => options.memoryPressure?.current() }), - instanceId: options.instanceId, - leases: options.leases, - ...(options.logger === undefined ? {} : { logger: options.logger }), - ...(options.enrollment === undefined ? {} : { enrollment: options.enrollment }), - ...(options.nuke === undefined ? {} : { nuke: options.nuke }), - ...(options.warmKinds === undefined ? {} : { warmKinds: options.warmKinds }), - ...(options.baselines === undefined ? {} : { baselines: options.baselines }), - ...(options.passthrough === undefined ? {} : { passthrough: options.passthrough }), - ...(options.inputs === undefined ? {} : { inputs: options.inputs }), - ...(options.processRunner === undefined ? {} : { processRunner: options.processRunner }), - ...(options.leaseFiles === undefined ? {} : { leaseFiles: options.leaseFiles }), - ...(options.execEnv === undefined ? {} : { execEnv: options.execEnv }), - queue: options.queue, - ...(options.presence === undefined ? {} : { presence: options.presence }), - reaper: options.reaper, - registry: options.registry, - ...(options.stalls === undefined ? {} : { stalls: options.stalls }), - ...(options.tokens === undefined ? {} : { tokens: options.tokens }), - ...(options.update === undefined ? {} : { update: options.update }), - version: options.version, - }); -} +import { + buildDispatcher, + type DaemonHealth, + type DaemonServerEngineOptions, + type DaemonServerOptions, +} from "./server-options.js"; +import { acceptRelayChannel, ConnectionRequests } from "./server-requests.js"; +export type { + DaemonServerCommonOptions, + DaemonServerEngineOptions, + DaemonServerOptions, +} from "./server-options.js"; export class DaemonServer { readonly #connections = new Set(); @@ -300,6 +68,7 @@ export class DaemonServer { readonly #workerDispatcher: Dispatcher | undefined; readonly #handshake: Handshake; readonly #pushes: ConnectionPushes; + readonly #requests: ConnectionRequests; /** * why: The worker engine, or `undefined` in gateway mode (ADR 0005 §33: a gateway has no * registry, capacity or lifecycle to hold). Every engine touchpoint below is written @@ -353,6 +122,14 @@ export class DaemonServer { // payload. Falls back to inert only for a gateway test harness that supplies neither. this.#ownerRoutedFacts = options.ownerRoutedFacts ?? inertOwnerRoutedFacts(); } + this.#requests = new ConnectionRequests({ + dispatcher: this.#dispatcher, + pushes: this.#pushes, + logger: this.#logger, + driverRejections: options.driverRejections ?? [], + leaseSnapshot: () => + this.#engine?.registry.snapshot.leases ?? this.options.leaseSnapshot?.() ?? [], + }); } get socketPath(): string { @@ -798,7 +575,7 @@ export class DaemonServer { } // why: accepted here, in the same turn the line was read, so a request that parks for startup already holds its channel (ADR 0028). - const relayChannel = this.#acceptRelayChannel(connection, frame); + const relayChannel = acceptRelayChannel(connection, frame); // why: `daemon.stop` is a frozen exception (ADR 0003 §6), but only with respect to the // *protocol-version* gate: "the daemon accepts it at any protocol version it has ever @@ -849,7 +626,7 @@ export class DaemonServer { } try { - const payload = await this.#handleRequest(connection, frame, relayChannel); + const payload = await this.#requests.handle(connection, frame, relayChannel); await writeFrame(connection.socket, { id: frame.id, ok: true, payload }); } catch (error: unknown) { relayChannel?.abort(); @@ -870,470 +647,12 @@ export class DaemonServer { connection.socket, frame.id, code, - this.#describeError(error), + this.#requests.describeError(error), error instanceof DispatchError ? error.details : undefined, ); } } - /** - * why: A `NO_DRIVER` for a platform whose driver refused to start says nothing on its own: - * it reads identically to "this host has no Xcode". Safety rule 9 promises Simlock - * reports *why* a platform is missing, so the refusal's own one-liner travels with the - * error on both paths a user meets it: leasing, and the `simlock simctl` / `simlock adb` - * wrapper, whose bare "No driver provides a adb passthrough" reads as a missing SDK and - * sends the operator off to install one instead of at the port conflict that caused it. - */ - #describeError(error: unknown): string { - const message = errorMessage(error); - const rejection = this.#rejectionFor(error); - return rejection === undefined ? message : `${message} (${rejection.summary})`; - } - - #rejectionFor(error: unknown): DriverRejection | undefined { - const rejections = this.options.driverRejections ?? []; - if (error instanceof NoDriverError) { - return rejections.find((candidate) => candidate.platform === error.platform); - } - if (error instanceof UnknownPassthroughToolError) { - return rejections.find( - (candidate) => candidate.passthroughTools?.includes(error.tool) === true, - ); - } - return undefined; - } - - // fallow-ignore-next-line complexity -- one case per request type; the switch is the transport's whole surface. - async #handleRequest( - connection: Connection, - frame: RequestFrame, - relayChannel: LinkChannel | undefined, - ): Promise { - switch (frame.type) { - case "lease.request": - return this.#requestLease(connection, frame.id, frame.payload); - case "lease.release": { - const input = parseInput(leaseRelease.input, frame.payload); - connection.selfInitiatedReleases.add(input.leaseId); - try { - return await this.#dispatcher.dispatch( - "lease.release", - frame.payload, - this.#session(connection), - ); - } finally { - connection.selfInitiatedReleases.delete(input.leaseId); - } - } - case "lease.release-all": { - // why: Every lease this principal owns is about to end, and this connection asked for it, - // so suppress the self-push for each. Read from the registry rather than from - // per-connection state, which no longer exists (ADR 0004 §3). - // - // The snapshot can skew either way against what the dispatched call actually - // releases, and both ways are acceptable: a lease that expires inside the window is - // suppressed here even though its `lease-lost` was the daemon's own doing (the caller - // asked for that lease to end and is about to be told every lease of theirs is gone), - // and one granted inside the window is not suppressed, so its holder gets the push it - // would have got for any other force-release. Neither can suppress a push for a lease - // this principal does not own, which is the property that matters. - // #118: a gateway has no `#engine`, so it falls back to `options.leaseSnapshot` (its - // fleet lease index) -- see that option's own doc comment. - const releasing = ( - this.#engine?.registry.snapshot.leases ?? - this.options.leaseSnapshot?.() ?? - [] - ) - .filter((lease) => connection.role === "admin" || lease.ownerId === connection.principal) - .map((lease) => lease.id); - for (const leaseId of releasing) connection.selfInitiatedReleases.add(leaseId); - try { - return await this.#dispatcher.dispatch( - "lease.release-all", - frame.payload ?? {}, - this.#session(connection), - ); - } finally { - for (const leaseId of releasing) connection.selfInitiatedReleases.delete(leaseId); - } - } - case "lease.renew": - return this.#dispatcher.dispatch("lease.renew", frame.payload, this.#session(connection)); - case "lease.cancel": - return this.#dispatcher.dispatch( - "lease.cancel", - frame.payload ?? {}, - this.#session(connection), - ); - case "lease.get": - return this.#dispatcher.dispatch( - "lease.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "lease.list": - return this.#dispatcher.dispatch( - "lease.list", - frame.payload ?? {}, - this.#session(connection), - ); - case "status.get": - return this.#dispatcher.dispatch( - "status.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "list.get": - return this.#dispatcher.dispatch( - "list.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "catalog.get": - return this.#dispatcher.dispatch( - "catalog.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "cleanup.run": - return this.#dispatcher.dispatch( - "cleanup.run", - frame.payload ?? {}, - this.#session(connection), - ); - case "doctor.run": - return this.#dispatcher.dispatch( - "doctor.run", - frame.payload ?? {}, - this.#session(connection), - ); - case "nuke.run": - return this.#dispatcher.dispatch( - "nuke.run", - frame.payload ?? {}, - this.#session(connection), - ); - case "warm.forget": - return this.#dispatcher.dispatch( - "warm.forget", - frame.payload ?? {}, - this.#session(connection), - ); - case "baseline.list": - return this.#dispatcher.dispatch( - "baseline.list", - frame.payload ?? {}, - this.#session(connection), - ); - case "baseline.remove": - return this.#dispatcher.dispatch( - "baseline.remove", - frame.payload ?? {}, - this.#session(connection), - ); - case "baseline.save": - return this.#dispatcher.dispatch( - "baseline.save", - frame.payload ?? {}, - this.#session(connection), - ); - case "events.replay": - return this.#dispatcher.dispatch( - "events.replay", - frame.payload ?? {}, - this.#session(connection), - ); - case "usage.get": - return this.#dispatcher.dispatch( - "usage.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "inbox.get": - return this.#dispatcher.dispatch( - "inbox.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "events.subscribe": - return this.#dispatcher.dispatch( - "events.subscribe", - frame.payload ?? {}, - this.#session(connection), - ); - case "events.unsubscribe": - return this.#dispatcher.dispatch( - "events.unsubscribe", - frame.payload ?? {}, - this.#session(connection), - ); - case "driver.passthrough": - // why: Resolution only: the daemon never runs the command. Spawning it here would attach - // a user's interactive `adb shell` to the daemon's stdio, and the CLI is the process - // that actually has a terminal. - return this.#dispatcher.dispatch( - "driver.passthrough", - frame.payload ?? {}, - this.#session(connection), - ); - case "device.exec": - // why: Unlike `driver.passthrough` above, this one runs the command here -- so it needs the - // per-call push wiring `lease.request` needs, and gets it the same way. - return this.#execDevice(connection, frame.id, frame.payload); - case "device.rotate": - return this.#dispatcher.dispatch("device.rotate", frame.payload, this.#session(connection)); - case "relay.request": - return this.#relayRequest(connection, frame.id, frame.payload, relayChannel); - case "relay.cancel": - return this.#dispatcher.dispatch("relay.cancel", frame.payload, this.#session(connection)); - case "config.get": - return this.#dispatcher.dispatch( - "config.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "identity.get": - return this.#dispatcher.dispatch( - "identity.get", - frame.payload ?? {}, - this.#session(connection), - ); - case "token.create": - return this.#dispatcher.dispatch("token.create", frame.payload, this.#session(connection)); - case "token.list": - return this.#dispatcher.dispatch( - "token.list", - frame.payload ?? {}, - this.#session(connection), - ); - case "token.revoke": - return this.#dispatcher.dispatch("token.revoke", frame.payload, this.#session(connection)); - // why: ADR 0005 §23. Listed here like any other operation: this switch is the socket's whole - // surface, and what a *given* daemon answers is the dispatcher's business, not the - // transport's -- a worker's dispatcher answers `worker.list` about itself and refuses the - // rest with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). - case "worker.list": - return this.#dispatcher.dispatch( - "worker.list", - frame.payload ?? {}, - this.#session(connection), - ); - case "worker.drain": - return this.#dispatcher.dispatch("worker.drain", frame.payload, this.#session(connection)); - case "worker.undrain": - return this.#dispatcher.dispatch( - "worker.undrain", - frame.payload, - this.#session(connection), - ); - case "worker.remove": - return this.#dispatcher.dispatch("worker.remove", frame.payload, this.#session(connection)); - case "worker.update": - return this.#dispatcher.dispatch("worker.update", frame.payload, this.#session(connection)); - case "alerts.test": - return this.#dispatcher.dispatch("alerts.test", frame.payload, this.#session(connection)); - case "component.install": - return this.#installComponent(connection, frame.id, "component.install", frame.payload); - case "worker.install-component": - return this.#installComponent( - connection, - frame.id, - "worker.install-component", - frame.payload, - ); - case "component.list": - return this.#dispatcher.dispatch( - "component.list", - frame.payload ?? {}, - this.#session(connection), - ); - case "component.remove": - return this.#dispatcher.dispatch( - "component.remove", - frame.payload, - this.#session(connection), - ); - case "device.add": - return this.#dispatcher.dispatch("device.add", frame.payload, this.#session(connection)); - case "device.remove": - return this.#dispatcher.dispatch("device.remove", frame.payload, this.#session(connection)); - case "daemon.update": - return this.#dispatcher.dispatch("daemon.update", frame.payload, this.#session(connection)); - // why: "daemon.stop" is deliberately absent from this switch: `#dispatchLine` intercepts it - // itself, ahead of the protocol-mismatch and `#stopping` gates (ADR §6's frozen exception, - // scoped to protocol version only -- still gated on a completed handshake and the `admin` - // role there), so it never reaches `#handleRequest`. - default: - throw new ProtocolError("UNKNOWN_REQUEST", `Unknown request type: ${frame.type}`); - } - } - - // why: `channel` is wire input, so its shape is checked before it names a channel; a name that fails, or one the link never saw, accepts nothing and the call fails `BAD_REQUEST` later. - #acceptRelayChannel(connection: Connection, frame: RequestFrame): LinkChannel | undefined { - if (frame.type !== "relay.request" || connection.link === undefined) return undefined; - const payload: unknown = frame.payload; - if (typeof payload !== "object" || payload === null) return undefined; - const name = relayChannelSchema.safeParse((payload as { channel?: unknown }).channel); - if (!name.success) return undefined; - try { - return connection.link.acceptChannel(name.data); - } catch { - return undefined; - } - } - - #session(connection: Connection): DispatchSession { - return { - isGatewayUplink: connection.isGatewayUplink, - manageEventSubscription: (subscribe) => - this.#pushes.manageEventSubscription(connection, subscribe), - principal: connection.principal, - relays: connection.relays, - role: connection.role, - via: connection.isGatewayUplink === true ? "uplink" : "socket", - }; - } - - /** - * why: The progress-delivery wiring ADR 0003 §2 keeps in `DaemonServer` ("DaemonServer keeps - * framing, connection lifecycle, and pushes; it loses the request switch"). The actual lease - * acquisition is `Dispatcher`'s `lease.request` handler; this method's job is what is left - * around that call that depends on *this connection* rather than on the operation itself, - * which since ADR 0004 is progress routing and nothing else -- a granted lease is not - * attached to the connection that asked for it in any way. - */ - async #requestLease( - connection: Connection, - requestId: RequestId, - value: unknown, - ): Promise { - let progressSocket: IpcConnection | undefined = connection.socket; - const disposeProgress = () => { - progressSocket = undefined; - }; - connection.progressDisposers.add(disposeProgress); - // why: A connection close only silences this call's pushes (`disposeProgress`): the request - // itself keeps waiting, and a client that reconnects repeats it to get its answer. - let grant; - try { - grant = await this.#dispatcher.dispatch("lease.request", value ?? {}, { - ...this.#session(connection), - onProgress: (progress) => { - if (progressSocket !== undefined) { - // why: Progress is advisory: one that does not fit the push contract (a `downloading` - // component the driver named past the contract's bound) or a socket that went away - // is dropped, and the request carries on to its answer. - void this.#pushes.progress(progressSocket, requestId, progress).catch(() => { - this.#logger.warn("A lease progress push was not sent", { stage: progress.stage }); - }); - } - }, - }); - } finally { - connection.progressDisposers.delete(disposeProgress); - disposeProgress(); - } - // why: A grant that lands after its requester's connection died (or during a stop) is left - // standing on purpose: ADR 0004 §3 -- connection close means nothing to a lease, so the - // client that reconnects can renew it, and if none does it expires at its deadline like - // any other. - return grant; - } - - /** - * why: ADR 0005 §19a's output relay, and the exact counterpart of `#requestLease` above: the - * operation itself is the dispatcher's `device.exec` handler; what lives here is the part - * that depends on *this connection* -- turning each chunk into an `output` push carrying this - * call's frame id, and stopping the moment the socket goes away. - * - * The command keeps running after a disconnect rather than being killed with it. That is the - * same answer ADR 0004 §3 gives for leases (a dead connection is not a dead lease) and the - * only safe one here: a half-applied `simctl install` interrupted because a client's tunnel - * blipped is worse than one that finishes with nobody listening, and the daemon has no way to - * know which half a kill would land in. Its exit code is simply lost with the connection, and - * `exec.timeoutMs` still bounds how long it can run. - */ - async #execDevice( - connection: Connection, - requestId: RequestId, - value: unknown, - ): Promise { - let outputSocket: IpcConnection | undefined = connection.socket; - const disposeOutput = () => { - outputSocket = undefined; - }; - connection.progressDisposers.add(disposeOutput); - try { - return await this.#dispatcher.dispatch("device.exec", value ?? {}, { - ...this.#session(connection), - // why: Returned rather than fired and forgotten, which is what applies backpressure: the - // dispatcher hands this promise to the process runner, which stops reading the child - // until the frame has actually been written (ADR 0005 §19e -- "streamed, never - // buffered" is only true end to end if a client that does not read slows the command - // down instead of filling this process). A write that fails on a socket which died - // between the check and the write (EPIPE) resolves rather than rejecting: the chunk is - // lost with the connection, the command carries on, and a rejection here would both - // wedge the child and, unhandled, take the daemon down over a client that hung up. - onOutput: async (stream, chunk) => { - if (outputSocket === undefined) return; - await this.#pushes.output(outputSocket, requestId, stream, chunk).catch(() => undefined); - }, - // why: ADR 0005 §19a: the process now exists. Fired and forgotten rather than awaited -- - // unlike `onOutput` this carries no backpressure meaning, and the dispatcher calls it - // synchronously between the spawn and the first chunk. Every peer that negotiates - // protocol 5 understands the frame (§31), so this is not a signal anyone opts out of. - onStarted: () => { - if (outputSocket === undefined) return; - void this.#pushes.started(outputSocket, requestId).catch(() => undefined); - }, - }); - } finally { - connection.progressDisposers.delete(disposeOutput); - disposeOutput(); - } - } - - #relayRequest( - connection: Connection, - requestId: RequestId, - value: unknown, - relayChannel: LinkChannel | undefined, - ): Promise { - return this.#dispatcher.dispatch("relay.request", value, { - ...this.#session(connection), - ...(relayChannel === undefined ? {} : { relayChannel }), - onRelayHead: (status, headers) => - this.#pushes.relayHead(connection.socket, requestId, status, headers), - }); - } - - /** - * why: ADR 0010 §6: the counterpart of `#requestLease` for an install. Each progress update becomes - * a `component-progress` push carrying this call's frame id. A closed connection only - * silences the pushes (`writeFrame` writes nothing to a closed socket, and a write that fails - * is dropped): the install carries on for every call joined to it, and a client that - * reconnects repeats the request to join it again. - * - * A gateway's `worker.install-component` takes the same path: its progress names the worker - * it came from, and the push carries that `workerId` (ADR 0010 §7). A worker's dispatcher - * refuses it with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012 §2). - */ - #installComponent( - connection: Connection, - requestId: RequestId, - operation: "component.install" | "worker.install-component", - value: unknown, - ): Promise { - return this.#dispatcher.dispatch(operation, value ?? {}, { - ...this.#session(connection), - onComponentProgress: (progress, workerId) => { - void this.#pushes - .componentProgress(connection.socket, requestId, progress, workerId) - .catch(() => undefined); - }, - }); - } - #closeConnection(connection: Connection): void { if (connection.closed) { return; From a163133026eaef34538af1cadb9b1239d12ef4ad Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:14:38 +0100 Subject: [PATCH 16/83] refactor(daemon): split dispatcher.ts into parts --- src/daemon/dispatcher-admin.ts | 76 ++ src/daemon/dispatcher-capabilities.ts | 71 ++ src/daemon/dispatcher-components.ts | 176 ++++ src/daemon/dispatcher-device-views.ts | 71 ++ src/daemon/dispatcher-devices.ts | 121 +++ src/daemon/dispatcher-events.ts | 65 ++ src/daemon/dispatcher-exec.ts | 268 +++++ src/daemon/dispatcher-fleet.ts | 171 +++ src/daemon/dispatcher-leases.ts | 141 +++ src/daemon/dispatcher-options.ts | 121 +++ src/daemon/dispatcher-status.ts | 88 ++ src/daemon/dispatcher.ts | 1390 ++----------------------- 12 files changed, 1450 insertions(+), 1309 deletions(-) create mode 100644 src/daemon/dispatcher-admin.ts create mode 100644 src/daemon/dispatcher-capabilities.ts create mode 100644 src/daemon/dispatcher-components.ts create mode 100644 src/daemon/dispatcher-device-views.ts create mode 100644 src/daemon/dispatcher-devices.ts create mode 100644 src/daemon/dispatcher-events.ts create mode 100644 src/daemon/dispatcher-exec.ts create mode 100644 src/daemon/dispatcher-fleet.ts create mode 100644 src/daemon/dispatcher-leases.ts create mode 100644 src/daemon/dispatcher-options.ts create mode 100644 src/daemon/dispatcher-status.ts diff --git a/src/daemon/dispatcher-admin.ts b/src/daemon/dispatcher-admin.ts new file mode 100644 index 000000000..2ca85dc77 --- /dev/null +++ b/src/daemon/dispatcher-admin.ts @@ -0,0 +1,76 @@ +import type { TokenOperations } from "../auth/index.js"; +import { DispatchError } from "./dispatch.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; + +export class DoctorUnavailableError extends Error { + constructor() { + super("Doctor is unavailable"); + this.name = "DoctorUnavailableError"; + } +} + +export class NukeUnavailableError extends Error { + constructor() { + super("Nuke is unavailable"); + this.name = "NukeUnavailableError"; + } +} + +export class AdminHandlers { + constructor(private readonly options: DispatcherOptions) {} + + readonly daemonUpdate: Handler<"daemon.update"> = async (input) => { + const update = this.options.update; + if (update === undefined) { + throw new DispatchError("UNKNOWN_REQUEST", "This daemon does not update itself"); + } + return { + ...(await update.request(input)), + version: this.options.version, + health: this.options.health(), + }; + }; + + readonly doctorRun: Handler<"doctor.run"> = async (input) => { + if (this.options.doctor === undefined) throw new DoctorUnavailableError(); + // why: `purgeOrphans` travels as its own flag all the way down, never folded into `fix`: + // someone already running `doctor --fix` unattended in CI must not acquire a + // destructive behaviour by upgrading (ADR 0001, decision 6). + return this.options.doctor.reconcile({ + fix: input.fix ?? false, + prerequisites: true, + purgeOrphans: input.purgeOrphans ?? false, + }); + }; + + readonly cleanupRun: Handler<"cleanup.run"> = async (input) => + this.options.reaper.run({ + dryRun: input.dryRun ?? false, + ...(input.rule === undefined ? {} : { rule: input.rule }), + }); + + readonly nukeRun: Handler<"nuke.run"> = async (input) => { + if (this.options.nuke === undefined) throw new NukeUnavailableError(); + return this.options.nuke.run({ deleteDevices: input.deleteDevices ?? false }); + }; + + readonly tokenCreate: Handler<"token.create"> = async (input) => { + const { record, secret } = await requireTokens(this.options).create(input.role, input.label); + return { secret, token: record }; + }; + + readonly tokenList: Handler<"token.list"> = async () => ({ + tokens: await requireTokens(this.options).list(), + }); + + readonly tokenRevoke: Handler<"token.revoke"> = async (input) => ({ + revoked: await requireTokens(this.options).revoke(input.id), + }); +} + +export function requireTokens(options: DispatcherOptions): TokenOperations { + if (options.tokens === undefined) { + throw new DispatchError("INTERNAL", "Token store is unavailable"); + } + return options.tokens; +} diff --git a/src/daemon/dispatcher-capabilities.ts b/src/daemon/dispatcher-capabilities.ts new file mode 100644 index 000000000..781105dd0 --- /dev/null +++ b/src/daemon/dispatcher-capabilities.ts @@ -0,0 +1,71 @@ +import type { z } from "zod"; + +import { capabilityReason, type capabilitySchema } from "../contract/index.js"; +import type { TimerHandle } from "../ports/index.js"; +import { DispatchError } from "./dispatch.js"; +import type { CapabilitySource, DispatcherOptions } from "./dispatcher-options.js"; + +const CAPABILITY_READ_BUDGET_MS = 1_000; + +type CapabilityEntry = z.infer; + +// why: a module can never make status.get fail or hang, which would cut the worker off from its gateway. +export async function readCapabilities( + options: DispatcherOptions, +): Promise { + const sources = options.capabilitySources; + if (sources === undefined || sources.length === 0) return undefined; + return Promise.all(sources.map((source) => readCapability(options, source))); +} + +async function readCapability( + options: DispatcherOptions, + source: CapabilitySource, +): Promise { + const unreadable: CapabilityEntry = { + name: source.name, + reason: capabilityReason(`could not read ${source.name}'s status`), + state: "unavailable", + }; + let timer: TimerHandle | undefined; + const expired = new Promise((resolve) => { + timer = options.clock.setTimer(CAPABILITY_READ_BUDGET_MS, () => resolve(undefined)); + }); + try { + const answer = await Promise.race([source.read().catch((): undefined => undefined), expired]); + if (answer === undefined) return unreadable; + if (answer.state === "available" || answer.state === "off") { + return { name: source.name, state: answer.state }; + } + if (answer.state !== "unavailable") { + return { name: source.name, reason: "reported an invalid status", state: "unavailable" }; + } + return { + name: source.name, + reason: capabilityReason(answer.reason ?? ""), + state: "unavailable", + }; + } finally { + if (timer !== undefined) options.clock.cancel(timer); + } +} + +// why: a request that waits in the worker's queue is not checked again; the check runs once, before leasing. +export async function requireCapabilities( + options: DispatcherOptions, + requires: readonly string[] | undefined, +): Promise { + if (requires === undefined || requires.length === 0) return; + const sources = options.capabilitySources ?? []; + const states = await Promise.all( + requires.map(async (name) => { + const source = sources.find((candidate) => candidate.name === name); + return source === undefined ? undefined : readCapability(options, source); + }), + ); + const missing = requires.filter((_name, index) => states[index]?.state !== "available"); + if (missing.length === 0) return; + throw new DispatchError("CAPABILITY_MISSING", `This worker cannot serve ${missing.join(", ")}`, { + capabilities: missing, + }); +} diff --git a/src/daemon/dispatcher-components.ts b/src/daemon/dispatcher-components.ts new file mode 100644 index 000000000..daec8ab73 --- /dev/null +++ b/src/daemon/dispatcher-components.ts @@ -0,0 +1,176 @@ +import type { ComponentProgress } from "../contract/index.js"; +import { + BaselineInUseError, + BaselineSaveFailedError, + ComponentInUseError, + effectiveAllowDownload, + UnknownBaselineError, + UnknownLeaseError, + type ComponentInstallerProgress, +} from "../core/index.js"; +import { DispatchError } from "./dispatch.js"; +import { describeEnrollmentError } from "./dispatcher-devices.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; + +export class ComponentHandlers { + constructor(private readonly options: DispatcherOptions) {} + + readonly warmForget: Handler<"warm.forget"> = async (input) => { + if (this.options.warmKinds === undefined) + throw new DispatchError("INTERNAL", "The warm pool is unavailable"); + const forgotten = this.options.warmKinds.forget(input); + return { + forgotten: forgotten.map((spec) => ({ + platform: spec.platform, + model: spec.model, + osVersion: spec.osVersion, + mode: spec.mode ?? "full", + ...(spec.imageTag === undefined ? {} : { imageTag: spec.imageTag }), + ...(spec.baseline === undefined ? {} : { baseline: spec.baseline.name }), + })), + }; + }; + + readonly baselineSave: Handler<"baseline.save"> = async (input, session) => { + if (this.options.baselines === undefined) + throw new DispatchError("INTERNAL", "Baselines are unavailable"); + const lease = this.options.registry.snapshot.leases.find( + (candidate) => candidate.id === input.leaseId, + ); + if (lease === undefined) throw new UnknownLeaseError(input.leaseId); + const held = this.options.registry.snapshot.devices.find( + (candidate) => candidate.id === lease.deviceId, + ); + if (held?.physical === true) { + throw new DispatchError( + "BAD_REQUEST", + `lease ${lease.id} is on a physical device; only a simulator or emulator can be saved as a baseline`, + ); + } + try { + return await this.options.baselines.save({ + leaseId: input.leaseId, + name: input.name, + ...(input.note === undefined ? {} : { note: input.note }), + requesterId: session.principal, + }); + } catch (error: unknown) { + if (error instanceof BaselineSaveFailedError) { + throw new DispatchError("BASELINE_SAVE_FAILED", error.message, { + leaseId: error.leaseId, + name: error.baselineName, + }); + } + throw error; + } + }; + + readonly baselineList: Handler<"baseline.list"> = async () => { + if (this.options.baselines === undefined) + throw new DispatchError("INTERNAL", "Baselines are unavailable"); + return { baselines: [...(await this.options.baselines.list())] }; + }; + + readonly baselineRemove: Handler<"baseline.remove"> = async (input, session) => { + if (this.options.baselines === undefined) + throw new DispatchError("INTERNAL", "Baselines are unavailable"); + if (input.workerId !== undefined) { + throw new DispatchError( + "BAD_REQUEST", + "workerId names a gateway's worker, and this daemon is not a gateway; remove without --worker", + ); + } + try { + const removed = await this.options.baselines.remove({ + name: input.name, + requesterId: session.principal, + }); + return { + deletedDevices: [...removed.deletedDevices], + name: removed.name, + versions: [...removed.versions], + }; + } catch (error: unknown) { + if (error instanceof BaselineInUseError) { + throw new DispatchError("BASELINE_IN_USE", error.message, { + leaseId: error.leaseId, + name: error.baselineName, + }); + } + if (error instanceof UnknownBaselineError) { + throw new DispatchError("UNKNOWN_BASELINE", error.message, { name: error.baselineName }); + } + throw describeEnrollmentError(error); + } + }; + + readonly componentInstall: Handler<"component.install"> = async (input, session) => { + if (!effectiveAllowDownload(this.options.config.downloads.policy, true)) { + throw new DispatchError( + "DOWNLOADS_DISABLED", + 'Downloads are disabled by configuration: downloads.policy is "never"', + { policy: "never" }, + ); + } + const onComponentProgress = session.onComponentProgress; + const onStarted = session.onStarted; + const result = await this.options.components.install({ + component: input.version, + platform: input.platform, + requesterId: session.principal, + ...(onComponentProgress === undefined + ? {} + : { onProgress: (progress) => onComponentProgress(toWireProgress(progress)) }), + ...(onStarted === undefined ? {} : { onAdmitted: onStarted }), + }); + // why: No `stillNeeded` is passed, so `not-needed` cannot come back; a version is always present + // for the two outcomes that can. + if (result.outcome === "not-needed" || result.version === undefined) { + throw new Error(`Internal: component install ended as ${result.outcome} with no version`); + } + return { + component: input.version, + outcome: result.outcome, + platform: input.platform, + version: result.version, + }; + }; + + readonly componentList: Handler<"component.list"> = async (input) => ({ + components: await this.options.components.list(input.platform), + }); + + readonly componentRemove: Handler<"component.remove"> = async (input, session) => { + try { + const result = await this.options.components.remove({ + platform: input.platform, + requesterId: session.principal, + version: input.version, + ...(input.dryRun === undefined ? {} : { dryRun: input.dryRun }), + }); + return { platform: input.platform, version: input.version, ...result }; + } catch (error: unknown) { + if (error instanceof ComponentInUseError) { + throw new DispatchError("COMPONENT_IN_USE", error.message, { + devices: error.devices, + foreignDevices: error.foreignDevices, + }); + } + throw error; + } + }; +} + +/** + * why: The installer's progress in the contract's words: its percentage, already within 0..100, + * becomes a fraction from 0 to 1, rounded to three decimals so that 1.1 percent is 0.011 and not + * the 0.011000000000000001 a plain division gives. Only 100 percent becomes 1: 99.96 would round + * up to it, and 1 is what an install says once it is complete. With none (the install has just + * started, or the driver's was not a number) there is no fraction. + */ +function toWireProgress(progress: ComponentInstallerProgress): ComponentProgress { + if (progress.stage === "waiting") return { stage: "waiting" }; + if (progress.percent === undefined) return { stage: "downloading" }; + const fraction = Math.round(progress.percent * 10) / 1000; + return { fraction: progress.percent < 100 ? Math.min(fraction, 0.999) : 1, stage: "downloading" }; +} diff --git a/src/daemon/dispatcher-device-views.ts b/src/daemon/dispatcher-device-views.ts new file mode 100644 index 000000000..6db93015e --- /dev/null +++ b/src/daemon/dispatcher-device-views.ts @@ -0,0 +1,71 @@ +import { + isStalledTransition, + transitionEnteredAt, + type DeviceRecord, + type LatestPresence, + type PhysicalClass, +} from "../core/index.js"; +import type { DispatchSession } from "./dispatch.js"; +import type { DispatcherOptions } from "./dispatcher-options.js"; + +// why: an agent token must not learn other principals' leases or requester ids; admin sees all. +export function leasesFor(options: DispatcherOptions, session: DispatchSession) { + return options.registry.snapshot.leases.filter( + (lease) => session.role === "admin" || lease.ownerId === session.principal, + ); +} + +// why: a waiting entry carries no owner, so the queue's recorded owner of its requester on that row's platform decides. +export function waitingFor(options: DispatcherOptions, session: DispatchSession) { + return options.queue + .waitingRequests() + .filter( + (request) => + session.role === "admin" || + options.queue.pendingRequestOwner(request.requesterId, request.spec.platform) === + session.principal, + ); +} + +export function decorateDevice( + options: DispatcherOptions, + device: DeviceRecord, +): DeviceRecord & { + readonly class?: PhysicalClass; + readonly servesDefaultMode: boolean; + readonly presence?: LatestPresence; + readonly transitionAgeMs?: number; + readonly stalled?: true; +} { + // why: `reclaimingSince` only ages a transition: it stays off the wire (`transitionAgeMs` carries it). + const { reclaimingSince: _internal, ...onTheWire } = device as DeviceRecord & { + readonly reclaimingSince?: number; + }; + const presence = device.physical ? options.presence?.(device.driverDeviceId) : undefined; + const decorated = { + ...onTheWire, + ...(device.physical ? { class: device.spec.class } : {}), + ...(presence === undefined ? {} : { presence }), + servesDefaultMode: options.deviceModes.servesDefaultMode(device.spec), + }; + const enteredAt = transitionEnteredAt(device); + if (enteredAt === undefined) return decorated; + const now = options.clock.now(); + return { + ...decorated, + transitionAgeMs: now - enteredAt, + ...(isStalled(options, device, now) ? { stalled: true as const } : {}), + }; +} + +function isStalled(options: DispatcherOptions, device: DeviceRecord, now: number): boolean { + const stalls = options.stalls; + if (stalls === undefined) return false; + return isStalledTransition({ + claims: stalls.claims, + config: options.config.stalledTransition, + device, + driver: stalls.drivers.find((driver) => driver.platform === device.spec.platform), + now, + }); +} diff --git a/src/daemon/dispatcher-devices.ts b/src/daemon/dispatcher-devices.ts new file mode 100644 index 000000000..95369ffe5 --- /dev/null +++ b/src/daemon/dispatcher-devices.ts @@ -0,0 +1,121 @@ +import { + DeviceBusyError, + DeviceNotEnrollableError, + isUnavailableInput, + toDriverDevice, + UnknownEnrolledDeviceError, + UnknownLeaseError, + type PhysicalEnrollment, +} from "../core/index.js"; +import { DispatchError } from "./dispatch.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; +import { rotationRefusal } from "./rotation-refusal.js"; + +export class DeviceHandlers { + constructor(private readonly options: DispatcherOptions) {} + + #leaseIsActive(leaseId: string): boolean { + return this.options.registry.snapshot.leases.some((candidate) => candidate.id === leaseId); + } + + readonly deviceRotate: Handler<"device.rotate"> = async (input) => { + if (this.options.inputs === undefined) { + throw new DispatchError("INTERNAL", "Device input is unavailable"); + } + const lease = this.options.registry.snapshot.leases.find( + (candidate) => candidate.id === input.leaseId, + ); + if (lease === undefined) throw new UnknownLeaseError(input.leaseId); + const record = this.options.registry.snapshot.devices.find( + (candidate) => candidate.id === lease.deviceId, + ); + if (record === undefined) throw new UnknownLeaseError(input.leaseId); + const refusal = rotationRefusal(input.leaseId, record.spec); + if (refusal !== undefined) throw refusal; + try { + const held = await this.options.inputs.leaseInput( + input.leaseId, + record.spec.platform, + toDriverDevice(record), + ); + try { + if (isUnavailableInput(held.input)) throw new Error(held.input.unavailable); + if (!this.#leaseIsActive(input.leaseId)) throw new UnknownLeaseError(input.leaseId); + await held.input.send({ kind: "orientation", orientation: input.orientation }); + } finally { + held.release(); + } + } catch (error) { + if (error instanceof UnknownLeaseError) throw error; + throw inputUnavailable(input.leaseId, error); + } + return { leaseId: input.leaseId, orientation: input.orientation }; + }; + + readonly deviceAdd: Handler<"device.add"> = async (input) => { + try { + const { device, reenrolled } = await this.#requireEnrollment().add(input.platform, input.id); + return { + appCount: device.enrolledApps.length, + class: device.spec.class, + driverDeviceId: device.driverDeviceId, + id: device.id, + model: device.spec.model, + osVersion: device.spec.osVersion, + platform: device.spec.platform, + reenrolled, + }; + } catch (error: unknown) { + throw describeEnrollmentError(error); + } + }; + + readonly deviceRemove: Handler<"device.remove"> = async (input) => { + try { + const device = await this.#requireEnrollment().remove(input.id); + return { driverDeviceId: device.driverDeviceId, id: device.id }; + } catch (error: unknown) { + throw describeEnrollmentError(error); + } + }; + + #requireEnrollment(): Pick { + if (this.options.enrollment === undefined) { + throw new DispatchError("INTERNAL", "Device enrollment is unavailable"); + } + return this.options.enrollment; + } +} + +export function describeEnrollmentError(error: unknown): unknown { + if (error instanceof DeviceNotEnrollableError) { + return new DispatchError("DEVICE_NOT_ENROLLABLE", error.message, { + id: error.id, + platform: error.platform, + reason: error.reason, + ...(error.canonicalId === undefined ? {} : { canonicalId: error.canonicalId }), + }); + } + if (error instanceof DeviceBusyError) { + return new DispatchError("DEVICE_BUSY", error.message, { + id: error.id, + reason: error.reason, + }); + } + if (error instanceof UnknownEnrolledDeviceError) { + return new DispatchError("UNKNOWN_DEVICE", error.message, { id: error.id }); + } + return error; +} + +const INPUT_REASON_MAX = 200; + +function inputUnavailable(leaseId: string, error: unknown): DispatchError { + const text = error instanceof Error ? error.message : String(error); + const reason = text === "" ? "unknown error" : text.slice(0, INPUT_REASON_MAX); + return new DispatchError( + "INPUT_UNAVAILABLE", + `Lease ${leaseId}'s simulator cannot take input: ${reason}`, + { leaseId, reason }, + ); +} diff --git a/src/daemon/dispatcher-events.ts b/src/daemon/dispatcher-events.ts new file mode 100644 index 000000000..713fee22f --- /dev/null +++ b/src/daemon/dispatcher-events.ts @@ -0,0 +1,65 @@ +import { EVENTS_PAGE_DEFAULT } from "../contract/index.js"; +import { createUsageReader, type UsageReader } from "../core/index.js"; +import { DispatchError, eventsPageAnswer, usageAnswer } from "./dispatch.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; + +export class EventHandlers { + readonly #usage: UsageReader; + + constructor(private readonly options: DispatcherOptions) { + this.#usage = createUsageReader({ + historyPath: options.historyPath ?? "", + opened: () => options.historyPath !== undefined && options.eventHistory.status().opened, + tokenLabels: async () => + new Map( + ((await options.tokens?.list()) ?? []).flatMap((token) => + token.label === undefined ? [] : [[token.id, token.label] as const], + ), + ), + workers: () => [ + { + id: options.instanceId, + ...(options.config.gateway.label === undefined + ? {} + : { label: options.config.gateway.label }), + }, + ], + }); + } + + readonly usageGet: Handler<"usage.get"> = (input) => usageAnswer(this.#usage.get(input)); + + readonly eventsReplay: Handler<"events.replay"> = (input) => { + // why: a worker's own events carry no workerId, so `worker` can only name this daemon or no one (ADR 0012 §1) + if (input.workerId !== undefined && input.workerId !== this.options.instanceId) { + return Promise.resolve([]); + } + return input.beforeId === undefined && input.limit === undefined + ? this.options.eventHistory.replay( + input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }, + ) + : eventsPageAnswer( + this.options.eventHistory.replayPage({ + ...(input.beforeId === undefined ? {} : { beforeId: input.beforeId }), + limit: input.limit ?? EVENTS_PAGE_DEFAULT, + }), + ); + }; + + readonly eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { + const subscriptionId = session.manageEventSubscription(true); + if (subscriptionId === undefined) { + throw new DispatchError("INTERNAL", "Transport did not provide a subscription id"); + } + return { subscribed: true, subscriptionId }; + }; + + readonly eventsUnsubscribe: Handler<"events.unsubscribe"> = (_input, session) => { + session.manageEventSubscription(false); + return { subscribed: false }; + }; + + dispose(): void { + void this.#usage.stop(); + } +} diff --git a/src/daemon/dispatcher-exec.ts b/src/daemon/dispatcher-exec.ts new file mode 100644 index 000000000..ccedaaf45 --- /dev/null +++ b/src/daemon/dispatcher-exec.ts @@ -0,0 +1,268 @@ +import type { z } from "zod"; + +import type { OPERATIONS } from "../contract/index.js"; +import { toDriverDevice, UnknownLeaseError, type PassthroughCommand } from "../core/index.js"; +import { + LeaseFilesError, + type LeaseFileView, + type LeaseFiles, + type LeaseFilesGrant, +} from "../lease-files/index.js"; +import { + exitCodeOf, + InputStreamFailedError, + ProcessSpawnError, + type ProcessRunner, + type StreamingProcessHandle, + type StreamingProcessResult, + type TimerHandle, +} from "../ports/index.js"; +import { DispatchError, type DispatchSession } from "./dispatch.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; + +/** + * why: How long a timed-out `device.exec` child gets between SIGTERM and SIGKILL (ADR 0005 §19e). + * Fixed rather than derived from `exec.timeoutMs`: it is a termination-cleanup budget, not a + * fraction of the command's own allowance -- the same reasoning, and the same ten seconds, as + * `NodeProcessRunner`'s `SIGTERM_TO_SIGKILL_GRACE_MS`. + */ +const EXEC_SIGKILL_GRACE_MS = 10_000; + +/** why: The `Promise.race` marker for "the timeout won". A unique object rather than a string, so + * it can never collide with something a `StreamingProcessHandle` could resolve with. */ +const EXEC_EXPIRED = Symbol("exec-expired"); + +export class ExecHandlers { + constructor(private readonly options: DispatcherOptions) {} + + /** + * why: Resolution only: the daemon builds the command and never runs it. Spawning it here would + * attach a user's interactive `adb shell` to the daemon's stdio, and the CLI is the process + * that actually has a terminal (ADR 0001, decision 7). + * + * Both failure modes leave as their own typed errors -- `PassthroughRefusedError` for a verb + * the driver will not proxy, `UnknownPassthroughToolError` for a tool no driver claims -- so + * `errorCode` maps them to `PASSTHROUGH_REFUSED` / `UNKNOWN_PASSTHROUGH_TOOL` and + * `DaemonServer#describeError` still gets the chance to append the driver's refusal summary. + */ + readonly driverPassthrough: Handler<"driver.passthrough"> = (input) => { + if (this.options.passthrough === undefined) { + throw new DispatchError("INTERNAL", "Tool passthrough is unavailable"); + } + return this.options.passthrough.passthrough(input.tool, undefined, input.args); + }; + + /** + * why: ADR 0005 §19a: the same command `driver.passthrough` would have handed back, run here + * instead of there. Everything a passthrough already decides is reused verbatim -- which + * flag scopes the tool to this daemon's root, which verbs the driver refuses -- because it + * is the same driver call: `driverPassthrough` above passes no device, this handler passes + * the lease's device and the lease's platform, and it also spawns the result. + * + * `leaseId` is checked against the registry here so an id that names no lease answers + * `UNKNOWN_LEASE` rather than running a command (`authorize`'s `ownsLease` deliberately lets + * an unknown id through so this handler can say that -- see `roles.ts`). Which device the + * command touches is the driver's answer: it fills the lease's device in where the tool + * expects it and refuses a command that names another (ADR 0047 §1-2); this daemon parses + * none of the arguments. That is the same accident boundary ADR 0001 draws for the local + * wrappers, reached over the wire. + * + * Nothing is buffered: each chunk goes straight to `session.onOutput` as it arrives (§19e), + * so a command that writes a gigabyte costs this process nothing, and a client sees the + * first line while the command is still running. + */ + readonly deviceExec: Handler<"device.exec"> = async (input, session) => { + if (this.options.passthrough === undefined) { + throw new DispatchError("INTERNAL", "Tool passthrough is unavailable"); + } + const processRunner = this.options.processRunner; + if (processRunner === undefined) { + throw new DispatchError("INTERNAL", "Command execution is unavailable"); + } + // why: checked here, not as a refinement on the shared input, which `.omit()` could no longer build `run_on_device` from. + if (input.stdin !== undefined && input.stdinFile !== undefined) { + throw new DispatchError("BAD_REQUEST", "Give stdin or stdinFile, not both"); + } + const lease = this.options.registry.snapshot.leases.find( + (candidate) => candidate.id === input.leaseId, + ); + if (lease === undefined) throw new UnknownLeaseError(input.leaseId); + const record = this.options.registry.snapshot.devices.find( + (candidate) => candidate.id === lease.deviceId, + ); + if (record === undefined) throw new UnknownLeaseError(input.leaseId); + + // why: `hasTerminal: false` tells the driver its caller has no terminal: the command runs + // here, with pipes and no pty (ADR 0005 §19c), so a driver may refuse something it allows + // a local `simlock ` invocation -- a bare `adb shell` would otherwise sit on those + // pipes until `exec.timeoutMs` killed it. `platform` is the leased device's own. + const files = this.options.leaseFiles; + if (files === undefined) throw new DispatchError("INTERNAL", "Lease files are unavailable"); + const command = await this.options.passthrough.passthrough( + input.tool, + toDriverDevice(record), + input.args, + { hasTerminal: false, platform: record.spec.platform }, + ); + return this.#runExec( + { + command, + files, + grant: { grantedAt: lease.grantedAt, id: lease.id, platform: record.spec.platform }, + input, + processRunner, + }, + session, + ); + }; + + async #runExec( + run: { + readonly command: PassthroughCommand; + readonly files: Pick; + readonly grant: LeaseFilesGrant; + readonly input: z.infer<(typeof OPERATIONS)["device.exec"]["input"]>; + readonly processRunner: ProcessRunner; + }, + session: DispatchSession, + ): Promise<{ readonly exitCode: number }> { + const { command, files, grant, input, processRunner } = run; + const execEnv: NodeJS.ProcessEnv = { ...this.options.execEnv, ...command.env }; + for (const key of command.unsetEnv ?? []) { + // why: the base comes from the daemon's environment, so a key it kept could re-aim the command. + delete execEnv[key]; + } + const cwd = await this.#folderOf(files, grant); + const file = + input.stdinFile === undefined + ? undefined + : await this.#openStdinFile(files, grant, input.stdinFile); + try { + const handle = processRunner.spawnStreaming(command.command, command.args, { + cwd, + env: execEnv, + // why: Returned, not fired and forgotten: whatever the transport hands back is what pauses + // the child until the chunk has actually gone somewhere (ADR 0005 §19e). + onChunk: (stream, chunk) => session.onOutput?.(stream, chunk), + ...(input.stdin === undefined ? {} : { input: input.stdin }), + ...(file === undefined ? {} : { inputStream: file.read() }), + }); + // why: Announced before the first chunk can arrive, because "it started" is what a transport + // needs to decide its response shape on -- see `DispatchSession.onStarted`. + session.onStarted?.(); + return { + exitCode: await this.#awaitExec(handle, input.tool, input.args, input.stdinFile), + }; + } catch (error) { + if (error instanceof ProcessSpawnError) { + // why: Node reports no reason for a start that failed, so the folder is asked again: a lease that ended after `folderOf` made the working folder vanish (404), anything else is the spawn's own failure. + await this.#folderOf(files, grant); + } + throw error; + } finally { + await file?.close(); + } + } + + async #folderOf(files: Pick, grant: LeaseFilesGrant): Promise { + try { + return await files.folderOf(grant); + } catch (error) { + if (error instanceof LeaseFilesError) { + throw new DispatchError(error.code, error.message, error.details); + } + throw error; + } + } + + async #openStdinFile( + files: Pick, + grant: LeaseFilesGrant, + name: string, + ): Promise { + try { + return await files.openFile(grant, name); + } catch (error) { + if (error instanceof LeaseFilesError) { + throw new DispatchError(error.code, error.message, error.details); + } + // why: the file was listed and then deleted before the open, so it is gone, not a failure of the daemon. + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + throw new DispatchError("UNKNOWN_FILE", `No file ${name} in this lease`, { name }); + } + throw error; + } + } + + async #awaitExec( + handle: StreamingProcessHandle, + tool: string, + args: readonly string[], + stdinFile: string | undefined, + ): Promise { + const timeoutMs = this.options.config.exec.timeoutMs; + const waited = handle.wait(); + // why: `inputFailed` only ever rejects, so the race below hears it as a value and never as a second way for the command to finish. + const inputFailed = handle.inputFailed?.catch((error: unknown) => error); + let timer: TimerHandle | undefined; + const expired = new Promise((resolve) => { + timer = this.options.clock.setTimer(timeoutMs, () => resolve(EXEC_EXPIRED)); + }); + let killTimer: TimerHandle | undefined; + try { + // why: Raced rather than decided by a flag the timer sets: a command that exits in the same + // turn the timer fires has *finished*, and reporting that as a timeout would blame the + // limit for a command that met it. `Promise.race` settles on whichever actually + // happened first, which is exactly the question. + const outcome = await Promise.race([ + waited, + expired, + ...(inputFailed === undefined ? [] : [inputFailed]), + ]); + if (outcome !== EXEC_EXPIRED && !(outcome instanceof InputStreamFailedError)) { + return exitCodeOf(outcome as StreamingProcessResult); + } + + // why: SIGTERM first, SIGKILL after a grace window: a tool that ignores the first must not be + // able to hold this operation -- and the caller's connection -- open forever. The same + // escalation `NodeProcessRunner#run` makes for its own timeout. + killQuietly(handle, "SIGTERM"); + killTimer = this.options.clock.setTimer(EXEC_SIGKILL_GRACE_MS, () => { + killQuietly(handle, "SIGKILL"); + }); + // why: Once the kill is issued, the timeout is the authoritative fact (ADR 0005 §19e) -- + // `waited` rejecting with `ExecOutputDeliveryStalledError` (a consumer that stopped + // reading is *why* the command outran its timeout in the first place: the pause that + // stops it finishing is the same pause that stalls delivery) is a consequence of the + // timeout, not a competing answer to it. Swallowed here rather than left to reject + // this `await` and skip the throw below. + await waited.catch(() => undefined); + if (outcome instanceof InputStreamFailedError && stdinFile !== undefined) { + throw new DispatchError( + "STDIN_FILE_FAILED", + `Reading ${stdinFile} failed after ${String(outcome.bytesWritten)} bytes; the command was stopped`, + { bytesRead: outcome.bytesWritten, name: stdinFile }, + ); + } + // why: `EXEC_TIMEOUT` rather than the exit code the kill produced: "we stopped it" and "it + // failed" are different facts, and only the first tells a caller to raise the limit. + throw new DispatchError( + "EXEC_TIMEOUT", + `\`${tool} ${args.join(" ")}\` exceeded exec.timeoutMs (${String(timeoutMs)}ms) and was killed`, + ); + } finally { + if (timer !== undefined) this.options.clock.cancel(timer); + if (killTimer !== undefined) this.options.clock.cancel(killTimer); + } + } +} + +/** why: A child that exited between the timer firing and the signal landing is not an error worth + * failing the operation over -- the wait below is about to report how it ended anyway. */ +function killQuietly(handle: StreamingProcessHandle, signal: NodeJS.Signals): void { + try { + handle.kill(signal); + } catch { + // Already gone. + } +} diff --git a/src/daemon/dispatcher-fleet.ts b/src/daemon/dispatcher-fleet.ts new file mode 100644 index 000000000..bb0d3cc33 --- /dev/null +++ b/src/daemon/dispatcher-fleet.ts @@ -0,0 +1,171 @@ +import { z } from "zod"; + +import { + inboxItems, + OPERATIONS, + type OperationName, + waitingRequestSchema, + WORKER_VIEW_CATALOG_EVENTS, + WORKER_VIEW_REFRESH_INTERVAL_MS, + workerViewFields, +} from "../contract/index.js"; +import { DispatchError, type DispatchSession, type ErasedHandler } from "./dispatch.js"; +import { requireTokens } from "./dispatcher-admin.js"; +import type { ComponentHandlers } from "./dispatcher-components.js"; +import type { LeaseHandlers } from "./dispatcher-leases.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; +import type { StatusHandlers } from "./dispatcher-status.js"; +import type { RelayConnection } from "./relay.js"; + +export class FleetHandlers { + /** + * why: `worker.list`'s catalog: the last `catalog.get` answer and when it was read. Kept for + * `WORKER_VIEW_REFRESH_INTERVAL_MS`, the rhythm a gateway re-reads a worker's catalog at, and + * dropped on `WORKER_VIEW_CATALOG_EVENTS`, the events a gateway re-reads it on. Without it + * every `worker.list` runs each driver's catalog read (`simctl list` on iOS), and the console + * polls that route. Holds the promise, so calls that arrive while a read runs + * share it; a read that fails is dropped, so the next call reads again. + */ + #viewCatalog: { readonly readAt: number; readonly catalog: Promise } | undefined; + readonly #unsubscribe: (() => void)[] = []; + + constructor( + private readonly options: DispatcherOptions, + private readonly parts: { + readonly status: StatusHandlers; + readonly components: ComponentHandlers; + readonly leases: LeaseHandlers; + }, + ) { + for (const event of WORKER_VIEW_CATALOG_EVENTS) { + const unsubscribe = options.eventBus?.subscribe(event, () => { + this.#viewCatalog = undefined; + }); + if (unsubscribe !== undefined) this.#unsubscribe.push(unsubscribe); + } + } + + dispose(): void { + for (const unsubscribe of this.#unsubscribe.splice(0)) unsubscribe(); + this.#viewCatalog = undefined; + } + + readonly workerList: Handler<"worker.list"> = async (_input, session) => { + // why: taken as status resolves, not after the slower catalog read, so the load is not made to look fresher. + let receivedAt = this.options.clock.now(); + const [status, devices, catalog, baselines] = await Promise.all([ + Promise.resolve(this.parts.status.statusGet({}, session)).then((answer) => { + receivedAt = this.options.clock.now(); + return answer; + }), + this.parts.status.listGet({ kind: "devices" }, session), + this.#keptViewCatalog(session), + this.options.baselines === undefined + ? undefined + : this.parts.components.baselineList({}, session), + ]); + const label = this.options.config.gateway.label; + return { + workers: [ + { + ...workerViewFields({ + ...(baselines === undefined + ? {} + : { baselines: OPERATIONS["baseline.list"].output.parse(baselines) }), + catalog: OPERATIONS["catalog.get"].output.parse(catalog), + config: OPERATIONS["config.get"].output.parse(this.parts.status.configGet({}, session)), + devices: OPERATIONS["list.get"].output.parse(devices), + receivedAt, + status: OPERATIONS["status.get"].output.parse(status), + }), + connection: "connected", + drained: false, + id: this.options.instanceId, + lastSeenAt: this.options.clock.now(), + version: this.options.version, + ...(label === undefined ? {} : { label }), + }, + ], + }; + }; + + #keptViewCatalog(session: DispatchSession): Promise { + const now = this.options.clock.now(); + const kept = this.#viewCatalog; + const age = kept === undefined ? undefined : now - kept.readAt; + if ( + kept !== undefined && + age !== undefined && + age >= 0 && + age < WORKER_VIEW_REFRESH_INTERVAL_MS + ) { + return kept.catalog; + } + const catalog = Promise.resolve(this.parts.status.catalogGet({}, session)); + this.#viewCatalog = { catalog, readAt: now }; + catalog.catch(() => { + this.#viewCatalog = undefined; + }); + return catalog; + } + + // why: a worker answers as a fleet of one (ADR 0012): the rules read its own view, leases and queue. + readonly inboxGet: Handler<"inbox.get"> = async (_input, session) => { + const [listed, tokens] = await Promise.all([ + this.workerList({}, session), + Promise.resolve() + .then(() => requireTokens(this.options).list()) + .catch(() => []), + ]); + return { + items: [ + ...inboxItems(OPERATIONS["worker.list"].output.parse(listed).workers, { + leases: OPERATIONS["lease.list"].output.parse(this.parts.leases.leaseList({}, session)) + .leases, + now: this.options.clock.now(), + requests: z.array(waitingRequestSchema).parse(this.options.queue.waitingRequests()), + tokens: OPERATIONS["token.list"].output.parse({ tokens }).tokens, + }), + ], + }; + }; + + #relays(session: DispatchSession): RelayConnection { + if (session.isGatewayUplink !== true) { + throw new DispatchError("FORBIDDEN", "Only the gateway's own uplink session may relay"); + } + if (session.relays === undefined) { + throw new DispatchError("INTERNAL", "The uplink session carries no relays"); + } + return session.relays; + } + + readonly relayRequest: Handler<"relay.request"> = (input, session) => { + const relays = this.#relays(session); + if (session.relayChannel === undefined) { + throw new DispatchError( + "BAD_REQUEST", + `The relay's channel ${input.channel} was not accepted`, + ); + } + return relays.request( + input, + { head: (status, headers) => session.onRelayHead?.(status, headers) ?? Promise.resolve() }, + session.relayChannel, + ); + }; + + readonly relayCancel: Handler<"relay.cancel"> = (input, session) => { + this.#relays(session).cancel(input.relayId); + return {}; + }; +} + +export function unsupportedOnWorker( + operation: OperationName, + message = `${operation} acts on a gateway's workers, and this daemon is not a gateway`, +): ErasedHandler { + return () => { + throw new DispatchError("UNSUPPORTED_IN_WORKER_MODE", message, { operation }); + }; +} diff --git a/src/daemon/dispatcher-leases.ts b/src/daemon/dispatcher-leases.ts new file mode 100644 index 000000000..8c98fc3f1 --- /dev/null +++ b/src/daemon/dispatcher-leases.ts @@ -0,0 +1,141 @@ +import { requestedDevice } from "../contract/index.js"; +import { + effectiveAllowDownload, + RuntimeMissingError, + UnknownLeaseError, + type DeviceRequest, +} from "../core/index.js"; +import { capacityReasonOf, NoCapacityError } from "../leasing/index.js"; +import { DispatchError, principalAsRequester } from "./dispatch.js"; +import { requireCapabilities } from "./dispatcher-capabilities.js"; +import { leasesFor } from "./dispatcher-device-views.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; +import { leaseRequestRefusal } from "./lease-request-refusal.js"; +import { UnadmittedRequests } from "./unadmitted-requests.js"; + +export class LeaseHandlers { + readonly #unadmitted = new UnadmittedRequests(); + + constructor(private readonly options: DispatcherOptions) {} + + readonly leaseRequest: Handler<"lease.request"> = async (input, session) => { + const request: DeviceRequest = requestedDevice(input); + this.#requireTtlWithinCap(input.ttlMs); + const requesterId = input.requesterId ?? principalAsRequester(session.principal); + // fallow-ignore-next-line complexity -- lease payload assembly and the download-policy rewrite are one transaction, moved verbatim from DaemonServer's former #requestLease. + return this.#unadmitted.track(requesterId, async (admittedHook) => { + const requestedAllowDownload = input.allowDownload ?? false; + const downloadsPolicy = this.options.config.downloads.policy; + // why: `owner` is read only from the gateway's uplink, never from role alone: HTTP's `operator` token + // maps to `admin`, so a role check would let any operator name someone else's owner. Refused, not ignored. + if (input.owner !== undefined && session.isGatewayUplink !== true) { + throw new DispatchError( + "FORBIDDEN", + "Only the gateway's own uplink session may set lease.request's `owner` field", + ); + } + // why: Uplink only, refused before anything is stored: a probe is never queued on a worker, so it must say `noWait`. + if (input.fleetRequestId !== undefined) { + if (session.isGatewayUplink !== true) { + throw new DispatchError( + "FORBIDDEN", + "Only the gateway's own uplink session may set lease.request's `fleetRequestId` field", + ); + } + if (input.noWait !== true) { + throw new DispatchError( + "BAD_REQUEST", + "lease.request's `fleetRequestId` is sent only with `noWait: true`", + ); + } + } + await requireCapabilities(this.options, input.requires); + const ownerId = input.owner ?? session.principal; + try { + const grant = await this.options.leases.request(request, { + allowDownload: effectiveAllowDownload(downloadsPolicy, requestedAllowDownload), + noWait: input.noWait ?? false, + ownerId, + requesterId, + ...(session.onProgress === undefined ? {} : { onProgress: session.onProgress }), + onAdmitted: admittedHook(session.onRequestAdmitted), + ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }), + ...(input.leaseId === undefined ? {} : { leaseId: input.leaseId }), + ...(input.fleetRequestId === undefined ? {} : { fleetRequestId: input.fleetRequestId }), + ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), + ...(input.ttlMs === undefined ? {} : { ttlMs: input.ttlMs }), + }); + const { device } = grant; + return { + ...grant, + device: device.physical ? { ...device, class: device.spec.class } : device, + }; + } catch (error: unknown) { + const refusal = leaseRequestRefusal(error); + if (refusal !== undefined) throw refusal; + if (error instanceof NoCapacityError) { + const capacityReason = capacityReasonOf(error.reason); + if (capacityReason !== undefined) { + throw new DispatchError("NO_CAPACITY", error.message, { capacityReason }); + } + } + // why: The lease path sees only the clamped-to-false permission, so this is the one place that can + // tell the caller that config, not missing consent, blocked the download. + if ( + downloadsPolicy === "never" && + error instanceof RuntimeMissingError && + error.downloadable + ) { + error.message = `${error.message} (downloads are disabled by configuration: downloads.policy is "never")`; + } + throw error; + } + }); + }; + + readonly leaseCancel: Handler<"lease.cancel"> = async (input, session) => { + const requesterId = input.requesterId ?? session.principal; + await this.#unadmitted.settled(requesterId); + const result = await this.options.queue.cancelPending(requesterId, input.platform); + return { result }; + }; + + readonly leaseRenew: Handler<"lease.renew"> = async (input) => { + this.#requireTtlWithinCap(input.ttlMs); + return this.options.leases.renew(input.leaseId, input.ttlMs); + }; + + #requireTtlWithinCap(ttlMs: number | undefined): void { + const maxTtlMs = this.options.config.lease.maxTtlMs; + if (ttlMs !== undefined && ttlMs > maxTtlMs) { + throw new DispatchError( + "BAD_REQUEST", + `ttlMs ${String(ttlMs)} exceeds lease.maxTtlMs (${String(maxTtlMs)})`, + ); + } + } + + readonly leaseRelease: Handler<"lease.release"> = async (input) => { + await this.options.leases.release(input.leaseId, "explicit"); + return { leaseId: input.leaseId }; + }; + + readonly leaseReleaseAll: Handler<"lease.release-all"> = async () => { + const leaseIds = await this.options.leases.releaseAll("killed"); + return { leaseIds: [...leaseIds] }; + }; + + readonly leaseGet: Handler<"lease.get"> = (input, session) => { + const lease = this.options.registry.snapshot.leases.find( + (candidate) => candidate.id === input.leaseId, + ); + if (lease === undefined || (session.role !== "admin" && lease.ownerId !== session.principal)) { + throw new UnknownLeaseError(input.leaseId); + } + return { lease }; + }; + + readonly leaseList: Handler<"lease.list"> = (_input, session) => ({ + leases: leasesFor(this.options, session), + }); +} diff --git a/src/daemon/dispatcher-options.ts b/src/daemon/dispatcher-options.ts new file mode 100644 index 000000000..1c9c8b64d --- /dev/null +++ b/src/daemon/dispatcher-options.ts @@ -0,0 +1,121 @@ +import type { z } from "zod"; + +import type { TokenOperations } from "../auth/index.js"; +import type { EventBus } from "../bus/index.js"; +import type { EventHistoryHandle } from "../history/index.js"; +import type { + BaselineService, + CapacityReader, + CatalogReader, + CleanupReaper, + ComponentInstaller, + Config, + Doctor, + Driver, + HostFacts, + LatestPresence, + Nuke, + PassthroughResolver, + PhysicalEnrollment, + Registry, + StallInput, + WarmKindForgetter, + WarmPoolReader, +} from "../core/index.js"; +import type { DeviceModeReader, LeaseCommands, QueueControl } from "../leasing/index.js"; +import type { LeaseFiles } from "../lease-files/index.js"; +import type { ModuleInputs } from "../modules/index.js"; +import type { Clock, Logger, ProcessRunner } from "../ports/index.js"; +import type { + OPERATIONS, + OperationName, + StatusLoad, + StatusMemoryPressure, +} from "../contract/index.js"; +import type { DispatchSession } from "./dispatch.js"; +import type { UpdateCoordinator } from "./update-coordinator.js"; + +export interface CapabilitySource { + readonly name: string; + readonly read: () => Promise<{ readonly state: string; readonly reason?: string }>; +} + +export interface DispatcherOptions { + readonly historyPath?: string; + readonly capabilitySources?: readonly CapabilitySource[]; + readonly capacity: CapacityReader; + readonly warmPool: WarmPoolReader; + readonly catalog: CatalogReader; + readonly clock: Clock; + readonly components: Pick; + readonly config: Config; + readonly doctor?: Doctor; + readonly eventHistory: Pick; + readonly deviceModes: DeviceModeReader; + readonly leases: LeaseCommands; + readonly logger?: Logger; + /** why: Classifies a thrown error for the `operation` log line (`classifyError` in production). + * Injected because `error-code.ts` imports this module. Unset or `undefined`: `INTERNAL`. */ + readonly errorCode?: (error: unknown) => string | undefined; + readonly nuke?: Nuke; + /** why: `warm.forget`'s one way into the warm pool's learned record; optional so tests that never forget need no pool. */ + readonly warmKinds?: WarmKindForgetter; + /** why: the baselines' one way in; optional so tests that never save need none. */ + readonly baselines?: Pick; + readonly enrollment?: Pick; + readonly passthrough?: PassthroughResolver; + /** why: the lease's device input, which `device.rotate` writes to; optional so tests that never rotate need none. */ + readonly inputs?: ModuleInputs; + readonly processRunner?: ProcessRunner; + readonly leaseFiles?: Pick; + /** + * why: The environment a `device.exec` child starts from, before the driver's own scoping keys are + * layered on top. Injected rather than read from `process.env` here (architecture rule 9), + * and layered rather than replaced because `ProcessRunner` replaces a child's environment + * wholesale: a child given only `--set`/`-P` scoping would lose `PATH` and never find the + * tool it was pointed at. + */ + readonly execEnv?: NodeJS.ProcessEnv; + readonly queue: QueueControl; + readonly presence?: (driverDeviceId: string) => LatestPresence | undefined; + readonly reaper: CleanupReaper; + readonly registry: Registry; + readonly stalls?: { + readonly drivers: readonly Pick[]; + readonly claims: NonNullable; + }; + /** + * why: ADR 0003 §11: "token create|list|revoke become daemon operations. The daemon is the only + * owner of the tokens." Optional so tests that don't exercise `token.*` (the overwhelming + * majority) don't need to fabricate one -- `tokenCreate`/`tokenList`/`tokenRevoke` throw a + * clear `DispatchError` when it is missing rather than crashing on `undefined.create(...)`. + */ + readonly tokens?: TokenOperations; + readonly health: () => "starting" | "running" | "failed"; + readonly hostFacts: () => HostFacts; + readonly load?: () => StatusLoad | undefined; + readonly memoryPressure?: () => StatusMemoryPressure | undefined; + readonly awaitReady: () => Promise; + readonly instanceId: string; + readonly version: string; + readonly eventBus?: Pick; + readonly update?: UpdateCoordinator; +} + +/** + * why: A handler's declared *input* type is tied to its operation's contract schema (real + * type-checking value: a handler that reads `input.wrongField` fails to compile). Its return + * type is deliberately left as `unknown` rather than `z.infer<...Output>`: core's domain types + * (`LeaseRecord`, `DoctorReport`, ...) are hand-mirrored by the contract's schemas, not reused + * (see `schemas.ts`'s module comment -- keeping private types off the public surface is the + * whole point), so they are not always structurally identical to what `z.infer` produces + * (`readonly` vs. mutable arrays, in particular). `dispatch()`'s `#parseOutput` is what actually + * enforces the contract at the boundary, exactly as it did before this PR when handlers lived + * in `DaemonServer` and returned `unknown` too -- this is not a new gap, just a preserved one. + */ +export type Handler = ( + input: z.infer<(typeof OPERATIONS)[Op]["input"]>, + session: DispatchSession, +) => Promise | unknown; + +export type WorkerOperationName = Exclude; diff --git a/src/daemon/dispatcher-status.ts b/src/daemon/dispatcher-status.ts new file mode 100644 index 000000000..bfafcf2ba --- /dev/null +++ b/src/daemon/dispatcher-status.ts @@ -0,0 +1,88 @@ +import { buildCapacityFigures, redactConfig } from "../core/index.js"; +import { + consoleUrlField, + fitPlatformCatalog, + withoutQuarantineMessages, +} from "../contract/index.js"; +import { readCapabilities } from "./dispatcher-capabilities.js"; +import { decorateDevice, leasesFor, waitingFor } from "./dispatcher-device-views.js"; +import type { DispatcherOptions, Handler } from "./dispatcher-options.js"; +import type { UpdateProgress } from "./update-coordinator.js"; + +export class StatusHandlers { + constructor(private readonly options: DispatcherOptions) {} + + readonly catalogGet: Handler<"catalog.get"> = async (input) => ({ + platforms: (await this.options.catalog.listCatalog(input.platform)) + .map(fitPlatformCatalog) + .map((entry) => + // why: ADR 0066 §5: a gateway routes an `except` lease by the iOS worker's default mode. + entry.platform === "ios" + ? { ...entry, defaultMode: this.options.config.ios.defaultMode } + : entry, + ), + }); + + readonly statusGet: Handler<"status.get"> = async (_input, session) => { + const daemon = { + health: this.options.health(), + mode: this.options.config.mode, + ...consoleUrlField(this.options.config.http), + version: this.options.version, + ...this.#updateFields(), + }; + const host = this.options.hostFacts(); + const load = this.options.load?.(); + const loadField = load === undefined ? {} : { load }; + const memoryPressure = this.options.memoryPressure?.(); + // why: while starting, the registry has not been checked against the machine, so what it holds + // is a claim nobody has looked at; the answer is the daemon, its host and the load reading. + const history = this.options.eventHistory.status(); + if (daemon.health === "starting") return { daemon, history, host, ...loadField }; + const capabilities = await readCapabilities(this.options); + const snapshot = this.options.registry.snapshot; + const answer = { + ...(capabilities === undefined ? {} : { capabilities }), + capacity: { + ...buildCapacityFigures(snapshot.virtualDevices, this.options.capacity), + ...(memoryPressure === undefined ? {} : { memoryPressure }), + }, + devices: snapshot.devices.map((device) => decorateDevice(this.options, device)), + daemon, + history, + host, + ...loadField, + installs: [...this.options.components.inProgress()], + leases: leasesFor(this.options, session), + physicalQueueDepth: this.options.queue.physicalQueueDepth, + queueDepth: this.options.queue.queueDepth, + waiting: waitingFor(this.options, session), + warmPool: this.options.warmPool.figures(), + }; + return session.role === "agent" ? withoutQuarantineMessages(answer) : answer; + }; + + #updateFields(): UpdateProgress { + return ( + this.options.update?.state() ?? { update: null, lastUpdateFailure: null, latestVersion: null } + ); + } + + readonly listGet: Handler<"list.get"> = (input) => { + const snapshot = this.options.registry.snapshot; + switch (input.kind) { + case "leases": + return [...snapshot.leases]; + case "rules": + return this.options.reaper.rules; + case "requests": + return [...this.options.queue.waitingRequests()]; + case "devices": + case undefined: + return snapshot.devices.map((device) => decorateDevice(this.options, device)); + } + }; + + readonly configGet: Handler<"config.get"> = (input) => + input.showSecrets === true ? this.options.config : redactConfig(this.options.config); +} diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 141ef384a..6b267c44c 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -1,229 +1,22 @@ -import { z } from "zod"; - -import type { EventBus } from "../bus/index.js"; -import type { EventHistoryHandle } from "../history/index.js"; -import { - buildCapacityFigures, - type CleanupReaper, - ComponentInUseError, - type ComponentInstaller, - type ComponentInstallerProgress, - type Config, - type DeviceRecord, - type DeviceRequest, - DeviceBusyError, - DeviceNotEnrollableError, - type Doctor, - type Driver, - type HostFacts, - isStalledTransition, - type LatestPresence, - type Nuke, - type PhysicalClass, - type PhysicalEnrollment, - type Registry, - type StallInput, - UnknownEnrolledDeviceError, - effectiveAllowDownload, - RuntimeMissingError, - transitionEnteredAt, - toDriverDevice, - UnknownLeaseError, - type CapacityReader, - BaselineInUseError, - BaselineSaveFailedError, - UnknownBaselineError, - type BaselineService, - type WarmKindForgetter, - type WarmPoolReader, - type CatalogReader, - type PassthroughCommand, - type PassthroughResolver, - isUnavailableInput, - createUsageReader, - type UsageReader, - redactConfig, -} from "../core/index.js"; -import type { ModuleInputs } from "../modules/index.js"; -import { leaseRequestRefusal } from "./lease-request-refusal.js"; -import { rotationRefusal } from "./rotation-refusal.js"; -import { - type DeviceModeReader, - type LeaseCommands, - capacityReasonOf, - NoCapacityError, - type QueueControl, -} from "../leasing/index.js"; -import type { - Clock, - Logger, - ProcessRunner, - StreamingProcessHandle, - StreamingProcessResult, - TimerHandle, -} from "../ports/index.js"; -import { - LeaseFilesError, - type LeaseFileView, - type LeaseFiles, - type LeaseFilesGrant, -} from "../lease-files/index.js"; -import { - exitCodeOf, - InputStreamFailedError, - NoopLogger, - ProcessSpawnError, -} from "../ports/index.js"; -import { - capabilityReason, - type capabilitySchema, - consoleUrlField, - EVENTS_PAGE_DEFAULT, - fitPlatformCatalog, - inboxItems, - OPERATIONS, - requestedDevice, - type ComponentProgress, - type OperationName, - type StatusLoad, - type StatusMemoryPressure, - waitingRequestSchema, - WORKER_VIEW_CATALOG_EVENTS, - WORKER_VIEW_REFRESH_INTERVAL_MS, - workerViewFields, - withoutQuarantineMessages, -} from "../contract/index.js"; -import type { TokenOperations } from "../auth/index.js"; -import { - DispatchError, - identityGet, - principalAsRequester, - runDispatch, - eventsPageAnswer, - usageAnswer, - type DispatchSession, - type ErasedHandler, -} from "./dispatch.js"; -import type { RelayConnection } from "./relay.js"; -import { UnadmittedRequests } from "./unadmitted-requests.js"; -import type { UpdateCoordinator, UpdateProgress } from "./update-coordinator.js"; +import type { z } from "zod"; + +import type { OPERATIONS, OperationName } from "../contract/index.js"; +import type { Logger } from "../ports/index.js"; +import { NoopLogger } from "../ports/index.js"; +import { identityGet, runDispatch, type DispatchSession, type ErasedHandler } from "./dispatch.js"; +import { AdminHandlers } from "./dispatcher-admin.js"; +import { ComponentHandlers } from "./dispatcher-components.js"; +import { DeviceHandlers } from "./dispatcher-devices.js"; +import { EventHandlers } from "./dispatcher-events.js"; +import { ExecHandlers } from "./dispatcher-exec.js"; +import { FleetHandlers, unsupportedOnWorker } from "./dispatcher-fleet.js"; +import { LeaseHandlers } from "./dispatcher-leases.js"; +import type { DispatcherOptions, WorkerOperationName } from "./dispatcher-options.js"; +import { StatusHandlers } from "./dispatcher-status.js"; export { DispatchError, type ContractDispatcher, type DispatchSession } from "./dispatch.js"; - -/** - * why: How long a timed-out `device.exec` child gets between SIGTERM and SIGKILL (ADR 0005 §19e). - * Fixed rather than derived from `exec.timeoutMs`: it is a termination-cleanup budget, not a - * fraction of the command's own allowance -- the same reasoning, and the same ten seconds, as - * `NodeProcessRunner`'s `SIGTERM_TO_SIGKILL_GRACE_MS`. - */ -const EXEC_SIGKILL_GRACE_MS = 10_000; - -/** why: The `Promise.race` marker for "the timeout won". A unique object rather than a string, so - * it can never collide with something a `StreamingProcessHandle` could resolve with. */ -const EXEC_EXPIRED = Symbol("exec-expired"); - -export class DoctorUnavailableError extends Error { - constructor() { - super("Doctor is unavailable"); - this.name = "DoctorUnavailableError"; - } -} - -export class NukeUnavailableError extends Error { - constructor() { - super("Nuke is unavailable"); - this.name = "NukeUnavailableError"; - } -} - -const CAPABILITY_READ_BUDGET_MS = 1_000; - -type CapabilityEntry = z.infer; - -export interface CapabilitySource { - readonly name: string; - readonly read: () => Promise<{ readonly state: string; readonly reason?: string }>; -} - -export interface DispatcherOptions { - readonly historyPath?: string; - readonly capabilitySources?: readonly CapabilitySource[]; - readonly capacity: CapacityReader; - readonly warmPool: WarmPoolReader; - readonly catalog: CatalogReader; - readonly clock: Clock; - readonly components: Pick; - readonly config: Config; - readonly doctor?: Doctor; - readonly eventHistory: Pick; - readonly deviceModes: DeviceModeReader; - readonly leases: LeaseCommands; - readonly logger?: Logger; - /** why: Classifies a thrown error for the `operation` log line (`classifyError` in production). - * Injected because `error-code.ts` imports this module. Unset or `undefined`: `INTERNAL`. */ - readonly errorCode?: (error: unknown) => string | undefined; - readonly nuke?: Nuke; - /** why: `warm.forget`'s one way into the warm pool's learned record; optional so tests that never forget need no pool. */ - readonly warmKinds?: WarmKindForgetter; - /** why: the baselines' one way in; optional so tests that never save need none. */ - readonly baselines?: Pick; - readonly enrollment?: Pick; - readonly passthrough?: PassthroughResolver; - /** why: the lease's device input, which `device.rotate` writes to; optional so tests that never rotate need none. */ - readonly inputs?: ModuleInputs; - readonly processRunner?: ProcessRunner; - readonly leaseFiles?: Pick; - /** - * why: The environment a `device.exec` child starts from, before the driver's own scoping keys are - * layered on top. Injected rather than read from `process.env` here (architecture rule 9), - * and layered rather than replaced because `ProcessRunner` replaces a child's environment - * wholesale: a child given only `--set`/`-P` scoping would lose `PATH` and never find the - * tool it was pointed at. - */ - readonly execEnv?: NodeJS.ProcessEnv; - readonly queue: QueueControl; - readonly presence?: (driverDeviceId: string) => LatestPresence | undefined; - readonly reaper: CleanupReaper; - readonly registry: Registry; - readonly stalls?: { - readonly drivers: readonly Pick[]; - readonly claims: NonNullable; - }; - /** - * why: ADR 0003 §11: "token create|list|revoke become daemon operations. The daemon is the only - * owner of the tokens." Optional so tests that don't exercise `token.*` (the overwhelming - * majority) don't need to fabricate one -- `#tokenCreate`/`#tokenList`/`#tokenRevoke` throw a - * clear `DispatchError` when it is missing rather than crashing on `undefined.create(...)`. - */ - readonly tokens?: TokenOperations; - readonly health: () => "starting" | "running" | "failed"; - readonly hostFacts: () => HostFacts; - readonly load?: () => StatusLoad | undefined; - readonly memoryPressure?: () => StatusMemoryPressure | undefined; - readonly awaitReady: () => Promise; - readonly instanceId: string; - readonly version: string; - readonly eventBus?: Pick; - readonly update?: UpdateCoordinator; -} - -/** - * why: A handler's declared *input* type is tied to its operation's contract schema (real - * type-checking value: a handler that reads `input.wrongField` fails to compile). Its return - * type is deliberately left as `unknown` rather than `z.infer<...Output>`: core's domain types - * (`LeaseRecord`, `DoctorReport`, ...) are hand-mirrored by the contract's schemas, not reused - * (see `schemas.ts`'s module comment -- keeping private types off the public surface is the - * whole point), so they are not always structurally identical to what `z.infer` produces - * (`readonly` vs. mutable arrays, in particular). `dispatch()`'s `#parseOutput` is what actually - * enforces the contract at the boundary, exactly as it did before this PR when handlers lived - * in `DaemonServer` and returned `unknown` too -- this is not a new gap, just a preserved one. - */ -type Handler = ( - input: z.infer<(typeof OPERATIONS)[Op]["input"]>, - session: DispatchSession, -) => Promise | unknown; - -type WorkerOperationName = Exclude; +export type { CapabilitySource, DispatcherOptions } from "./dispatcher-options.js"; +export { DoctorUnavailableError, NukeUnavailableError } from "./dispatcher-admin.js"; export class Dispatcher { readonly #logger: Logger; @@ -240,87 +33,75 @@ export class Dispatcher { * what a worker answers. */ readonly #handlers: Record; - /** - * why: `worker.list`'s catalog: the last `catalog.get` answer and when it was read. Kept for - * `WORKER_VIEW_REFRESH_INTERVAL_MS`, the rhythm a gateway re-reads a worker's catalog at, and - * dropped on `WORKER_VIEW_CATALOG_EVENTS`, the events a gateway re-reads it on. Without it - * every `worker.list` runs each driver's catalog read (`simctl list` on iOS), and the console - * polls that route. Holds the promise, so calls that arrive while a read runs - * share it; a read that fails is dropped, so the next call reads again. - */ - #viewCatalog: { readonly readAt: number; readonly catalog: Promise } | undefined; - readonly #unsubscribe: (() => void)[] = []; - readonly #unadmitted = new UnadmittedRequests(); - - readonly #usage: UsageReader; + readonly #events: EventHandlers; + readonly #status: StatusHandlers; + readonly #leases: LeaseHandlers; + readonly #components: ComponentHandlers; + readonly #admin: AdminHandlers; + readonly #fleet: FleetHandlers; + readonly #exec: ExecHandlers; + readonly #devices: DeviceHandlers; constructor(private readonly options: DispatcherOptions) { this.#logger = options.logger ?? new NoopLogger(); - this.#usage = createUsageReader({ - historyPath: options.historyPath ?? "", - opened: () => options.historyPath !== undefined && options.eventHistory.status().opened, - tokenLabels: async () => - new Map( - ((await options.tokens?.list()) ?? []).flatMap((token) => - token.label === undefined ? [] : [[token.id, token.label] as const], - ), - ), - workers: () => [ - { - id: options.instanceId, - ...(options.config.gateway.label === undefined - ? {} - : { label: options.config.gateway.label }), - }, - ], + this.#events = new EventHandlers(options); + this.#status = new StatusHandlers(options); + this.#leases = new LeaseHandlers(options); + this.#components = new ComponentHandlers(options); + this.#admin = new AdminHandlers(options); + this.#fleet = new FleetHandlers(options, { + components: this.#components, + leases: this.#leases, + status: this.#status, }); - for (const event of WORKER_VIEW_CATALOG_EVENTS) { - const unsubscribe = options.eventBus?.subscribe(event, () => { - this.#viewCatalog = undefined; - }); - if (unsubscribe !== undefined) this.#unsubscribe.push(unsubscribe); - } this.#dispatchLogger = this.#logger.child("dispatch"); + this.#exec = new ExecHandlers(options); + this.#devices = new DeviceHandlers(options); + // why: Arrow-function class fields (not methods): each closes over `this` so it can be stored in + // `#handlers` and invoked as a plain function, the same way any other transport-independent + // callback would be. None of these run a role or ownership check -- `dispatch()` above + // already did, per the ADR's ordering. + this.#handlers = { - "catalog.get": this.#catalogGet, - "status.get": this.#statusGet, - "lease.request": this.#leaseRequest, - "lease.cancel": this.#leaseCancel, - "lease.renew": this.#leaseRenew, - "lease.release": this.#leaseRelease, - "lease.list": this.#leaseList, - "lease.get": this.#leaseGet, - "driver.passthrough": this.#driverPassthrough, - "device.exec": this.#deviceExec, - "device.rotate": this.#deviceRotate, - "relay.request": this.#relayRequest, - "relay.cancel": this.#relayCancel, - "doctor.run": this.#doctorRun, - "lease.release-all": this.#leaseReleaseAll, - "list.get": this.#listGet, - "cleanup.run": this.#cleanupRun, - "nuke.run": this.#nukeRun, - "warm.forget": this.#warmForget, - "baseline.save": this.#baselineSave, - "baseline.list": this.#baselineList, - "baseline.remove": this.#baselineRemove, - "config.get": this.#configGet, - "events.replay": this.#eventsReplay, - "usage.get": this.#usageGet, - "events.subscribe": this.#eventsSubscribe, - "events.unsubscribe": this.#eventsUnsubscribe, + "catalog.get": this.#status.catalogGet, + "status.get": this.#status.statusGet, + "lease.request": this.#leases.leaseRequest, + "lease.cancel": this.#leases.leaseCancel, + "lease.renew": this.#leases.leaseRenew, + "lease.release": this.#leases.leaseRelease, + "lease.list": this.#leases.leaseList, + "lease.get": this.#leases.leaseGet, + "driver.passthrough": this.#exec.driverPassthrough, + "device.exec": this.#exec.deviceExec, + "device.rotate": this.#devices.deviceRotate, + "relay.request": this.#fleet.relayRequest, + "relay.cancel": this.#fleet.relayCancel, + "doctor.run": this.#admin.doctorRun, + "lease.release-all": this.#leases.leaseReleaseAll, + "list.get": this.#status.listGet, + "cleanup.run": this.#admin.cleanupRun, + "nuke.run": this.#admin.nukeRun, + "warm.forget": this.#components.warmForget, + "baseline.save": this.#components.baselineSave, + "baseline.list": this.#components.baselineList, + "baseline.remove": this.#components.baselineRemove, + "config.get": this.#status.configGet, + "events.replay": this.#events.eventsReplay, + "usage.get": this.#events.usageGet, + "events.subscribe": this.#events.eventsSubscribe, + "events.unsubscribe": this.#events.eventsUnsubscribe, "identity.get": identityGet, - "token.create": this.#tokenCreate, - "inbox.get": this.#inboxGet, - "token.list": this.#tokenList, - "token.revoke": this.#tokenRevoke, - "component.install": this.#componentInstall, - "component.list": this.#componentList, - "component.remove": this.#componentRemove, - "device.add": this.#deviceAdd, - "device.remove": this.#deviceRemove, - "daemon.update": this.#daemonUpdate, - "worker.list": this.#workerList, + "token.create": this.#admin.tokenCreate, + "inbox.get": this.#fleet.inboxGet, + "token.list": this.#admin.tokenList, + "token.revoke": this.#admin.tokenRevoke, + "component.install": this.#components.componentInstall, + "component.list": this.#components.componentList, + "component.remove": this.#components.componentRemove, + "device.add": this.#devices.deviceAdd, + "device.remove": this.#devices.deviceRemove, + "daemon.update": this.#admin.daemonUpdate, + "worker.list": this.#fleet.workerList, "worker.drain": unsupportedOnWorker("worker.drain"), "worker.undrain": unsupportedOnWorker("worker.undrain"), "worker.remove": unsupportedOnWorker("worker.remove"), @@ -337,9 +118,8 @@ export class Dispatcher { } dispose(): void { - for (const unsubscribe of this.#unsubscribe.splice(0)) unsubscribe(); - this.#viewCatalog = undefined; - void this.#usage.stop(); + this.#fleet.dispose(); + this.#events.dispose(); } dispatch( @@ -371,1012 +151,4 @@ export class Dispatcher { }, }); } - - // why: Arrow-function class fields (not methods): each closes over `this` so it can be stored in - // `#handlers` and invoked as a plain function, the same way any other transport-independent - // callback would be. None of these run a role or ownership check -- `dispatch()` above - // already did, per the ADR's ordering. - - #catalogGet: Handler<"catalog.get"> = async (input) => ({ - platforms: (await this.options.catalog.listCatalog(input.platform)) - .map(fitPlatformCatalog) - .map((entry) => - // why: ADR 0066 §5: a gateway routes an `except` lease by the iOS worker's default mode. - entry.platform === "ios" - ? { ...entry, defaultMode: this.options.config.ios.defaultMode } - : entry, - ), - }); - - #statusGet: Handler<"status.get"> = async (_input, session) => { - const daemon = { - health: this.options.health(), - mode: this.options.config.mode, - ...consoleUrlField(this.options.config.http), - version: this.options.version, - ...this.#updateFields(), - }; - const host = this.options.hostFacts(); - const load = this.options.load?.(); - const loadField = load === undefined ? {} : { load }; - const memoryPressure = this.options.memoryPressure?.(); - // why: while starting, the registry has not been checked against the machine, so what it holds - // is a claim nobody has looked at; the answer is the daemon, its host and the load reading. - const history = this.options.eventHistory.status(); - if (daemon.health === "starting") return { daemon, history, host, ...loadField }; - const capabilities = await this.#readCapabilities(); - const snapshot = this.options.registry.snapshot; - const answer = { - ...(capabilities === undefined ? {} : { capabilities }), - capacity: { - ...buildCapacityFigures(snapshot.virtualDevices, this.options.capacity), - ...(memoryPressure === undefined ? {} : { memoryPressure }), - }, - devices: snapshot.devices.map((device) => this.#decorateDevice(device)), - daemon, - history, - host, - ...loadField, - installs: [...this.options.components.inProgress()], - leases: this.#leasesFor(session), - physicalQueueDepth: this.options.queue.physicalQueueDepth, - queueDepth: this.options.queue.queueDepth, - waiting: this.#waitingFor(session), - warmPool: this.options.warmPool.figures(), - }; - return session.role === "agent" ? withoutQuarantineMessages(answer) : answer; - }; - - #updateFields(): UpdateProgress { - return ( - this.options.update?.state() ?? { update: null, lastUpdateFailure: null, latestVersion: null } - ); - } - - #daemonUpdate: Handler<"daemon.update"> = async (input) => { - const update = this.options.update; - if (update === undefined) { - throw new DispatchError("UNKNOWN_REQUEST", "This daemon does not update itself"); - } - return { - ...(await update.request(input)), - version: this.options.version, - health: this.options.health(), - }; - }; - - // why: a module can never make status.get fail or hang, which would cut the worker off from its gateway. - async #readCapabilities(): Promise { - const sources = this.options.capabilitySources; - if (sources === undefined || sources.length === 0) return undefined; - return Promise.all(sources.map((source) => this.#readCapability(source))); - } - - async #readCapability(source: CapabilitySource): Promise { - const unreadable: CapabilityEntry = { - name: source.name, - reason: capabilityReason(`could not read ${source.name}'s status`), - state: "unavailable", - }; - let timer: TimerHandle | undefined; - const expired = new Promise((resolve) => { - timer = this.options.clock.setTimer(CAPABILITY_READ_BUDGET_MS, () => resolve(undefined)); - }); - try { - const answer = await Promise.race([source.read().catch((): undefined => undefined), expired]); - if (answer === undefined) return unreadable; - if (answer.state === "available" || answer.state === "off") { - return { name: source.name, state: answer.state }; - } - if (answer.state !== "unavailable") { - return { name: source.name, reason: "reported an invalid status", state: "unavailable" }; - } - return { - name: source.name, - reason: capabilityReason(answer.reason ?? ""), - state: "unavailable", - }; - } finally { - if (timer !== undefined) this.options.clock.cancel(timer); - } - } - - // why: a request that waits in the worker's queue is not checked again; the check runs once, before leasing. - async #requireCapabilities(requires: readonly string[] | undefined): Promise { - if (requires === undefined || requires.length === 0) return; - const sources = this.options.capabilitySources ?? []; - const states = await Promise.all( - requires.map(async (name) => { - const source = sources.find((candidate) => candidate.name === name); - return source === undefined ? undefined : this.#readCapability(source); - }), - ); - const missing = requires.filter((_name, index) => states[index]?.state !== "available"); - if (missing.length === 0) return; - throw new DispatchError( - "CAPABILITY_MISSING", - `This worker cannot serve ${missing.join(", ")}`, - { capabilities: missing }, - ); - } - - #leaseRequest: Handler<"lease.request"> = async (input, session) => { - const request: DeviceRequest = requestedDevice(input); - this.#requireTtlWithinCap(input.ttlMs); - const requesterId = input.requesterId ?? principalAsRequester(session.principal); - // fallow-ignore-next-line complexity -- lease payload assembly and the download-policy rewrite are one transaction, moved verbatim from DaemonServer's former #requestLease. - return this.#unadmitted.track(requesterId, async (admittedHook) => { - const requestedAllowDownload = input.allowDownload ?? false; - const downloadsPolicy = this.options.config.downloads.policy; - // why: `owner` is read only from the gateway's uplink, never from role alone: HTTP's `operator` token - // maps to `admin`, so a role check would let any operator name someone else's owner. Refused, not ignored. - if (input.owner !== undefined && session.isGatewayUplink !== true) { - throw new DispatchError( - "FORBIDDEN", - "Only the gateway's own uplink session may set lease.request's `owner` field", - ); - } - // why: Uplink only, refused before anything is stored: a probe is never queued on a worker, so it must say `noWait`. - if (input.fleetRequestId !== undefined) { - if (session.isGatewayUplink !== true) { - throw new DispatchError( - "FORBIDDEN", - "Only the gateway's own uplink session may set lease.request's `fleetRequestId` field", - ); - } - if (input.noWait !== true) { - throw new DispatchError( - "BAD_REQUEST", - "lease.request's `fleetRequestId` is sent only with `noWait: true`", - ); - } - } - await this.#requireCapabilities(input.requires); - const ownerId = input.owner ?? session.principal; - try { - const grant = await this.options.leases.request(request, { - allowDownload: effectiveAllowDownload(downloadsPolicy, requestedAllowDownload), - noWait: input.noWait ?? false, - ownerId, - requesterId, - ...(session.onProgress === undefined ? {} : { onProgress: session.onProgress }), - onAdmitted: admittedHook(session.onRequestAdmitted), - ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }), - ...(input.leaseId === undefined ? {} : { leaseId: input.leaseId }), - ...(input.fleetRequestId === undefined ? {} : { fleetRequestId: input.fleetRequestId }), - ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }), - ...(input.ttlMs === undefined ? {} : { ttlMs: input.ttlMs }), - }); - const { device } = grant; - return { - ...grant, - device: device.physical ? { ...device, class: device.spec.class } : device, - }; - } catch (error: unknown) { - const refusal = leaseRequestRefusal(error); - if (refusal !== undefined) throw refusal; - if (error instanceof NoCapacityError) { - const capacityReason = capacityReasonOf(error.reason); - if (capacityReason !== undefined) { - throw new DispatchError("NO_CAPACITY", error.message, { capacityReason }); - } - } - // why: The lease path sees only the clamped-to-false permission, so this is the one place that can - // tell the caller that config, not missing consent, blocked the download. - if ( - downloadsPolicy === "never" && - error instanceof RuntimeMissingError && - error.downloadable - ) { - error.message = `${error.message} (downloads are disabled by configuration: downloads.policy is "never")`; - } - throw error; - } - }); - }; - - #leaseCancel: Handler<"lease.cancel"> = async (input, session) => { - const requesterId = input.requesterId ?? session.principal; - await this.#unadmitted.settled(requesterId); - const result = await this.options.queue.cancelPending(requesterId, input.platform); - return { result }; - }; - - #leaseRenew: Handler<"lease.renew"> = async (input) => { - this.#requireTtlWithinCap(input.ttlMs); - return this.options.leases.renew(input.leaseId, input.ttlMs); - }; - - #requireTtlWithinCap(ttlMs: number | undefined): void { - const maxTtlMs = this.options.config.lease.maxTtlMs; - if (ttlMs !== undefined && ttlMs > maxTtlMs) { - throw new DispatchError( - "BAD_REQUEST", - `ttlMs ${String(ttlMs)} exceeds lease.maxTtlMs (${String(maxTtlMs)})`, - ); - } - } - - #leaseRelease: Handler<"lease.release"> = async (input) => { - await this.options.leases.release(input.leaseId, "explicit"); - return { leaseId: input.leaseId }; - }; - - #leaseReleaseAll: Handler<"lease.release-all"> = async () => { - const leaseIds = await this.options.leases.releaseAll("killed"); - return { leaseIds: [...leaseIds] }; - }; - - #leaseGet: Handler<"lease.get"> = (input, session) => { - const lease = this.options.registry.snapshot.leases.find( - (candidate) => candidate.id === input.leaseId, - ); - if (lease === undefined || (session.role !== "admin" && lease.ownerId !== session.principal)) { - throw new UnknownLeaseError(input.leaseId); - } - return { lease }; - }; - - #leaseList: Handler<"lease.list"> = (_input, session) => ({ leases: this.#leasesFor(session) }); - - // why: an agent token must not learn other principals' leases or requester ids; admin sees all. - #leasesFor(session: DispatchSession) { - return this.options.registry.snapshot.leases.filter( - (lease) => session.role === "admin" || lease.ownerId === session.principal, - ); - } - - // why: a waiting entry carries no owner, so the queue's recorded owner of its requester on that row's platform decides. - #waitingFor(session: DispatchSession) { - return this.options.queue - .waitingRequests() - .filter( - (request) => - session.role === "admin" || - this.options.queue.pendingRequestOwner(request.requesterId, request.spec.platform) === - session.principal, - ); - } - - /** - * why: Resolution only: the daemon builds the command and never runs it. Spawning it here would - * attach a user's interactive `adb shell` to the daemon's stdio, and the CLI is the process - * that actually has a terminal (ADR 0001, decision 7). - * - * Both failure modes leave as their own typed errors -- `PassthroughRefusedError` for a verb - * the driver will not proxy, `UnknownPassthroughToolError` for a tool no driver claims -- so - * `errorCode` maps them to `PASSTHROUGH_REFUSED` / `UNKNOWN_PASSTHROUGH_TOOL` and - * `DaemonServer#describeError` still gets the chance to append the driver's refusal summary. - */ - #driverPassthrough: Handler<"driver.passthrough"> = (input) => { - if (this.options.passthrough === undefined) { - throw new DispatchError("INTERNAL", "Tool passthrough is unavailable"); - } - return this.options.passthrough.passthrough(input.tool, undefined, input.args); - }; - - #relays(session: DispatchSession): RelayConnection { - if (session.isGatewayUplink !== true) { - throw new DispatchError("FORBIDDEN", "Only the gateway's own uplink session may relay"); - } - if (session.relays === undefined) { - throw new DispatchError("INTERNAL", "The uplink session carries no relays"); - } - return session.relays; - } - - #relayRequest: Handler<"relay.request"> = (input, session) => { - const relays = this.#relays(session); - if (session.relayChannel === undefined) { - throw new DispatchError( - "BAD_REQUEST", - `The relay's channel ${input.channel} was not accepted`, - ); - } - return relays.request( - input, - { head: (status, headers) => session.onRelayHead?.(status, headers) ?? Promise.resolve() }, - session.relayChannel, - ); - }; - - #relayCancel: Handler<"relay.cancel"> = (input, session) => { - this.#relays(session).cancel(input.relayId); - return {}; - }; - - #leaseIsActive(leaseId: string): boolean { - return this.options.registry.snapshot.leases.some((candidate) => candidate.id === leaseId); - } - - #deviceRotate: Handler<"device.rotate"> = async (input) => { - if (this.options.inputs === undefined) { - throw new DispatchError("INTERNAL", "Device input is unavailable"); - } - const lease = this.options.registry.snapshot.leases.find( - (candidate) => candidate.id === input.leaseId, - ); - if (lease === undefined) throw new UnknownLeaseError(input.leaseId); - const record = this.options.registry.snapshot.devices.find( - (candidate) => candidate.id === lease.deviceId, - ); - if (record === undefined) throw new UnknownLeaseError(input.leaseId); - const refusal = rotationRefusal(input.leaseId, record.spec); - if (refusal !== undefined) throw refusal; - try { - const held = await this.options.inputs.leaseInput( - input.leaseId, - record.spec.platform, - toDriverDevice(record), - ); - try { - if (isUnavailableInput(held.input)) throw new Error(held.input.unavailable); - if (!this.#leaseIsActive(input.leaseId)) throw new UnknownLeaseError(input.leaseId); - await held.input.send({ kind: "orientation", orientation: input.orientation }); - } finally { - held.release(); - } - } catch (error) { - if (error instanceof UnknownLeaseError) throw error; - throw inputUnavailable(input.leaseId, error); - } - return { leaseId: input.leaseId, orientation: input.orientation }; - }; - - /** - * why: ADR 0005 §19a: the same command `driver.passthrough` would have handed back, run here - * instead of there. Everything a passthrough already decides is reused verbatim -- which - * flag scopes the tool to this daemon's root, which verbs the driver refuses -- because it - * is the same driver call: `#driverPassthrough` above passes no device, this handler passes - * the lease's device and the lease's platform, and it also spawns the result. - * - * `leaseId` is checked against the registry here so an id that names no lease answers - * `UNKNOWN_LEASE` rather than running a command (`authorize`'s `ownsLease` deliberately lets - * an unknown id through so this handler can say that -- see `roles.ts`). Which device the - * command touches is the driver's answer: it fills the lease's device in where the tool - * expects it and refuses a command that names another (ADR 0047 §1-2); this daemon parses - * none of the arguments. That is the same accident boundary ADR 0001 draws for the local - * wrappers, reached over the wire. - * - * Nothing is buffered: each chunk goes straight to `session.onOutput` as it arrives (§19e), - * so a command that writes a gigabyte costs this process nothing, and a client sees the - * first line while the command is still running. - */ - #deviceExec: Handler<"device.exec"> = async (input, session) => { - if (this.options.passthrough === undefined) { - throw new DispatchError("INTERNAL", "Tool passthrough is unavailable"); - } - const processRunner = this.options.processRunner; - if (processRunner === undefined) { - throw new DispatchError("INTERNAL", "Command execution is unavailable"); - } - // why: checked here, not as a refinement on the shared input, which `.omit()` could no longer build `run_on_device` from. - if (input.stdin !== undefined && input.stdinFile !== undefined) { - throw new DispatchError("BAD_REQUEST", "Give stdin or stdinFile, not both"); - } - const lease = this.options.registry.snapshot.leases.find( - (candidate) => candidate.id === input.leaseId, - ); - if (lease === undefined) throw new UnknownLeaseError(input.leaseId); - const record = this.options.registry.snapshot.devices.find( - (candidate) => candidate.id === lease.deviceId, - ); - if (record === undefined) throw new UnknownLeaseError(input.leaseId); - - // why: `hasTerminal: false` tells the driver its caller has no terminal: the command runs - // here, with pipes and no pty (ADR 0005 §19c), so a driver may refuse something it allows - // a local `simlock ` invocation -- a bare `adb shell` would otherwise sit on those - // pipes until `exec.timeoutMs` killed it. `platform` is the leased device's own. - const files = this.options.leaseFiles; - if (files === undefined) throw new DispatchError("INTERNAL", "Lease files are unavailable"); - const command = await this.options.passthrough.passthrough( - input.tool, - toDriverDevice(record), - input.args, - { hasTerminal: false, platform: record.spec.platform }, - ); - return this.#runExec( - { - command, - files, - grant: { grantedAt: lease.grantedAt, id: lease.id, platform: record.spec.platform }, - input, - processRunner, - }, - session, - ); - }; - - async #runExec( - run: { - readonly command: PassthroughCommand; - readonly files: Pick; - readonly grant: LeaseFilesGrant; - readonly input: z.infer<(typeof OPERATIONS)["device.exec"]["input"]>; - readonly processRunner: ProcessRunner; - }, - session: DispatchSession, - ): Promise<{ readonly exitCode: number }> { - const { command, files, grant, input, processRunner } = run; - const execEnv: NodeJS.ProcessEnv = { ...this.options.execEnv, ...command.env }; - for (const key of command.unsetEnv ?? []) { - // why: the base comes from the daemon's environment, so a key it kept could re-aim the command. - delete execEnv[key]; - } - const cwd = await this.#folderOf(files, grant); - const file = - input.stdinFile === undefined - ? undefined - : await this.#openStdinFile(files, grant, input.stdinFile); - try { - const handle = processRunner.spawnStreaming(command.command, command.args, { - cwd, - env: execEnv, - // why: Returned, not fired and forgotten: whatever the transport hands back is what pauses - // the child until the chunk has actually gone somewhere (ADR 0005 §19e). - onChunk: (stream, chunk) => session.onOutput?.(stream, chunk), - ...(input.stdin === undefined ? {} : { input: input.stdin }), - ...(file === undefined ? {} : { inputStream: file.read() }), - }); - // why: Announced before the first chunk can arrive, because "it started" is what a transport - // needs to decide its response shape on -- see `DispatchSession.onStarted`. - session.onStarted?.(); - return { - exitCode: await this.#awaitExec(handle, input.tool, input.args, input.stdinFile), - }; - } catch (error) { - if (error instanceof ProcessSpawnError) { - // why: Node reports no reason for a start that failed, so the folder is asked again: a lease that ended after `folderOf` made the working folder vanish (404), anything else is the spawn's own failure. - await this.#folderOf(files, grant); - } - throw error; - } finally { - await file?.close(); - } - } - - async #folderOf(files: Pick, grant: LeaseFilesGrant): Promise { - try { - return await files.folderOf(grant); - } catch (error) { - if (error instanceof LeaseFilesError) { - throw new DispatchError(error.code, error.message, error.details); - } - throw error; - } - } - - async #openStdinFile( - files: Pick, - grant: LeaseFilesGrant, - name: string, - ): Promise { - try { - return await files.openFile(grant, name); - } catch (error) { - if (error instanceof LeaseFilesError) { - throw new DispatchError(error.code, error.message, error.details); - } - // why: the file was listed and then deleted before the open, so it is gone, not a failure of the daemon. - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - throw new DispatchError("UNKNOWN_FILE", `No file ${name} in this lease`, { name }); - } - throw error; - } - } - - async #awaitExec( - handle: StreamingProcessHandle, - tool: string, - args: readonly string[], - stdinFile: string | undefined, - ): Promise { - const timeoutMs = this.options.config.exec.timeoutMs; - const waited = handle.wait(); - // why: `inputFailed` only ever rejects, so the race below hears it as a value and never as a second way for the command to finish. - const inputFailed = handle.inputFailed?.catch((error: unknown) => error); - let timer: TimerHandle | undefined; - const expired = new Promise((resolve) => { - timer = this.options.clock.setTimer(timeoutMs, () => resolve(EXEC_EXPIRED)); - }); - let killTimer: TimerHandle | undefined; - try { - // why: Raced rather than decided by a flag the timer sets: a command that exits in the same - // turn the timer fires has *finished*, and reporting that as a timeout would blame the - // limit for a command that met it. `Promise.race` settles on whichever actually - // happened first, which is exactly the question. - const outcome = await Promise.race([ - waited, - expired, - ...(inputFailed === undefined ? [] : [inputFailed]), - ]); - if (outcome !== EXEC_EXPIRED && !(outcome instanceof InputStreamFailedError)) { - return exitCodeOf(outcome as StreamingProcessResult); - } - - // why: SIGTERM first, SIGKILL after a grace window: a tool that ignores the first must not be - // able to hold this operation -- and the caller's connection -- open forever. The same - // escalation `NodeProcessRunner#run` makes for its own timeout. - killQuietly(handle, "SIGTERM"); - killTimer = this.options.clock.setTimer(EXEC_SIGKILL_GRACE_MS, () => { - killQuietly(handle, "SIGKILL"); - }); - // why: Once the kill is issued, the timeout is the authoritative fact (ADR 0005 §19e) -- - // `waited` rejecting with `ExecOutputDeliveryStalledError` (a consumer that stopped - // reading is *why* the command outran its timeout in the first place: the pause that - // stops it finishing is the same pause that stalls delivery) is a consequence of the - // timeout, not a competing answer to it. Swallowed here rather than left to reject - // this `await` and skip the throw below. - await waited.catch(() => undefined); - if (outcome instanceof InputStreamFailedError && stdinFile !== undefined) { - throw new DispatchError( - "STDIN_FILE_FAILED", - `Reading ${stdinFile} failed after ${String(outcome.bytesWritten)} bytes; the command was stopped`, - { bytesRead: outcome.bytesWritten, name: stdinFile }, - ); - } - // why: `EXEC_TIMEOUT` rather than the exit code the kill produced: "we stopped it" and "it - // failed" are different facts, and only the first tells a caller to raise the limit. - throw new DispatchError( - "EXEC_TIMEOUT", - `\`${tool} ${args.join(" ")}\` exceeded exec.timeoutMs (${String(timeoutMs)}ms) and was killed`, - ); - } finally { - if (timer !== undefined) this.options.clock.cancel(timer); - if (killTimer !== undefined) this.options.clock.cancel(killTimer); - } - } - - #doctorRun: Handler<"doctor.run"> = async (input) => { - if (this.options.doctor === undefined) throw new DoctorUnavailableError(); - // why: `purgeOrphans` travels as its own flag all the way down, never folded into `fix`: - // someone already running `doctor --fix` unattended in CI must not acquire a - // destructive behaviour by upgrading (ADR 0001, decision 6). - return this.options.doctor.reconcile({ - fix: input.fix ?? false, - prerequisites: true, - purgeOrphans: input.purgeOrphans ?? false, - }); - }; - - #listGet: Handler<"list.get"> = (input) => { - const snapshot = this.options.registry.snapshot; - switch (input.kind) { - case "leases": - return [...snapshot.leases]; - case "rules": - return this.options.reaper.rules; - case "requests": - return [...this.options.queue.waitingRequests()]; - case "devices": - case undefined: - return snapshot.devices.map((device) => this.#decorateDevice(device)); - } - }; - - #cleanupRun: Handler<"cleanup.run"> = async (input) => - this.options.reaper.run({ - dryRun: input.dryRun ?? false, - ...(input.rule === undefined ? {} : { rule: input.rule }), - }); - - #nukeRun: Handler<"nuke.run"> = async (input) => { - if (this.options.nuke === undefined) throw new NukeUnavailableError(); - return this.options.nuke.run({ deleteDevices: input.deleteDevices ?? false }); - }; - - #warmForget: Handler<"warm.forget"> = async (input) => { - if (this.options.warmKinds === undefined) - throw new DispatchError("INTERNAL", "The warm pool is unavailable"); - const forgotten = this.options.warmKinds.forget(input); - return { - forgotten: forgotten.map((spec) => ({ - platform: spec.platform, - model: spec.model, - osVersion: spec.osVersion, - mode: spec.mode ?? "full", - ...(spec.imageTag === undefined ? {} : { imageTag: spec.imageTag }), - ...(spec.baseline === undefined ? {} : { baseline: spec.baseline.name }), - })), - }; - }; - - #baselineSave: Handler<"baseline.save"> = async (input, session) => { - if (this.options.baselines === undefined) - throw new DispatchError("INTERNAL", "Baselines are unavailable"); - const lease = this.options.registry.snapshot.leases.find( - (candidate) => candidate.id === input.leaseId, - ); - if (lease === undefined) throw new UnknownLeaseError(input.leaseId); - const held = this.options.registry.snapshot.devices.find( - (candidate) => candidate.id === lease.deviceId, - ); - if (held?.physical === true) { - throw new DispatchError( - "BAD_REQUEST", - `lease ${lease.id} is on a physical device; only a simulator or emulator can be saved as a baseline`, - ); - } - try { - return await this.options.baselines.save({ - leaseId: input.leaseId, - name: input.name, - ...(input.note === undefined ? {} : { note: input.note }), - requesterId: session.principal, - }); - } catch (error: unknown) { - if (error instanceof BaselineSaveFailedError) { - throw new DispatchError("BASELINE_SAVE_FAILED", error.message, { - leaseId: error.leaseId, - name: error.baselineName, - }); - } - throw error; - } - }; - - #baselineList: Handler<"baseline.list"> = async () => { - if (this.options.baselines === undefined) - throw new DispatchError("INTERNAL", "Baselines are unavailable"); - return { baselines: [...(await this.options.baselines.list())] }; - }; - - #baselineRemove: Handler<"baseline.remove"> = async (input, session) => { - if (this.options.baselines === undefined) - throw new DispatchError("INTERNAL", "Baselines are unavailable"); - if (input.workerId !== undefined) { - throw new DispatchError( - "BAD_REQUEST", - "workerId names a gateway's worker, and this daemon is not a gateway; remove without --worker", - ); - } - try { - const removed = await this.options.baselines.remove({ - name: input.name, - requesterId: session.principal, - }); - return { - deletedDevices: [...removed.deletedDevices], - name: removed.name, - versions: [...removed.versions], - }; - } catch (error: unknown) { - if (error instanceof BaselineInUseError) { - throw new DispatchError("BASELINE_IN_USE", error.message, { - leaseId: error.leaseId, - name: error.baselineName, - }); - } - if (error instanceof UnknownBaselineError) { - throw new DispatchError("UNKNOWN_BASELINE", error.message, { name: error.baselineName }); - } - throw describeEnrollmentError(error); - } - }; - - #usageGet: Handler<"usage.get"> = (input) => usageAnswer(this.#usage.get(input)); - - #eventsReplay: Handler<"events.replay"> = (input) => { - // why: a worker's own events carry no workerId, so `worker` can only name this daemon or no one (ADR 0012 §1) - if (input.workerId !== undefined && input.workerId !== this.options.instanceId) { - return Promise.resolve([]); - } - return input.beforeId === undefined && input.limit === undefined - ? this.options.eventHistory.replay( - input.sinceTs === undefined ? {} : { sinceTs: input.sinceTs }, - ) - : eventsPageAnswer( - this.options.eventHistory.replayPage({ - ...(input.beforeId === undefined ? {} : { beforeId: input.beforeId }), - limit: input.limit ?? EVENTS_PAGE_DEFAULT, - }), - ); - }; - - #eventsSubscribe: Handler<"events.subscribe"> = (_input, session) => { - const subscriptionId = session.manageEventSubscription(true); - if (subscriptionId === undefined) { - throw new DispatchError("INTERNAL", "Transport did not provide a subscription id"); - } - return { subscribed: true, subscriptionId }; - }; - - #eventsUnsubscribe: Handler<"events.unsubscribe"> = (_input, session) => { - session.manageEventSubscription(false); - return { subscribed: false }; - }; - - #configGet: Handler<"config.get"> = (input) => - input.showSecrets === true ? this.options.config : redactConfig(this.options.config); - - #workerList: Handler<"worker.list"> = async (_input, session) => { - // why: taken as status resolves, not after the slower catalog read, so the load is not made to look fresher. - let receivedAt = this.options.clock.now(); - const [status, devices, catalog, baselines] = await Promise.all([ - Promise.resolve(this.#statusGet({}, session)).then((answer) => { - receivedAt = this.options.clock.now(); - return answer; - }), - this.#listGet({ kind: "devices" }, session), - this.#keptViewCatalog(session), - this.options.baselines === undefined ? undefined : this.#baselineList({}, session), - ]); - const label = this.options.config.gateway.label; - return { - workers: [ - { - ...workerViewFields({ - ...(baselines === undefined - ? {} - : { baselines: OPERATIONS["baseline.list"].output.parse(baselines) }), - catalog: OPERATIONS["catalog.get"].output.parse(catalog), - config: OPERATIONS["config.get"].output.parse(this.#configGet({}, session)), - devices: OPERATIONS["list.get"].output.parse(devices), - receivedAt, - status: OPERATIONS["status.get"].output.parse(status), - }), - connection: "connected", - drained: false, - id: this.options.instanceId, - lastSeenAt: this.options.clock.now(), - version: this.options.version, - ...(label === undefined ? {} : { label }), - }, - ], - }; - }; - - #keptViewCatalog(session: DispatchSession): Promise { - const now = this.options.clock.now(); - const kept = this.#viewCatalog; - const age = kept === undefined ? undefined : now - kept.readAt; - if ( - kept !== undefined && - age !== undefined && - age >= 0 && - age < WORKER_VIEW_REFRESH_INTERVAL_MS - ) { - return kept.catalog; - } - const catalog = Promise.resolve(this.#catalogGet({}, session)); - this.#viewCatalog = { catalog, readAt: now }; - catalog.catch(() => { - this.#viewCatalog = undefined; - }); - return catalog; - } - - #componentInstall: Handler<"component.install"> = async (input, session) => { - if (!effectiveAllowDownload(this.options.config.downloads.policy, true)) { - throw new DispatchError( - "DOWNLOADS_DISABLED", - 'Downloads are disabled by configuration: downloads.policy is "never"', - { policy: "never" }, - ); - } - const onComponentProgress = session.onComponentProgress; - const onStarted = session.onStarted; - const result = await this.options.components.install({ - component: input.version, - platform: input.platform, - requesterId: session.principal, - ...(onComponentProgress === undefined - ? {} - : { onProgress: (progress) => onComponentProgress(toWireProgress(progress)) }), - ...(onStarted === undefined ? {} : { onAdmitted: onStarted }), - }); - // why: No `stillNeeded` is passed, so `not-needed` cannot come back; a version is always present - // for the two outcomes that can. - if (result.outcome === "not-needed" || result.version === undefined) { - throw new Error(`Internal: component install ended as ${result.outcome} with no version`); - } - return { - component: input.version, - outcome: result.outcome, - platform: input.platform, - version: result.version, - }; - }; - - #componentList: Handler<"component.list"> = async (input) => ({ - components: await this.options.components.list(input.platform), - }); - - #componentRemove: Handler<"component.remove"> = async (input, session) => { - try { - const result = await this.options.components.remove({ - platform: input.platform, - requesterId: session.principal, - version: input.version, - ...(input.dryRun === undefined ? {} : { dryRun: input.dryRun }), - }); - return { platform: input.platform, version: input.version, ...result }; - } catch (error: unknown) { - if (error instanceof ComponentInUseError) { - throw new DispatchError("COMPONENT_IN_USE", error.message, { - devices: error.devices, - foreignDevices: error.foreignDevices, - }); - } - throw error; - } - }; - - #deviceAdd: Handler<"device.add"> = async (input) => { - try { - const { device, reenrolled } = await this.#requireEnrollment().add(input.platform, input.id); - return { - appCount: device.enrolledApps.length, - class: device.spec.class, - driverDeviceId: device.driverDeviceId, - id: device.id, - model: device.spec.model, - osVersion: device.spec.osVersion, - platform: device.spec.platform, - reenrolled, - }; - } catch (error: unknown) { - throw describeEnrollmentError(error); - } - }; - - #deviceRemove: Handler<"device.remove"> = async (input) => { - try { - const device = await this.#requireEnrollment().remove(input.id); - return { driverDeviceId: device.driverDeviceId, id: device.id }; - } catch (error: unknown) { - throw describeEnrollmentError(error); - } - }; - - #requireEnrollment(): Pick { - if (this.options.enrollment === undefined) { - throw new DispatchError("INTERNAL", "Device enrollment is unavailable"); - } - return this.options.enrollment; - } - - #tokenCreate: Handler<"token.create"> = async (input) => { - const { record, secret } = await this.#requireTokens().create(input.role, input.label); - return { secret, token: record }; - }; - - // why: a worker answers as a fleet of one (ADR 0012): the rules read its own view, leases and queue. - #inboxGet: Handler<"inbox.get"> = async (_input, session) => { - const [listed, tokens] = await Promise.all([ - this.#workerList({}, session), - Promise.resolve() - .then(() => this.#requireTokens().list()) - .catch(() => []), - ]); - return { - items: [ - ...inboxItems(OPERATIONS["worker.list"].output.parse(listed).workers, { - leases: OPERATIONS["lease.list"].output.parse(this.#leaseList({}, session)).leases, - now: this.options.clock.now(), - requests: z.array(waitingRequestSchema).parse(this.options.queue.waitingRequests()), - tokens: OPERATIONS["token.list"].output.parse({ tokens }).tokens, - }), - ], - }; - }; - - #tokenList: Handler<"token.list"> = async () => ({ tokens: await this.#requireTokens().list() }); - - #tokenRevoke: Handler<"token.revoke"> = async (input) => ({ - revoked: await this.#requireTokens().revoke(input.id), - }); - - #requireTokens(): TokenOperations { - if (this.options.tokens === undefined) { - throw new DispatchError("INTERNAL", "Token store is unavailable"); - } - return this.options.tokens; - } - - #decorateDevice(device: DeviceRecord): DeviceRecord & { - readonly class?: PhysicalClass; - readonly servesDefaultMode: boolean; - readonly presence?: LatestPresence; - readonly transitionAgeMs?: number; - readonly stalled?: true; - } { - // why: `reclaimingSince` only ages a transition: it stays off the wire (`transitionAgeMs` carries it). - const { reclaimingSince: _internal, ...onTheWire } = device as DeviceRecord & { - readonly reclaimingSince?: number; - }; - const presence = device.physical ? this.options.presence?.(device.driverDeviceId) : undefined; - const decorated = { - ...onTheWire, - ...(device.physical ? { class: device.spec.class } : {}), - ...(presence === undefined ? {} : { presence }), - servesDefaultMode: this.options.deviceModes.servesDefaultMode(device.spec), - }; - const enteredAt = transitionEnteredAt(device); - if (enteredAt === undefined) return decorated; - const now = this.options.clock.now(); - return { - ...decorated, - transitionAgeMs: now - enteredAt, - ...(this.#isStalled(device, now) ? { stalled: true as const } : {}), - }; - } - - #isStalled(device: DeviceRecord, now: number): boolean { - const stalls = this.options.stalls; - if (stalls === undefined) return false; - return isStalledTransition({ - claims: stalls.claims, - config: this.options.config.stalledTransition, - device, - driver: stalls.drivers.find((driver) => driver.platform === device.spec.platform), - now, - }); - } -} - -/** - * why: The installer's progress in the contract's words: its percentage, already within 0..100, - * becomes a fraction from 0 to 1, rounded to three decimals so that 1.1 percent is 0.011 and not - * the 0.011000000000000001 a plain division gives. Only 100 percent becomes 1: 99.96 would round - * up to it, and 1 is what an install says once it is complete. With none (the install has just - * started, or the driver's was not a number) there is no fraction. - */ -function toWireProgress(progress: ComponentInstallerProgress): ComponentProgress { - if (progress.stage === "waiting") return { stage: "waiting" }; - if (progress.percent === undefined) return { stage: "downloading" }; - const fraction = Math.round(progress.percent * 10) / 1000; - return { fraction: progress.percent < 100 ? Math.min(fraction, 0.999) : 1, stage: "downloading" }; -} - -/** why: A child that exited between the timer firing and the signal landing is not an error worth - * failing the operation over -- the wait below is about to report how it ended anyway. */ -function killQuietly(handle: StreamingProcessHandle, signal: NodeJS.Signals): void { - try { - handle.kill(signal); - } catch { - // Already gone. - } -} - -function describeEnrollmentError(error: unknown): unknown { - if (error instanceof DeviceNotEnrollableError) { - return new DispatchError("DEVICE_NOT_ENROLLABLE", error.message, { - id: error.id, - platform: error.platform, - reason: error.reason, - ...(error.canonicalId === undefined ? {} : { canonicalId: error.canonicalId }), - }); - } - if (error instanceof DeviceBusyError) { - return new DispatchError("DEVICE_BUSY", error.message, { - id: error.id, - reason: error.reason, - }); - } - if (error instanceof UnknownEnrolledDeviceError) { - return new DispatchError("UNKNOWN_DEVICE", error.message, { id: error.id }); - } - return error; -} - -const INPUT_REASON_MAX = 200; - -function inputUnavailable(leaseId: string, error: unknown): DispatchError { - const text = error instanceof Error ? error.message : String(error); - const reason = text === "" ? "unknown error" : text.slice(0, INPUT_REASON_MAX); - return new DispatchError( - "INPUT_UNAVAILABLE", - `Lease ${leaseId}'s simulator cannot take input: ${reason}`, - { leaseId, reason }, - ); -} - -function unsupportedOnWorker( - operation: OperationName, - message = `${operation} acts on a gateway's workers, and this daemon is not a gateway`, -): ErasedHandler { - return () => { - throw new DispatchError("UNSUPPORTED_IN_WORKER_MODE", message, { operation }); - }; } From 7bd5a7d05469b3e1ddf990229f0cfcaec1eb3c5b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:14:49 +0100 Subject: [PATCH 17/83] refactor(gateway): split fleet-coordinator.ts into parts --- src/gateway/fleet-admission.ts | 383 +++++++++ src/gateway/fleet-coordinator.ts | 1197 +++------------------------- src/gateway/fleet-dispatch.ts | 556 +++++++++++++ src/gateway/fleet-facts.ts | 74 ++ src/gateway/fleet-grant.ts | 97 +++ src/gateway/fleet-index-upkeep.ts | 75 ++ src/gateway/fleet-request-shape.ts | 53 ++ 7 files changed, 1327 insertions(+), 1108 deletions(-) create mode 100644 src/gateway/fleet-admission.ts create mode 100644 src/gateway/fleet-dispatch.ts create mode 100644 src/gateway/fleet-facts.ts create mode 100644 src/gateway/fleet-grant.ts create mode 100644 src/gateway/fleet-index-upkeep.ts create mode 100644 src/gateway/fleet-request-shape.ts diff --git a/src/gateway/fleet-admission.ts b/src/gateway/fleet-admission.ts new file mode 100644 index 000000000..28babfc3b --- /dev/null +++ b/src/gateway/fleet-admission.ts @@ -0,0 +1,383 @@ +import { type AdmissionDecision, type AdmissionOptions, askAdmission } from "./admission.js"; +import { + admissionDeniedError, + admissionFailedError, + leaseEndedError, + type Platform, +} from "../contract/index.js"; +import { + type DeviceRequest, + type LeaseRequestFailure, + type LeaseRequestLimits, + type MaxPerRequester, + newLeaseRequestId, + SerializedDecision, +} from "../core/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import { + InMemoryLeaseRequestStore, + LeaseRequestBook, + requesterLimitDecision, +} from "../leasing/index.js"; +import type { Clock, IdGenerator, Logger } from "../ports/index.js"; +import type { EndedLeaseIds } from "./ended-lease-ids.js"; +import type { FleetDispatch } from "./fleet-dispatch.js"; +import type { FleetFacts } from "./fleet-facts.js"; +import { leaseIdTaken } from "./fleet-grant.js"; +import type { FleetViews, WorkerDirectory } from "./fleet-ports.js"; +import type { FleetLeaseIndex } from "./lease-index.js"; +import { + RequestCancelledError, + type FleetLeaseGrant, + type FleetQueue, + type LeaseRequestOptions, +} from "./queue.js"; + +/** + * why: ADR 0074 §1: `admissionRequest` is the `lease.request` input as the caller sent it, which the + * admission command reads; the gateway dispatcher fills `ttlMs` and drops `allowDownload` after that. + */ +export type FleetRequestOptions = LeaseRequestOptions & { + readonly admissionRequest?: Readonly>; + /** why: ADR 0074 §3: the caller's `metadata` goes to the admission command only, never into an event, a record or a worker call. */ + readonly admissionMetadata?: Readonly>; +}; + +/** why: a request whose admission command is running: `lease.cancel` and `dispose` end it by aborting `controller`. */ +interface AskingRequest { + readonly requesterId: string; + readonly platform: Platform; + readonly controller: AbortController; + end?: "cancelled" | "stopping"; +} + +export interface FleetAdmissionOptions { + readonly admission?: AdmissionOptions; + readonly clock: Clock; + readonly idGenerator: IdGenerator; + readonly describeFailure: (error: unknown) => LeaseRequestFailure; + readonly leaseRequestLimits: LeaseRequestLimits; + readonly maxPerRequester?: MaxPerRequester; + readonly leaseIndex: FleetLeaseIndex; + readonly views: FleetViews; + readonly endedLeaseIds: EndedLeaseIds; + readonly directory: WorkerDirectory; + readonly logger: Logger; + readonly queue: FleetQueue; + readonly facts: FleetFacts; + readonly dispatch: Pick; +} + +export class FleetAdmission { + readonly #decisions = new SerializedDecision(); + readonly #asking = new Set(); + readonly requests: LeaseRequestBook; + + constructor(private readonly options: FleetAdmissionOptions) { + this.requests = new LeaseRequestBook({ + decisions: this.#decisions, + describeFailure: options.describeFailure, + store: new InMemoryLeaseRequestStore({ + clock: options.clock, + idGenerator: options.idGenerator, + limits: options.leaseRequestLimits, + }), + }); + } + + /** + * why: Admits a request and returns the eventual grant. The one-lease check and the enqueue commit + * together, inside `SerializedDecision.run`, so two concurrent requests for the same + * `requesterId` can never both be admitted (see the module doc's "two different fields"). + * Everything past admission -- reaching a worker at all -- is I/O and stays outside it. + */ + async request( + deviceRequest: DeviceRequest, + options: FleetRequestOptions, + ): Promise { + // why: ADR 0074 §1: with a command, the checks run under the lock, then the command outside it, then the checks again with the answer under the lock; no await without a command, so a `lease.cancel` sent right after the request finds it in the same tick (#1492). The request is listed as asking before the first await, so a cancel or a stop in that window ends it too. + const { admission } = this.options; + let decision: AdmissionDecision | undefined; + if (admission !== undefined) { + const asking: AskingRequest = { + controller: new AbortController(), + platform: deviceRequest.platform, + requesterId: options.requesterId, + }; + this.#asking.add(asking); + try { + const early = await this.#decisions.run(async () => + this.#checkAdmission(deviceRequest, options), + ); + if ("replay" in early) return early.replay; + // why: the caller is answered at once; the runner kills the command on its own time (it kills with SIGKILL). + if (asking.end === undefined) { + decision = await Promise.race([ + this.#askAdmission(admission, deviceRequest, options, asking.controller), + new Promise((resolve) => + asking.controller.signal.addEventListener("abort", () => resolve(undefined), { + once: true, + }), + ), + ]); + } + } finally { + this.#asking.delete(asking); + } + if (asking.end !== undefined) + this.#refuseEndedWhileAsking(asking.end, deviceRequest, options); + } + const admitted = await this.#decisions.run(async () => { + const checked = this.#checkAdmission(deviceRequest, options); + if ("replay" in checked) return checked; + const { requestId } = checked; + if (decision !== undefined && decision.kind !== "priority") { + this.#refuseByAdmission(decision, deviceRequest, options, requestId); + } + const { id, started } = await this.requests.admit( + deviceRequest, + options, + (id, onProgress) => { + const waiter = this.options.queue.create( + deviceRequest, + { ...options, onProgress }, + id, + decision?.kind === "priority" ? decision.priority : 0, + ); + return { promise: this.#afterViewRead(waiter.promise), waiter }; + }, + requestId, + ); + const created = started.waiter; + this.options.dispatch.noteCreated(created); + this.options.facts.emit("lease.requested", { + requestId: id, + requestSpec: deviceRequest, + requester: options.requesterId, + waitPolicy: options.noWait === true ? "no-wait" : "wait", + }); + return { promise: started.promise, waiter: created }; + }); + if ("replay" in admitted) return admitted.replay; + this.options.dispatch.admit(admitted.waiter); + return admitted.promise; + } + + #endAsking(asking: AskingRequest, end: "cancelled" | "stopping"): void { + asking.end = end; + asking.controller.abort(); + } + + /** why: replay, the requester limit, a taken id and an ended id, in this order; a refusal is thrown, a replay returned. */ + #checkAdmission( + deviceRequest: DeviceRequest, + options: LeaseRequestOptions, + ): { readonly replay: Promise } | { readonly requestId: string } { + const replay = this.requests.replay(deviceRequest, options); + if (replay !== undefined) return { replay }; + const requestId = newLeaseRequestId(this.options.idGenerator); + this.#refuseIfAlreadyLeased(deviceRequest, options.requesterId, requestId); + this.#refuseIfLeaseIdTaken(deviceRequest, options, requestId); + this.#refuseIfLeaseIdEnded(deviceRequest, options, requestId); + return { requestId }; + } + + /** why: ADR 0074 §1: the operator's command answers outside the decision lock, since it may take seconds. */ + #askAdmission( + admission: AdmissionOptions, + deviceRequest: DeviceRequest, + options: FleetRequestOptions, + controller: AbortController, + ): Promise { + return askAdmission( + admission, + { + metadata: options.admissionMetadata ?? {}, + platform: deviceRequest.platform, + principal: options.ownerId, + request: options.admissionRequest ?? deviceRequest, + requesterId: options.requesterId, + }, + this.options.logger, + controller.signal, + ); + } + + /** why: the caller cancelled, or the gateway stopped, while the command ran: nothing was stored, so there is nothing to undo. */ + #refuseEndedWhileAsking( + end: "cancelled" | "stopping", + deviceRequest: DeviceRequest, + options: FleetRequestOptions, + ): never { + if (end === "stopping") throw new DispatchError("DAEMON_STOPPING", "Gateway is stopping"); + const requestId = newLeaseRequestId(this.options.idGenerator); + this.options.facts.emit("lease.rejected", { + requestId, + requester: options.requesterId, + requestSpec: deviceRequest, + reason: "cancelled", + }); + throw new RequestCancelledError(requestId); + } + + #refuseByAdmission( + decision: Exclude, + deviceRequest: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): never { + this.options.facts.emit("lease.rejected", { + requestId, + requester: options.requesterId, + requestSpec: deviceRequest, + reason: decision.kind === "deny" ? "admission-denied" : "admission-failed", + }); + const refusal = + decision.kind === "deny" + ? admissionDeniedError(decision.reason) + : admissionFailedError(decision.cause); + throw new DispatchError(refusal.code, refusal.message, refusal.details); + } + + /** + * why: the request book stores `granted` and wakes its watchers when this promise settles, and the + * worker's own `lease.granted` push starts its view read later; so the grant waits for one read of the + * granting worker's view here, and a `lease.list` right after any answer lists the lease (#1370). A + * failed read is the next tick's. + */ + async #afterViewRead(grant: Promise): Promise { + const granted = await grant; + const entry = this.options.leaseIndex.resolve(granted.lease.id); + if (entry !== undefined) { + await this.options.directory + .target(entry.workerId) + ?.refresh() + .catch((error: unknown) => { + this.options.logger.debug("Failed to refresh a worker's view after granting it a lease", { + error: String(error), + workerId: entry.workerId, + }); + }); + } + return granted; + } + + #refuseIfAlreadyLeased( + deviceRequest: DeviceRequest, + requesterId: string, + requestId: string, + ): void { + const refusal = requesterLimitDecision({ + held: this.options.leaseIndex + .held(requesterId) + .filter((lease) => !this.#isPastDeadlineOnDisconnectedWorker(lease.id)), + maxPerRequester: this.options.maxPerRequester, + platform: deviceRequest.platform, + probe: false, + requesterId, + waiting: this.options.queue.pendingPlatforms(requesterId), + }); + if (refusal === undefined) return; + this.options.facts.emit("lease.rejected", { + requestId, + requester: requesterId, + requestSpec: deviceRequest, + reason: "already-leased", + }); + throw refusal; + } + + /** + * why: a lease past its last reported deadline on a disconnected worker cannot have been renewed + * through this gateway, so it must not lock its requester out of the fleet. The entry stays in + * the index: it is still routed to its worker (ADR 0005 §28). + */ + #isPastDeadlineOnDisconnectedWorker(gatewayLeaseId: string): boolean { + const entry = this.options.leaseIndex.resolve(gatewayLeaseId); + if (entry === undefined) return false; + const view = this.options.views.view(entry.workerId); + if (view === undefined || view.connection !== "disconnected") return false; + const reported = view.leases?.find((lease) => lease.id === entry.workerLeaseId); + return reported !== undefined && reported.ttlDeadline <= this.options.clock.now(); + } + + /** + * why: ADR 0020: the one clash check on a gateway, after the one-lease check and in the same + * admission section. A caller-chosen id is held by one of this gateway's leases and by one of + * its own requests still open; leases a local client holds on a worker are not checked (a + * worker refuses its own, and the requester's uniqueness guarantee makes a clash unlikely). + */ + #refuseIfLeaseIdTaken( + deviceRequest: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + if ( + this.options.leaseIndex.resolve(leaseId) === undefined && + !this.requests.holdsLeaseId(leaseId) + ) { + return; + } + this.options.facts.emit("lease.rejected", { + requestId, + requester: options.requesterId, + requestSpec: deviceRequest, + reason: "lease-id-taken", + }); + throw leaseIdTaken(leaseId); + } + + /** why: ADR 0074 §4: after the taken-id check, a chosen id whose lease this gateway learned ended is refused without a dispatch; a worker answers for ends the gateway missed. */ + #refuseIfLeaseIdEnded( + deviceRequest: DeviceRequest, + options: LeaseRequestOptions, + requestId: string, + ): void { + const { leaseId } = options; + if (leaseId === undefined) return; + const endedAt = this.options.endedLeaseIds.endedAt(leaseId); + if (endedAt === undefined) return; + this.options.facts.emit("lease.rejected", { + requestId, + requester: options.requesterId, + requestSpec: deviceRequest, + reason: "lease-ended", + }); + const ended = leaseEndedError(leaseId, endedAt); + throw new DispatchError(ended.code, ended.message, ended.details); + } + + /** + * why: a `new` waiter is admitted but not yet walked: no worker was asked, so it is as safe to reject as + * a `queued` one (#1492). Only `processing` means a worker already has the request. + */ + async cancelPending( + requesterId: string, + platform?: Platform, + ): Promise<"cancelled" | "not-found" | "not-cancellable"> { + for (const asking of this.#asking) { + if ( + asking.requesterId === requesterId && + (platform === undefined || asking.platform === platform) + ) { + this.#endAsking(asking, "cancelled"); + return "cancelled"; + } + } + return this.#decisions.run(async () => { + const waiter = this.options.queue.findPendingWaiter(requesterId, platform); + if (waiter === undefined) return "not-found"; + if (waiter.state === "processing") return "not-cancellable"; + if (this.options.queue.reject(waiter, new RequestCancelledError(waiter.id))) { + this.options.facts.rejected(waiter, "cancelled"); + } + return "cancelled"; + }); + } + + stopAsking(): void { + for (const asking of this.#asking) this.#endAsking(asking, "stopping"); + } +} diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index 4df94385e..dc23d14e2 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -5,26 +5,8 @@ * worker either grants a forwarded `lease.request` immediately or refuses it, because every * dispatch is `noWait` (§12). * - * ## The dispatch race (see the module's own tests for the one that catches a naive fix) - * - * Dispatch re-runs on every worker-view change (`FleetViews#onViewsChanged`) and once more per - * settled attempt. (C1/C2, round 3 review: `#attempt` now calls `#dispatch()` itself, right after - * `#settleGrant` and right after its own terminal `reject` -- see `#attempt`'s own doc for why - * this was previously false. `#admit` has no look of its own at all: it hands the - * brand-new waiter to that same walk as its `candidate` -- see `#admit`'s own doc, and - * architecture rule 10.) A waiter whose forwarded `lease.request` is still in - * flight to worker A must never be picked up by a second pass and sent to worker B too -- if both granted, one requester - * would hold two devices on two machines, and the fleet-wide admission check cannot catch it - * (it runs once, at admission, before either RPC). `#beginAttempt` calls `queue.markProcessing` - * *before* issuing the RPC, taking the waiter out of `queued` state, so `#dispatch`'s own - * `queue.list()` scan skips it for the whole in-flight window; the waiter only becomes `queued` - * again (via `#enqueue`, from `#staleView`) once that attempt has fully settled, or is rejected. There used to be - * a second guard here too -- a `Map` keyed by in-flight target -- but once - * every exit from `#attempt` clears its waiter's processing state in exactly one place (the - * `finally` below), the two checks can never disagree: nothing can observe a waiter that is both - * `queued` and mid-attempt. Keeping a guard with no state it can ever catch was worse than - * deleting it (round 2 review, C1/C2) -- if a future change reintroduces that race, the fix is a - * new guard proven by a new failing test, not resurrecting one that only ever watched itself. + * The dispatch race (why a waiter in flight is never offered to a second worker) is documented in + * `fleet-dispatch.ts`'s module doc. * * ## Two different fields, two different rules * @@ -35,112 +17,38 @@ * (§26) -- the gateway's own inbound principal, resolved by `GatewayDispatcher`'s * `authorizeLookups` from `ownerId()`/`leaseRequesterId()` below, never by this class directly. */ -import type { EventBus, EventMap } from "../bus/index.js"; +import type { BaselineSaveInput, BaselineSaveOutput, RotateOutput } from "../admin/index.js"; +import type { EventBus } from "../bus/index.js"; +import type { Platform } from "../contract/index.js"; import type { - BaselineSaveInput, - BaselineSaveOutput, - LeaseGrant, - RotateOutput, -} from "../admin/index.js"; -import { - admissionDeniedError, - admissionFailedError, - isSimlockError, - leaseEndedError, - requestedClass, - requestedDevice, - type Platform, -} from "../contract/index.js"; -import { DispatchError, type DispatchSession } from "../daemon/dispatch.js"; -import { - type LeaseRequestFailure, - type DeviceRequest, - type LeaseRequestLimits, - type MaxPerRequester, - newLeaseRequestId, - SerializedDecision, + DeviceRequest, + LeaseRequestFailure, + LeaseRequestLimits, + MaxPerRequester, } from "../core/index.js"; -import { - InMemoryLeaseRequestStore, - LeaseRequestBook, - requesterLimitDecision, - type WaitingRequest, -} from "../leasing/index.js"; +import { DispatchError, type DispatchSession } from "../daemon/dispatch.js"; +import type { LeaseRequestBook, WaitingRequest } from "../leasing/index.js"; import type { Clock, IdGenerator, Logger } from "../ports/index.js"; import { NoopLogger } from "../ports/index.js"; -import { - classifyRelayedError, - FleetForwarding, - type FleetExecInput, - type FleetRotateInput, -} from "./fleet-forwarding.js"; -import { liveClient, type WorkerDirectory } from "./fleet-ports.js"; -import type { FleetViews } from "./fleet-ports.js"; -import { FleetRefusals, refusedCapacityReason, type WorkerEntry } from "./fleet-refusals.js"; -import { askAdmission, type AdmissionDecision, type AdmissionOptions } from "./admission.js"; +import type { AdmissionOptions } from "./admission.js"; import type { EndedLeaseIds } from "./ended-lease-ids.js"; -import { type FleetLeaseEntry, type FleetLeaseIndex } from "./lease-index.js"; -import type { WorkerView } from "./worker-registry.js"; -import { - FleetQueue, - lineOf, - RequestCancelledError, - type FleetLeaseGrant, - type FleetLeaseRecord, - type FleetWaiter, - type LeaseRequestOptions, -} from "./queue.js"; -import { - matchRequest, - type RoutableRequest, - type RoutingDecision, - type RoutingPolicy, -} from "./routing.js"; -import { type Rejection, type WorkerReason } from "./routing/index.js"; +import { FleetAdmission, type FleetRequestOptions } from "./fleet-admission.js"; +import { FleetDispatch } from "./fleet-dispatch.js"; +import { FleetFacts, FleetOutcomes } from "./fleet-facts.js"; +import { FleetForwarding, type FleetExecInput, type FleetRotateInput } from "./fleet-forwarding.js"; +import { FleetGrants } from "./fleet-grant.js"; +import { FleetIndexUpkeep } from "./fleet-index-upkeep.js"; +import type { FleetViews, WorkerDirectory } from "./fleet-ports.js"; +import type { WorkerEntry } from "./fleet-refusals.js"; +import type { FleetLeaseIndex } from "./lease-index.js"; +import { FleetQueue, type FleetLeaseGrant, type FleetLeaseRecord } from "./queue.js"; +import type { RoutingPolicy } from "./routing.js"; + +export type { FleetRequestOptions } from "./fleet-admission.js"; // why: the gateway adds this to the leasing module's entry; the contract's schema is its wire shape. export type FleetWaitingRequest = WaitingRequest & { readonly workers?: WorkerEntry[] }; -// why: the filters that hold a worker back only for now (it is full, busy, not ready or unhealthy), so it could take the request later; any other reason (drained, disconnected, a missing driver, runtime, model or capability) means it will not take it by waiting. -const COULD_TAKE_LATER: ReadonlySet = new Set([ - "busy", - "full", - "not-ready", - "unhealthy", - "updating", -]); - -// why: ADR 0042 §3: a worker that began its update after the gateway's last read refuses with `DAEMON_UPDATING`, and its view is as stale as after `NO_CAPACITY`. -const STALE_VIEW_CODES: ReadonlySet = new Set(["NO_CAPACITY", "DAEMON_UPDATING"]); - -const CANNOT_SERVE_CODES: ReadonlySet = new Set([ - "UNKNOWN_MODEL", - "RUNTIME_MISSING", - "NO_DRIVER", - "CAPABILITY_MISSING", - "UNKNOWN_BASELINE", - "BASELINE_MISMATCH", - "BASELINE_STALE", -]); - -/** - * why: ADR 0074 §1: `admissionRequest` is the `lease.request` input as the caller sent it, which the - * admission command reads; the gateway dispatcher fills `ttlMs` and drops `allowDownload` after that. - */ -export type FleetRequestOptions = LeaseRequestOptions & { - readonly admissionRequest?: Readonly>; - /** why: ADR 0074 §3: the caller's `metadata` goes to the admission command only, never into an event, a record or a worker call. */ - readonly admissionMetadata?: Readonly>; -}; - -/** why: a request whose admission command is running: `lease.cancel` and `dispose` end it by aborting `controller`. */ -interface AskingRequest { - readonly requesterId: string; - readonly platform: Platform; - readonly controller: AbortController; - end?: "cancelled" | "stopping"; -} - export interface FleetLeaseCoordinatorOptions { readonly clock: Clock; readonly idGenerator: IdGenerator; @@ -174,46 +82,20 @@ export interface FleetLeaseCoordinatorOptions { readonly admission?: AdmissionOptions; } -type FleetEventName = - | "lease.requested" - | "lease.queued" - | "lease.rejected" - | "queue.changed" - | "request.dispatched" - | "request.granted"; - export class FleetLeaseCoordinator { readonly #queue: FleetQueue; - readonly #decisions = new SerializedDecision(); - readonly #asking = new Set(); + readonly #facts: FleetFacts; + readonly #forwarding: FleetForwarding; + readonly #dispatch: FleetDispatch; + readonly #admission: FleetAdmission; + readonly #upkeep: FleetIndexUpkeep; readonly requests: LeaseRequestBook; readonly #logger: Logger; - #knownWorkerIds = new Set(); - /** why: C1 (round 2 review): the `view.leases` array reference this coordinator last reconciled - * into `leaseIndex` for each worker, keyed by worker id. `onViewsChanged` fires for *every* - * worker-view mutation on the whole registry (`connected`, `refresh`, `disconnected`, - * `setDrained`, `remove`, `pruneExpired` -- see `FleetViews#onViewsChanged`'s own doc), not - * just a change to the worker being reconciled, so calling `rebuildFromWorker` for every view - * on every notification bumped a worker's reconciliation generation without a new snapshot - * ever having arrived for it (see `#onViewsChanged`'s own doc for the eviction this caused). - * `WorkerRegistry` only ever replaces a view's `leases` array when a snapshot naming `leases` - * actually lands for that specific worker (`refresh`'s partial-update contract; `connected` and - * `disconnected` never touch it) -- so this reference is exactly "a new snapshot for this - * worker arrived" and nothing else, with no need to widen `FleetViews` to carry worker - * identity on the notification itself. */ - readonly #lastReconciledLeases = new Map>(); readonly #unsubscribeViews: () => void; - /** why: A `WeakMap` so a waiter that never dispatches (rejected, cancelled, timed out) leaks no entry. */ - readonly #createdAt = new WeakMap(); - readonly #workerReasons = new WeakMap(); - readonly #refusals: FleetRefusals; - readonly #forwarding: FleetForwarding; - #dispatchDepth = 0; - #passRequested = false; constructor(private readonly options: FleetLeaseCoordinatorOptions) { this.#logger = options.logger ?? new NoopLogger(); - this.#refusals = new FleetRefusals({ routing: options.routing, views: options.views }); + this.#facts = new FleetFacts({ eventBus: options.eventBus }); this.#forwarding = new FleetForwarding({ clock: options.clock, directory: options.directory, @@ -227,22 +109,58 @@ export class FleetLeaseCoordinator { clock: options.clock, idGenerator: options.idGenerator, onDepthChange: (depth, line) => - this.#emit("queue.changed", line === "physical" ? { depth, physical: true } : { depth }), - onTimeout: (waiter) => this.#emitRejected(waiter, "timeout"), + this.#facts.emit( + "queue.changed", + line === "physical" ? { depth, physical: true } : { depth }, + ), + onTimeout: (waiter) => this.#facts.rejected(waiter, "timeout"), + }); + const outcomes = new FleetOutcomes({ queue: this.#queue, facts: this.#facts }); + const grants = new FleetGrants({ + forwarding: this.#forwarding, + leaseIndex: options.leaseIndex, + logger: this.#logger, + queue: this.#queue, + facts: this.#facts, + outcomes, }); - this.requests = new LeaseRequestBook({ - decisions: this.#decisions, + this.#dispatch = new FleetDispatch({ + clock: options.clock, + directory: options.directory, + views: options.views, + routing: options.routing, + leaseIndex: options.leaseIndex, + logger: this.#logger, + queue: this.#queue, + forwarding: this.#forwarding, + facts: this.#facts, + outcomes, + grants, + }); + this.#admission = new FleetAdmission({ + ...(options.admission === undefined ? {} : { admission: options.admission }), + clock: options.clock, + idGenerator: options.idGenerator, describeFailure: options.describeFailure, - store: new InMemoryLeaseRequestStore({ - clock: options.clock, - idGenerator: options.idGenerator, - limits: options.leaseRequestLimits, - }), + leaseRequestLimits: options.leaseRequestLimits, + ...(options.maxPerRequester === undefined + ? {} + : { maxPerRequester: options.maxPerRequester }), + leaseIndex: options.leaseIndex, + views: options.views, + endedLeaseIds: options.endedLeaseIds, + directory: options.directory, + logger: this.#logger, + queue: this.#queue, + facts: this.#facts, + dispatch: this.#dispatch, }); - options.leaseIndex.onEnded((entry) => { - if (entry.idChosenByRequester === true) { - options.endedLeaseIds.record(entry.gatewayLeaseId, options.clock.now()); - } + this.requests = this.#admission.requests; + this.#upkeep = new FleetIndexUpkeep({ + clock: options.clock, + endedLeaseIds: options.endedLeaseIds, + leaseIndex: options.leaseIndex, + views: options.views, }); this.#unsubscribeViews = options.views.onViewsChanged(() => this.#onViewsChanged()); } @@ -257,14 +175,14 @@ export class FleetLeaseCoordinator { } start(): void { - this.#emit("queue.changed", { depth: this.#queue.depth }); + this.#facts.emit("queue.changed", { depth: this.#queue.depth }); } waitingRequests(): readonly FleetWaitingRequest[] { const queued = new Map(this.#queue.list().map((waiter) => [waiter.id, waiter])); return this.requests.waiting(this.#queue.places()).map((entry) => { const waiter = queued.get(entry.id); - const workers = waiter === undefined ? undefined : this.#workerReasons.get(waiter); + const workers = waiter === undefined ? undefined : this.#dispatch.workerReasons(waiter); return entry.stage === "queued" && workers !== undefined ? { ...entry, workers: [...workers] } : entry; @@ -286,296 +204,15 @@ export class FleetLeaseCoordinator { return this.#queue.findPendingWaiter(requesterId, platform)?.options.ownerId; } - /** - * why: Admits a request and returns the eventual grant. The one-lease check and the enqueue commit - * together, inside `SerializedDecision.run`, so two concurrent requests for the same - * `requesterId` can never both be admitted (see the module doc's "two different fields"). - * Everything past admission -- reaching a worker at all -- is I/O and stays outside it. - */ - async request( - deviceRequest: DeviceRequest, - options: FleetRequestOptions, - ): Promise { - // why: ADR 0074 §1: with a command, the checks run under the lock, then the command outside it, then the checks again with the answer under the lock; no await without a command, so a `lease.cancel` sent right after the request finds it in the same tick (#1492). The request is listed as asking before the first await, so a cancel or a stop in that window ends it too. - const { admission } = this.options; - let decision: AdmissionDecision | undefined; - if (admission !== undefined) { - const asking: AskingRequest = { - controller: new AbortController(), - platform: deviceRequest.platform, - requesterId: options.requesterId, - }; - this.#asking.add(asking); - try { - const early = await this.#decisions.run(async () => - this.#checkAdmission(deviceRequest, options), - ); - if ("replay" in early) return early.replay; - // why: the caller is answered at once; the runner kills the command on its own time (it kills with SIGKILL). - if (asking.end === undefined) { - decision = await Promise.race([ - this.#askAdmission(admission, deviceRequest, options, asking.controller), - new Promise((resolve) => - asking.controller.signal.addEventListener("abort", () => resolve(undefined), { - once: true, - }), - ), - ]); - } - } finally { - this.#asking.delete(asking); - } - if (asking.end !== undefined) - this.#refuseEndedWhileAsking(asking.end, deviceRequest, options); - } - const admitted = await this.#decisions.run(async () => { - const checked = this.#checkAdmission(deviceRequest, options); - if ("replay" in checked) return checked; - const { requestId } = checked; - if (decision !== undefined && decision.kind !== "priority") { - this.#refuseByAdmission(decision, deviceRequest, options, requestId); - } - const { id, started } = await this.requests.admit( - deviceRequest, - options, - (id, onProgress) => { - const waiter = this.#queue.create( - deviceRequest, - { ...options, onProgress }, - id, - decision?.kind === "priority" ? decision.priority : 0, - ); - return { promise: this.#afterViewRead(waiter.promise), waiter }; - }, - requestId, - ); - const created = started.waiter; - this.#createdAt.set(created, this.options.clock.now()); - this.#emit("lease.requested", { - requestId: id, - requestSpec: deviceRequest, - requester: options.requesterId, - waitPolicy: options.noWait === true ? "no-wait" : "wait", - }); - return { promise: started.promise, waiter: created }; - }); - if ("replay" in admitted) return admitted.replay; - this.#admit(admitted.waiter); - return admitted.promise; - } - - #endAsking(asking: AskingRequest, end: "cancelled" | "stopping"): void { - asking.end = end; - asking.controller.abort(); - } - - /** why: replay, the requester limit, a taken id and an ended id, in this order; a refusal is thrown, a replay returned. */ - #checkAdmission( - deviceRequest: DeviceRequest, - options: LeaseRequestOptions, - ): { readonly replay: Promise } | { readonly requestId: string } { - const replay = this.requests.replay(deviceRequest, options); - if (replay !== undefined) return { replay }; - const requestId = newLeaseRequestId(this.options.idGenerator); - this.#refuseIfAlreadyLeased(deviceRequest, options.requesterId, requestId); - this.#refuseIfLeaseIdTaken(deviceRequest, options, requestId); - this.#refuseIfLeaseIdEnded(deviceRequest, options, requestId); - return { requestId }; - } - - /** why: ADR 0074 §1: the operator's command answers outside the decision lock, since it may take seconds. */ - #askAdmission( - admission: AdmissionOptions, - deviceRequest: DeviceRequest, - options: FleetRequestOptions, - controller: AbortController, - ): Promise { - return askAdmission( - admission, - { - metadata: options.admissionMetadata ?? {}, - platform: deviceRequest.platform, - principal: options.ownerId, - request: options.admissionRequest ?? deviceRequest, - requesterId: options.requesterId, - }, - this.#logger, - controller.signal, - ); - } - - /** why: the caller cancelled, or the gateway stopped, while the command ran: nothing was stored, so there is nothing to undo. */ - #refuseEndedWhileAsking( - end: "cancelled" | "stopping", - deviceRequest: DeviceRequest, - options: FleetRequestOptions, - ): never { - if (end === "stopping") throw new DispatchError("DAEMON_STOPPING", "Gateway is stopping"); - const requestId = newLeaseRequestId(this.options.idGenerator); - this.#emit("lease.rejected", { - requestId, - requester: options.requesterId, - requestSpec: deviceRequest, - reason: "cancelled", - }); - throw new RequestCancelledError(requestId); - } - - #refuseByAdmission( - decision: Exclude, - deviceRequest: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): never { - this.#emit("lease.rejected", { - requestId, - requester: options.requesterId, - requestSpec: deviceRequest, - reason: decision.kind === "deny" ? "admission-denied" : "admission-failed", - }); - const refusal = - decision.kind === "deny" - ? admissionDeniedError(decision.reason) - : admissionFailedError(decision.cause); - throw new DispatchError(refusal.code, refusal.message, refusal.details); - } - - /** - * why: the request book stores `granted` and wakes its watchers when this promise settles, and the - * worker's own `lease.granted` push starts its view read later; so the grant waits for one read of the - * granting worker's view here, and a `lease.list` right after any answer lists the lease (#1370). A - * failed read is the next tick's. - */ - async #afterViewRead(grant: Promise): Promise { - const granted = await grant; - const entry = this.options.leaseIndex.resolve(granted.lease.id); - if (entry !== undefined) { - await this.options.directory - .target(entry.workerId) - ?.refresh() - .catch((error: unknown) => { - this.#logger.debug("Failed to refresh a worker's view after granting it a lease", { - error: String(error), - workerId: entry.workerId, - }); - }); - } - return granted; - } - - #refuseIfAlreadyLeased( - deviceRequest: DeviceRequest, - requesterId: string, - requestId: string, - ): void { - const refusal = requesterLimitDecision({ - held: this.options.leaseIndex - .held(requesterId) - .filter((lease) => !this.#isPastDeadlineOnDisconnectedWorker(lease.id)), - maxPerRequester: this.options.maxPerRequester, - platform: deviceRequest.platform, - probe: false, - requesterId, - waiting: this.#queue.pendingPlatforms(requesterId), - }); - if (refusal === undefined) return; - this.#emit("lease.rejected", { - requestId, - requester: requesterId, - requestSpec: deviceRequest, - reason: "already-leased", - }); - throw refusal; - } - - /** - * why: a lease past its last reported deadline on a disconnected worker cannot have been renewed - * through this gateway, so it must not lock its requester out of the fleet. The entry stays in - * the index: it is still routed to its worker (ADR 0005 §28). - */ - #isPastDeadlineOnDisconnectedWorker(gatewayLeaseId: string): boolean { - const entry = this.options.leaseIndex.resolve(gatewayLeaseId); - if (entry === undefined) return false; - const view = this.options.views.view(entry.workerId); - if (view === undefined || view.connection !== "disconnected") return false; - const reported = view.leases?.find((lease) => lease.id === entry.workerLeaseId); - return reported !== undefined && reported.ttlDeadline <= this.options.clock.now(); - } - - /** - * why: ADR 0020: the one clash check on a gateway, after the one-lease check and in the same - * admission section. A caller-chosen id is held by one of this gateway's leases and by one of - * its own requests still open; leases a local client holds on a worker are not checked (a - * worker refuses its own, and the requester's uniqueness guarantee makes a clash unlikely). - */ - #refuseIfLeaseIdTaken( - deviceRequest: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): void { - const { leaseId } = options; - if (leaseId === undefined) return; - if ( - this.options.leaseIndex.resolve(leaseId) === undefined && - !this.requests.holdsLeaseId(leaseId) - ) { - return; - } - this.#emit("lease.rejected", { - requestId, - requester: options.requesterId, - requestSpec: deviceRequest, - reason: "lease-id-taken", - }); - throw leaseIdTaken(leaseId); - } - - /** why: ADR 0074 §4: after the taken-id check, a chosen id whose lease this gateway learned ended is refused without a dispatch; a worker answers for ends the gateway missed. */ - #refuseIfLeaseIdEnded( - deviceRequest: DeviceRequest, - options: LeaseRequestOptions, - requestId: string, - ): void { - const { leaseId } = options; - if (leaseId === undefined) return; - const endedAt = this.options.endedLeaseIds.endedAt(leaseId); - if (endedAt === undefined) return; - this.#emit("lease.rejected", { - requestId, - requester: options.requesterId, - requestSpec: deviceRequest, - reason: "lease-ended", - }); - const ended = leaseEndedError(leaseId, endedAt); - throw new DispatchError(ended.code, ended.message, ended.details); + request(deviceRequest: DeviceRequest, options: FleetRequestOptions): Promise { + return this.#admission.request(deviceRequest, options); } - /** - * why: a `new` waiter is admitted but not yet walked: no worker was asked, so it is as safe to reject as - * a `queued` one (#1492). Only `processing` means a worker already has the request. - */ - async cancelPending( + cancelPending( requesterId: string, platform?: Platform, ): Promise<"cancelled" | "not-found" | "not-cancellable"> { - for (const asking of this.#asking) { - if ( - asking.requesterId === requesterId && - (platform === undefined || asking.platform === platform) - ) { - this.#endAsking(asking, "cancelled"); - return "cancelled"; - } - } - return this.#decisions.run(async () => { - const waiter = this.#queue.findPendingWaiter(requesterId, platform); - if (waiter === undefined) return "not-found"; - if (waiter.state === "processing") return "not-cancellable"; - if (this.#queue.reject(waiter, new RequestCancelledError(waiter.id))) { - this.#emitRejected(waiter, "cancelled"); - } - return "cancelled"; - }); + return this.#admission.cancelPending(requesterId, platform); } renew(gatewayLeaseId: string, ttlMs: number | undefined): Promise { @@ -615,668 +252,12 @@ export class FleetLeaseCoordinator { * again. */ dispose(): void { this.#unsubscribeViews(); - for (const asking of this.#asking) this.#endAsking(asking, "stopping"); + this.#admission.stopAsking(); this.#queue.cancelAll(() => new DispatchError("DAEMON_STOPPING", "Gateway is stopping")); } - /** - * why: A brand-new waiter's admission. **Architecture rule 10: `noWait` is enforced in exactly one - * place.** Every admission goes through `#dispatch`'s one ordered walk, carrying this waiter - * as its `candidate` -- there is no second, out-of-order look of its own. A `noWait` waiter - * whose worker answered `NO_CAPACITY` comes back through here too, for its one more walk - * (ADR 0009 §5), so that rejection is decided here as well. - * - * There used to be two paths here: a direct `routing.select` when the queue was empty, and an - * enqueue-then-dispatch when it was not. They answered the same question and drifted, exactly - * as rule 10 describes. Four consecutive review rounds each found a fresh defect in that - * disagreement, ending with a request that answered `NO_CAPACITY` immediately or waited in the - * queue depending on whether *unrelated* requests happened to be queued at the time. - * - * Queue membership is committed only once the waiter is actually going to wait. That ordering - * is load-bearing rather than tidy: `WaitQueue#enqueue` pushes a `queued` progress frame to - * the caller and `#enqueue` emits `lease.queued`, and a `noWait` request that is about to be - * refused must produce neither. ADR §10 requires "the same codes and progress states a worker - * uses", and the worker's own `#defer` rejects a `noWait` waiter *before* it enqueues. - * - * H9 (round 2 review): throws a plain `DispatchError("NO_CAPACITY", ...)` rather than a - * fleet-native error class -- `DispatchError`'s own code is used verbatim by - * `daemon/error-code.ts#classifyError`'s first branch, so no gateway-specific class or import - * is needed there to answer the same code the worker's own `NoCapacityError` answers. Before - * this, `src/daemon` (every worker-mode daemon, not just gateway mode) imported that class - * from `src/gateway/fleet-coordinator.js` just to recognize it, pulling the whole gateway - * module graph into ordinary worker startup with no lint rule covering that direction. - */ - #admit(waiter: FleetWaiter): void { - // why: Attempted. If that attempt is refused, `#attempt` decides the rest: one more walk for a - // `noWait` caller (ADR 0009 §5). A target that turned out unreachable re-queues it. A - // candidate the walk rejected as unservable (ADR 0009 §4) is terminal, so the rejection and - // the enqueue below are no-ops on it: `WaitQueue` refuses both for a terminal waiter. - if (this.#dispatch(waiter)) return; - if (waiter.options.noWait === true) { - this.#reject( - waiter, - new DispatchError("NO_CAPACITY", "No worker in the fleet can currently serve this request"), - "no-wait", - ); - return; - } - this.#enqueue(waiter); - } - - #dispatch(candidate?: FleetWaiter): boolean { - if (this.#dispatchDepth > 0) { - // why: Round 4 review: a nested pass used to be *dropped*. The outer walk iterates a snapshot - // and has already claimed workers, so a waiter freed mid-pass would never be reconsidered - // -- C1's stall, reintroduced silently by the guard meant to prevent recursion. Defer - // instead of dropping: the outer call runs another pass once this one unwinds. - this.#passRequested = true; - return false; - } - this.#dispatchDepth += 1; - try { - const candidateAttempted = this.#dispatchPass(candidate); - while (this.#passRequested) { - this.#passRequested = false; - this.#dispatchPass(); - } - return candidateAttempted; - } finally { - this.#dispatchDepth -= 1; - this.#passRequested = false; - } - } - - /** - * why: `candidate` walks last, because it is the newest: that keeps the single fleet FIFO honest. - * The return says whether it was attempted, which `waiter.state` alone cannot tell `#admit`. - */ - #dispatchPass(candidate?: FleetWaiter): boolean { - // why: a virtual pick holds a worker; a physical pick holds a worker for the same match only. - const claimedThisPass = { physical: new Set(), virtual: new Set() }; - const attempted = new Set(); - const waiters = - candidate === undefined || this.#queue.isQueued(candidate) - ? this.#queue.list() - : [...this.#queue.list(), candidate]; - for (const waiter of waiters) { - // why: The candidate is mid-admission, not `queued`, even when it holds a queue position. - if (waiter !== candidate && waiter.state !== "queued") continue; - if (this.#walkWaiter(waiter, claimedThisPass)) attempted.add(waiter); - } - return attempted.has(candidate); - } - - #walkWaiter( - waiter: FleetWaiter, - claims: { readonly physical: Set; readonly virtual: Set }, - ): boolean { - const isPhysical = lineOf(waiter.request) === "physical"; - const claimedThisPass = isPhysical ? claims.physical : claims.virtual; - const claimKey = (workerId: string): string => - isPhysical ? `${workerId}\u0000${physicalMatchKey(waiter.request)}` : workerId; - const refusals = this.#refusals.cannotServe(waiter); - // why: A worker claimed this pass or one that answered `NO_CAPACITY` is busy, not unable, so it - // stays in the table's view. - const allViews = this.options.views.routingViews(); - const views = allViews.filter((worker) => refusals?.has(worker.id) !== true); - const request = routable(waiter); - const verdict = this.options.routing.assess(request, views); - if (verdict.kind === "reject") { - this.#rejectUnservable(waiter, verdict, refusals); - return false; - } - const reasons = this.#refusals.reasonsFor(waiter, request, allViews, { - claimed: (workerId) => claimedThisPass.has(claimKey(workerId)), - }); - this.#workerReasons.set(waiter, reasons); - const eligibleWorkers = views.filter( - (worker) => - !claimedThisPass.has(claimKey(worker.id)) && !this.#refusals.refused(waiter, worker), - ); - const decision = this.options.routing.select(request, eligibleWorkers); - if (decision === undefined) { - this.#rejectIfEveryWorkerFailedProvisioning(waiter, request, views); - return false; - } - claimedThisPass.add(claimKey(decision.workerId)); - this.#beginAttempt(waiter, decision, forwardedModel(request, eligibleWorkers, decision)); - return true; - } - - // why: a second failed provision walks on like a cannot-serve code, and only when every worker that could take the request failed so does it end with the last answer; a worker held back only for now (full, busy, not ready, unhealthy) could take it later, so it keeps the request queued. - #rejectIfEveryWorkerFailedProvisioning( - waiter: FleetWaiter, - request: RoutableRequest, - views: readonly WorkerView[], - ): void { - const refusals = this.#refusals.refusals(waiter); - if (refusals === undefined) return; - const open = this.options.routing - .explain(request, views) - .filter((entry) => entry.filter === undefined || COULD_TAKE_LATER.has(entry.filter.reason)) - .map((entry) => entry.workerId); - const failed = [...refusals] - .filter(([workerId]) => open.includes(workerId)) - .flatMap(([workerId, refusal]) => - refusal.provisionFailure === undefined - ? [] - : [{ failure: refusal.provisionFailure, workerId }], - ); - const last = failed.at(-1); - if (last === undefined || failed.length < open.length) return; - this.#reject(waiter, last.failure, "worker-failed", { - code: last.failure.code, - worker: last.workerId, - }); - } - - /** why: Ends a request the table of ADR 0009 §4 says no worker can serve, with the code and - * details a worker gives the same request. A request that holds a worker's "cannot serve" - * refusal ends with the last one instead (ADR 0009 §5). The worker only declined the dispatch - * (`lease.declined`), so the end is this gateway's own fact (ADR 0021 §4): `lease.rejected` - * with reason `unresolvable-spec`, whether or not the request had entered the gateway queue. */ - #rejectUnservable( - waiter: FleetWaiter, - verdict: Rejection, - refusals: ReadonlyMap | undefined, - ): void { - const last = refusals === undefined ? undefined : [...refusals.values()].at(-1); - if (last !== undefined) { - this.#reject(waiter, last, "unresolvable-spec"); - return; - } - this.#reject( - waiter, - new DispatchError(verdict.code, verdict.message, verdict.details), - verdict.reason, - ); - } - - /** - * why: H2 (round 2 review): `markProcessing`'s own `false` return -- the waiter is already terminal - * -- used to be ignored, issuing the RPC anyway. Masked in practice by every caller - * (`#admit`/`#dispatch`) only ever reaching this with a live waiter today, but nothing enforced - * that: a grant landing for a waiter that settled a moment earlier would still be indexed by - * `#settleGrant` while `queue.resolve` quietly answers `false`, leaving an orphan lease no - * client holds a reference to release. - */ - #beginAttempt(waiter: FleetWaiter, decision: RoutingDecision, rename: ModelName): void { - if (!this.#queue.markProcessing(waiter)) return; - void this.#attempt(waiter, decision, rename); - } - - /** - * why: Every exit from this method leaves `waiter` either terminal (`resolve`/`reject`, inside - * `#settleGrant`/the catch below), back in `queued` (`#staleView`'s `#enqueue`), or -- a - * `noWait` waiter refused with `NO_CAPACITY`, or any waiter refused with a cannot-serve code - * before progress (ADR 0009 §5) -- handed back to `#admit`, which leaves it - * terminal or `processing` in a new attempt that these same rules govern -- never - * stuck `processing` with nothing left to drive it, and never in two places disagreeing about - * which. C1 (round 2 review): the first early return used to skip straight to `#staleView` - * without first clearing a separate `#dispatchTargets` mark this method used to set -- the mark - * survived the re-queue, and `#dispatch`'s guard on it then skipped this waiter forever. Since - * `queue.markProcessing`/`#enqueue` (`WaitQueue`'s own state, not a second map this class kept - * beside it) is now the *only* bookkeeping a waiter's in-flight-ness lives in, every branch - * below already leaves it in one of exactly those states on its own -- there is no second - * clear-up step left to forget. - * - * C1 (round 3 review): the two branches that leave `waiter` *terminal* -- `#settleGrant`, and - * the catch block's own `queue.reject` -- also each call `#dispatch()` themselves, right after. - * `#staleView`'s own branches (the unreachable-target check above, the catch's immediate - * `NO_CAPACITY`, and the mismatched-grant retry) do not need to: all already call a worker's `refresh()`, and a real snapshot - * landing for it fires `#onViewsChanged` -> `#dispatch()`. That pass offers the waiter to every - * other worker; it offers it to a worker that answered `NO_CAPACITY` only once that worker's - * view has changed (ADR 0009 §5), so a worker that keeps refusing is asked once per change of - * its state, not on every refresh. A `noWait` waiter refused with `NO_CAPACITY` is not - * re-queued: it gets one more walk here, and is rejected if no worker is picked in it. The - * terminal branches have no such follow-up of their own -- a worker this - * class gave up on (`WORKER_UNREACHABLE`, `INTERNAL`) or was refused for a reason genuinely - * unrelated to capacity (any other worker-side code) may still be exactly the worker a different - * queued waiter needs, and nothing else would ever schedule that second look. - */ - // fallow-ignore-next-line complexity -- one attempt, every exit of which is named in the doc comment above. - async #attempt(waiter: FleetWaiter, decision: RoutingDecision, rename: ModelName): Promise { - const workerId = decision.workerId; - const client = liveClient(this.options.directory.target(workerId)); - if (client === undefined) { - // why: The view named a worker that is no longer reachable by the time we got here -- the same - // "stale, not wrong" story as a worker's own `NO_CAPACITY`, just discovered a step earlier. - this.#staleView(waiter, workerId, { unreachable: true }); - return; - } - - const namespacedRequesterId = `${this.options.leaseIndex.requesterPrefix}${waiter.options.requesterId}`; - let announced = false; - const announceDispatched = (): void => { - if (announced) return; - announced = true; - const createdAt = this.#createdAt.get(waiter); - this.#emit("request.dispatched", { - ...(waiter.request.model === undefined ? {} : { model: waiter.request.model }), - ...(waiter.request.class === undefined ? {} : { class: waiter.request.class }), - ...(waiter.request.mode === undefined ? {} : { mode: waiter.request.mode }), - ...(lineOf(waiter.request) === "physical" ? { physical: true as const } : {}), - platform: waiter.request.platform, - queuedMs: createdAt === undefined ? 0 : Math.max(0, this.options.clock.now() - createdAt), - reason: decision.reason, - requestId: waiter.id, - requesterId: waiter.options.requesterId, - stage: decision.stage, - workerId, - }); - }; - - // why: P4 (round 3 review): H4's own reasoning applies verbatim here -- - // `FleetForwarding.withLeaseRequestTimeout` settling `WORKER_UNREACHABLE` only stops *this* - // call's returned promise from being awaited - // any further, not the worker's own still-pending `lease.request` RPC, whose `onProgress` - // closure below stays live for as long as the worker keeps pushing progress. Before this, - // a late `progress` push after the timeout had already rejected the waiter still called - // `announceDispatched()` and emitted `request.dispatched` for a request this gateway had - // already told its caller was `WORKER_UNREACHABLE` -- a false past-tense fact against - // `docs/internal/EVENTS.md`'s own "and the worker took it". Same mechanism as `exec`'s `detached`, not - // a second bespoke one: flipped by `FleetForwarding.withLeaseRequestTimeout`'s own `onTimeout` - // hook, checked here before either `announceDispatched()` or `notifyProgress` run. - let detached = false; - let grant: LeaseGrant; - try { - grant = await this.#forwarding.withLeaseRequestTimeout( - client.requestLease( - { - ...requestedDevice({ ...waiter.request, ...rename }), - requesterId: namespacedRequesterId, - // why: ADR §27a (narrowed, round 3 review, H3): only the worker's own gateway-uplink - // session may set `owner` -- and this RPC always travels over exactly that - // connection (`WorkerLink#connect`, via `acceptUplink` on the worker's own end, see - // `session.isGatewayUplink`'s doc). The worker stores this verbatim as the lease's - // `ownerId` instead of deriving it from the connection -- see - // `daemon/dispatcher.ts`'s `#leaseRequest`. - owner: waiter.options.ownerId, - // why: ADR 0009 §3: a download is never triggered through a gateway. - allowDownload: false, - // why: ADR §12: worker queues never hold gateway traffic. Every dispatch is `noWait` - // regardless of what the original caller asked the gateway for -- the *gateway's - // own* queue is where a "wait" request actually waits. - noWait: true, - ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), - // why: ADR 0020: sent as the requester sent it. The worker grants exactly this id, and - // its refusal (`LEASE_ID_TAKEN`) is the caller's answer: no other worker is tried. - ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), - // why: ADR 0021: this gateway's id for the request marks the dispatch a probe. The worker - // declines a probe it refuses (`lease.declined`) rather than rejecting it, and names - // the id on the lease events of the dispatch. - fleetRequestId: waiter.id, - ...(waiter.options.requires === undefined - ? {} - : { requires: [...waiter.options.requires] }), - }, - { - onProgress: (progress) => { - if (detached) return; - // why: ADR §11: "the first `progress` push" is one of the two signals a request is - // this worker's now -- device work having started means it is committed here even - // before the grant itself arrives. - announceDispatched(); - this.#queue.notifyProgress(waiter, progress); - }, - }, - ), - workerId, - () => { - detached = true; - }, - ); - // why: the grant crossed a wire; a worker must not be able to crash the gateway with an answer without a lease, so it fails this attempt like any other worker failure. - if (typeof (grant as Partial | null)?.lease?.id !== "string") { - throw new DispatchError( - "INTERNAL", - `Worker ${workerId} answered lease.request with a malformed grant`, - { workerId }, - ); - } - } catch (error: unknown) { - // why: P1 (round 2 review): "an *immediate* NO_CAPACITY is the only answer that leaves it - // queued ... the request waits" (§11) -- `announced` is exactly the "work has begun" - // distinction the ADR draws, already maintained above for `request.dispatched`. A worker - // can answer `NO_CAPACITY` *after* pushing progress (its own `#evictManaged` failure path, - // reached from a `noWait` waiter's `#defer`, runs after `provisioning`/`reclaiming`), and - // that is this request's own terminal failure -- not a stale view to retry, because device - // work already started means the request was this worker's (§11's own wording). Re-queuing - // it anyway silently reversed a dispatch the caller had already been told about. - if (isSimlockError(error) && STALE_VIEW_CODES.has(error.code) && !announced) { - this.#refusals.rememberRefusal(waiter, workerId, { - capacityReason: refusedCapacityReason(error.details), - }); - if (waiter.options.noWait === true) { - this.#refreshView(workerId); - // why: ADR 0009 §5: one more walk, with this worker left out until its view changes. A - // `noWait` caller never asked to wait for that change, so `#admit` rejects it if no - // worker is picked. - this.#admit(waiter); - return; - } - this.#staleView(waiter, workerId); - return; - } - // why: a provision failure before the worker's progress push did no device work, so another worker may serve it and the walk goes on without this one; after a progress push it falls through to the terminal branch below, as today. - if (isSimlockError(error) && error.code === "PROVISION_FAILED" && !announced) { - this.#refusals.rememberRefusal(waiter, workerId, { - provisionFailure: classifyRelayedError(error, workerId), - }); - this.#refreshView(workerId); - this.#admit(waiter); - return; - } - if (isSimlockError(error) && CANNOT_SERVE_CODES.has(error.code) && !announced) { - this.#refusals.rememberCannotServe(waiter, workerId, classifyRelayedError(error, workerId)); - this.#refreshView(workerId, { includeCatalog: true }); - this.#admit(waiter); - return; - } - // why: C1 (round 3 review): this waiter's own outcome is terminal, but the worker it just gave - // up (or never actually reached) may still be free for whoever else is queued behind it -- - // and unlike `#staleView`'s branch above, nothing else here schedules another look at them. - // `WORKER_UNREACHABLE`/`INTERNAL` in particular need not produce any worker-side event, so - // without this a waiter passed over in this same pass had nothing left to wake it until the - // next real worker-view change (the 30s refresh tick, at best). - // ADR 0021 §4: this is the request's end, and the worker emits no `lease.rejected` for a - // probe (at most a `lease.declined`; for an unreachable worker or a gateway-side `INTERNAL` - // there is usually none, and a dispatch timeout only stops this gateway waiting: the - // worker's `lease.request` keeps running and can still end in a relayed `lease.declined` - // or a grant), so this gateway records it, with the code the caller got and the worker it - // failed on. - const failure = this.#classifyLeaseRequestError(error, workerId); - this.#reject(waiter, failure, "worker-failed", { code: failure.code, worker: workerId }); - this.#dispatch(); - return; - } - - announceDispatched(); - if (this.#settleGrant(waiter, workerId, grant) === "retry") { - // why: The worker answered with an id the gateway did not ask for: the attempt failed, and the - // request goes back to the queue as after an unreachable worker. This worker is left out - // until its view changes, so a worker that keeps answering so is not asked in a loop. - this.#refusals.rememberRefusal(waiter, workerId); - this.#staleView(waiter, workerId); - return; - } - // why: C1 (round 3 review): see the catch branch above -- a grant settling this waiter is just as - // much a reason for whoever else is queued to get another look, not only a terminal failure. - this.#dispatch(); - } - - /** - * why: a terminal failure past `#attempt`'s `NO_CAPACITY`/stale-view check. A worker never - * rejects a probe: where it answered, it only declined (`lease.declined`, relayed onto this bus with - * `workerId` added by `WorkerLink`), and where it did not (`WORKER_UNREACHABLE`, a gateway-side - * `INTERNAL`) it usually emitted nothing; after a dispatch timeout the worker's request is - * still running and may yet end in a relayed `lease.declined` or a grant, since the timeout - * only stops this gateway waiting. Either way the request's end is this - * gateway's own `lease.rejected` (ADR 0021 §4). The error code is - * preserved so the caller sees what the worker actually said, with three exceptions: - * - * - a transport failure (the uplink itself, not the worker's own answer) -- ADR §28/§29 name - * `WORKER_UNREACHABLE` for that, not whatever the client's own connection loss happens to be - * called (`daemon/dispatch.js`'s `SimlockError.kind` is what tells the two apart); - * - `FleetForwarding.withLeaseRequestTimeout`'s own `DispatchError` (P2, round 2 review) -- - * never a `SimlockError`, since it is raised here, by this gateway's own timer, not by the worker's - * answer (the request itself did reach the worker and keeps running there) -- forwarded as-is; - * - anything else that is neither of those (H1, round 2 review): not a fact about the worker, - * since every real transport failure already arrives as a `kind: "transport"` `SimlockError`, - * so this is a bug in this coordinator's own request-building code. `WORKER_UNREACHABLE` - * used to answer this case too, misreporting a gateway-side crash as "the machine is - * unreachable"; `INTERNAL` is what `FleetForwarding#forwardToWorker`'s matching branch - * answers for the same shape of failure. - */ - #classifyLeaseRequestError(error: unknown, workerId: string): DispatchError { - if (error instanceof DispatchError) return error; - if (isSimlockError(error)) return classifyRelayedError(error, workerId); - return new DispatchError( - "INTERNAL", - `Unexpected error forwarding lease.request to worker ${workerId}`, - { workerId }, - ); - } - - /** why: ADR §11: "an immediate `NO_CAPACITY` is the only answer that leaves it queued ... the - * request waits", because this is a stale view, not a real refusal. A waiter reaches - * this from a target that turned out unreachable, and from a grant whose id was not the one - * asked for (`noWait` included, in both); a `noWait` caller refused with `NO_CAPACITY` is - * settled in `#attempt` instead (ADR 0009 §5). */ - #staleView( - waiter: FleetWaiter, - workerId: string, - options: { readonly unreachable?: boolean } = {}, - ): void { - // why: the walk that picked this worker ran before its answer; the entries must say what is true now. - this.#workerReasons.set( - waiter, - this.#refusals.reasonsFor( - waiter, - routable(waiter), - this.options.views.routingViews(), - options.unreachable === true ? { unreachable: workerId } : {}, - ), - ); - this.#enqueue(waiter); - this.#refreshView(workerId); - } - - #refreshView(workerId: string, options?: { readonly includeCatalog: true }): void { - const target = this.options.directory.target(workerId); - void target?.refresh(options).catch((error: unknown) => { - this.#logger.debug("Failed to refresh a worker's view after a worker's refusal", { - workerId, - message: error instanceof Error ? error.message : String(error), - }); - }); - } - - /** - * why: H6 (round 2 review): the entry's `ownerId` comes from `waiter.options.ownerId` -- what this - * gateway itself forwarded as `owner` in the `lease.request` a moment ago -- never from the - * worker's own echo of it (`grant.lease.ownerId`). Everything this gateway authorizes - * afterwards (`lease.renew`/`release`/`device.exec`) keys on the index's `ownerId`, so a worker - * that ignored or rewrote the field would otherwise redefine ownership at the gateway; the - * rebuild path (`FleetLeaseIndex#rebuildFromWorker`) has no better source and must trust the - * worker, but this path already knows the answer. A mismatched echo is logged -- it means - * either a worker bug or something worth knowing about, never silently swallowed. - */ - #settleGrant(waiter: FleetWaiter, workerId: string, grant: LeaseGrant): "settled" | "retry" { - // why: ADR 0020: a forwarded `leaseId` is the gateway lease id, bare, and never the worker's echo - // of it -- a worker that granted something else is not believed, and its lease is given back. - const forwardedLeaseId = waiter.options.leaseId; - if (forwardedLeaseId !== undefined && grant.lease.id !== forwardedLeaseId) { - this.#logger.warn( - "Worker granted a lease ID different from the leaseId the gateway forwarded", - { - forwardedLeaseId, - grantedLeaseId: grant.lease.id, - workerId, - }, - ); - this.#forwarding.releaseOnWorker(workerId, grant.lease.id); - return "retry"; - } - const gatewayLeaseId = forwardedLeaseId ?? `${workerId}.${grant.lease.id}`; - if (grant.lease.ownerId !== waiter.options.ownerId) { - this.#logger.warn("Worker echoed an ownerId different from the one the gateway forwarded", { - echoedOwnerId: grant.lease.ownerId, - forwardedOwnerId: waiter.options.ownerId, - gatewayLeaseId, - workerId, - }); - } - const entry: FleetLeaseEntry = { - gatewayLeaseId, - grantedAt: grant.lease.grantedAt, - idChosenByRequester: forwardedLeaseId !== undefined, - ownerId: waiter.options.ownerId, - requesterId: waiter.options.requesterId, - workerId, - workerLeaseId: grant.lease.id, - }; - if (!this.options.leaseIndex.add(entry, waiter.request.platform)) { - // why: The index already routes this bare id to another worker, which got it there first: the - // new worker's lease is given back and the caller is told the id is taken. - this.#forwarding.releaseOnWorker(workerId, grant.lease.id); - this.#reject(waiter, leaseIdTaken(gatewayLeaseId), "lease-id-taken"); - return "settled"; - } - const fleetGrant: FleetLeaseGrant = { - device: grant.device, - environment: grant.environment, - lease: this.#forwarding.projectRecord(grant.lease, entry), - timing: grant.timing, - }; - // why: ADR 0021 §4: the grant is the request's end only if the waiter was still open. One that - // landed after a timeout, a cancel or `dispose()` handed nothing to a caller, so it is no fact. - if (this.#queue.resolve(waiter, fleetGrant)) { - this.#emit("request.granted", { - leaseId: gatewayLeaseId, - requestId: waiter.id, - worker: workerId, - workerLeaseId: grant.lease.id, - }); - } - return "settled"; - } - - #enqueue(waiter: FleetWaiter): void { - const alreadyQueued = this.#queue.isQueued(waiter); - if (this.#queue.enqueue(waiter) && !alreadyQueued) { - this.#emit("lease.queued", { - queuePosition: this.#queue.positionOf(waiter), - requestId: waiter.id, - }); - } - } - - #reject( - waiter: FleetWaiter, - error: Error, - reason: - | Rejection["reason"] - | "no-wait" - | "cancelled" - | "timeout" - | "lease-id-taken" - | "worker-failed", - failedOn?: { readonly code: string; readonly worker: string }, - ): void { - if (this.#queue.reject(waiter, error)) { - this.#emitRejected(waiter, reason, failedOn); - } - } - - #emitRejected( - waiter: FleetWaiter, - reason: EventMap["lease.rejected"]["reason"], - failedOn?: { readonly code: string; readonly worker: string }, - ): void { - this.#emit("lease.rejected", { - requestId: waiter.id, - requester: waiter.options.requesterId, - requestSpec: waiter.request, - reason, - // why: `worker`, never `workerId`: that key marks an event relayed from a worker (ADR 0014 §6). - ...(failedOn === undefined ? {} : { code: failedOn.code, worker: failedOn.worker }), - }); - } - - /** - * why: §30's reconnect rebuild, and its own upkeep on every worker-view change: reconciles each - * worker's entries against its current view -- adding any gateway-issued lease the index does - * not already know and, a generation later, forgetting one the view has stopped reporting at - * all (see `FleetLeaseIndex#rebuildFromWorker`'s own doc comment, C3 round 2 review) -- forgets - * a worker's entries entirely once its view has disappeared (`worker.remove`, or retention), - * then runs a dispatch pass. - * - * C1 (round 2 review): `rebuildFromWorker` runs *only* for a worker whose `view.leases` - * reference actually changed since the last time this method reconciled it -- see - * `#lastReconciledLeases`'s own doc. Before this fix, every view was reconciled on *every* - * notification regardless of which worker it was about: two view changes on worker B alone - * bumped worker A's reconciliation generation twice against A's own byte-identical, stale - * cached snapshot, which is enough for `rebuildFromWorker`'s two-consecutive-misses rule to - * evict a lease on A that was never actually missing from a real snapshot at all -- silently - * violating ADR §14 (the fleet-wide one-lease check reads this same index) and losing the - * `lease-lost` push `GatewayOwnerRoutedFacts` would otherwise emit for it. - */ #onViewsChanged(): void { - const views = this.options.views.views(); - const currentIds = new Set(views.map((view) => view.id)); - for (const view of views) { - // why: A starting worker's view has no `leases`: nothing was read, so the index keeps what it - // holds for that worker until the worker answers `running`. - if (view.leases === undefined) continue; - if (this.#lastReconciledLeases.get(view.id) === view.leases) continue; - this.options.leaseIndex.rebuildFromWorker(view.id, view.leases, view.devices); - this.#lastReconciledLeases.set(view.id, view.leases); - } - for (const workerId of this.#knownWorkerIds) { - if (!currentIds.has(workerId)) { - this.options.leaseIndex.forgetWorker(workerId); - this.#lastReconciledLeases.delete(workerId); - } - } - this.#knownWorkerIds = currentIds; - this.#dispatch(); - } - - #emit(event: Event, payload: EventMap[Event]): void { - this.options.eventBus.emit(event, payload, "fleet-lease-coordinator"); + this.#upkeep.reconcile(); + this.#dispatch.dispatch(); } } - -/** why: A class request has no model to send: the worker applies its own rules to the class. */ -function forwardedModel( - request: RoutableRequest, - views: readonly WorkerView[], - decision: RoutingDecision, -): ModelName { - if (request.model === undefined || request.physical === true) return {}; - const view = views.find((worker) => worker.id === decision.workerId); - return { model: (view === undefined ? undefined : matchRequest(view, request)) ?? request.model }; -} - -function leaseIdTaken(leaseId: string): DispatchError { - return new DispatchError("LEASE_ID_TAKEN", `lease ID ${leaseId} is already in use`, { leaseId }); -} - -interface ModelName { - readonly model?: string; -} - -// why: keyed as the core's `fitsPhysical` compares: platform, model in any letter case or else the class (`phone` when absent), and the OS text. -function physicalMatchKey(request: FleetWaiter["request"]): string { - const target = - request.model === undefined - ? `class:${requestedClass({ class: request.class })}` - : `model:${request.model.toLowerCase()}`; - return [request.platform, target, request.osVersion ?? ""].join("\u0000"); -} - -// why: a request field that is absent stays absent in the routable request (never `undefined`), as `exactOptionalPropertyTypes` needs. -function present(fields: T): { [K in keyof T]?: Exclude } { - return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined)) as { - [K in keyof T]?: Exclude; - }; -} - -function routable(waiter: FleetWaiter): RoutableRequest { - const { request, options } = waiter; - return { - platform: request.platform as Platform, - ...present({ - model: request.model, - class: request.class, - osVersion: request.osVersion, - imageTag: request.imageTag, - baseline: request.baseline, - mode: request.mode, - requires: options.requires, - except: request.except, - }), - ...(lineOf(request) === "physical" ? { physical: true as const } : {}), - }; -} diff --git a/src/gateway/fleet-dispatch.ts b/src/gateway/fleet-dispatch.ts new file mode 100644 index 000000000..9301f124f --- /dev/null +++ b/src/gateway/fleet-dispatch.ts @@ -0,0 +1,556 @@ +import type { LeaseGrant } from "../admin/index.js"; +import { isSimlockError, requestedDevice } from "../contract/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import type { Clock, Logger } from "../ports/index.js"; +import type { FleetFacts, FleetOutcomes } from "./fleet-facts.js"; +import type { FleetGrants } from "./fleet-grant.js"; +import { classifyRelayedError, type FleetForwarding } from "./fleet-forwarding.js"; +import { liveClient, type FleetViews, type WorkerDirectory } from "./fleet-ports.js"; +import { FleetRefusals, refusedCapacityReason, type WorkerEntry } from "./fleet-refusals.js"; +import { + forwardedModel, + physicalMatchKey, + routable, + type ModelName, +} from "./fleet-request-shape.js"; +import type { FleetLeaseIndex } from "./lease-index.js"; +import { lineOf, type FleetQueue, type FleetWaiter } from "./queue.js"; +import type { RoutableRequest, RoutingDecision, RoutingPolicy } from "./routing.js"; +import type { Rejection, WorkerReason } from "./routing/index.js"; +import type { WorkerView } from "./worker-registry.js"; + +/** + * ## The dispatch race (see the module's own tests for the one that catches a naive fix) + * + * Dispatch re-runs on every worker-view change (`FleetViews#onViewsChanged`) and once more per + * settled attempt. (C1/C2, round 3 review: `#attempt` now calls `#dispatch()` itself, right after + * `#settleGrant` and right after its own terminal `reject` -- see `#attempt`'s own doc for why + * this was previously false. `#admit` has no look of its own at all: it hands the + * brand-new waiter to that same walk as its `candidate` -- see `#admit`'s own doc, and + * architecture rule 10.) A waiter whose forwarded `lease.request` is still in + * flight to worker A must never be picked up by a second pass and sent to worker B too -- if both granted, one requester + * would hold two devices on two machines, and the fleet-wide admission check cannot catch it + * (it runs once, at admission, before either RPC). `#beginAttempt` calls `queue.markProcessing` + * *before* issuing the RPC, taking the waiter out of `queued` state, so `#dispatch`'s own + * `queue.list()` scan skips it for the whole in-flight window; the waiter only becomes `queued` + * again (via `#enqueue`, from `#staleView`) once that attempt has fully settled, or is rejected. There used to be + * a second guard here too -- a `Map` keyed by in-flight target -- but once + * every exit from `#attempt` clears its waiter's processing state in exactly one place (the + * `finally` below), the two checks can never disagree: nothing can observe a waiter that is both + * `queued` and mid-attempt. Keeping a guard with no state it can ever catch was worse than + * deleting it (round 2 review, C1/C2) -- if a future change reintroduces that race, the fix is a + * new guard proven by a new failing test, not resurrecting one that only ever watched itself. + */ + +// why: the filters that hold a worker back only for now (it is full, busy, not ready or unhealthy), so it could take the request later; any other reason (drained, disconnected, a missing driver, runtime, model or capability) means it will not take it by waiting. +const COULD_TAKE_LATER: ReadonlySet = new Set([ + "busy", + "full", + "not-ready", + "unhealthy", + "updating", +]); + +// why: ADR 0042 §3: a worker that began its update after the gateway's last read refuses with `DAEMON_UPDATING`, and its view is as stale as after `NO_CAPACITY`. +const STALE_VIEW_CODES: ReadonlySet = new Set(["NO_CAPACITY", "DAEMON_UPDATING"]); + +const CANNOT_SERVE_CODES: ReadonlySet = new Set([ + "UNKNOWN_MODEL", + "RUNTIME_MISSING", + "NO_DRIVER", + "CAPABILITY_MISSING", + "UNKNOWN_BASELINE", + "BASELINE_MISMATCH", + "BASELINE_STALE", +]); + +export interface FleetDispatchOptions { + readonly clock: Clock; + readonly directory: WorkerDirectory; + readonly views: FleetViews; + readonly routing: RoutingPolicy; + readonly leaseIndex: FleetLeaseIndex; + readonly logger: Logger; + readonly queue: FleetQueue; + readonly forwarding: FleetForwarding; + readonly facts: FleetFacts; + readonly outcomes: FleetOutcomes; + readonly grants: FleetGrants; +} + +export class FleetDispatch { + /** why: A `WeakMap` so a waiter that never dispatches (rejected, cancelled, timed out) leaks no entry. */ + readonly #createdAt = new WeakMap(); + readonly #workerReasons = new WeakMap(); + readonly #refusals: FleetRefusals; + #dispatchDepth = 0; + #passRequested = false; + + constructor(private readonly options: FleetDispatchOptions) { + this.#refusals = new FleetRefusals({ routing: options.routing, views: options.views }); + } + + /** + * why: A brand-new waiter's admission. **Architecture rule 10: `noWait` is enforced in exactly one + * place.** Every admission goes through `#dispatch`'s one ordered walk, carrying this waiter + * as its `candidate` -- there is no second, out-of-order look of its own. A `noWait` waiter + * whose worker answered `NO_CAPACITY` comes back through here too, for its one more walk + * (ADR 0009 §5), so that rejection is decided here as well. + * + * There used to be two paths here: a direct `routing.select` when the queue was empty, and an + * enqueue-then-dispatch when it was not. They answered the same question and drifted, exactly + * as rule 10 describes. Four consecutive review rounds each found a fresh defect in that + * disagreement, ending with a request that answered `NO_CAPACITY` immediately or waited in the + * queue depending on whether *unrelated* requests happened to be queued at the time. + * + * Queue membership is committed only once the waiter is actually going to wait. That ordering + * is load-bearing rather than tidy: `WaitQueue#enqueue` pushes a `queued` progress frame to + * the caller and `#enqueue` emits `lease.queued`, and a `noWait` request that is about to be + * refused must produce neither. ADR §10 requires "the same codes and progress states a worker + * uses", and the worker's own `#defer` rejects a `noWait` waiter *before* it enqueues. + * + * H9 (round 2 review): throws a plain `DispatchError("NO_CAPACITY", ...)` rather than a + * fleet-native error class -- `DispatchError`'s own code is used verbatim by + * `daemon/error-code.ts#classifyError`'s first branch, so no gateway-specific class or import + * is needed there to answer the same code the worker's own `NoCapacityError` answers. Before + * this, `src/daemon` (every worker-mode daemon, not just gateway mode) imported that class + * from `src/gateway/fleet-coordinator.js` just to recognize it, pulling the whole gateway + * module graph into ordinary worker startup with no lint rule covering that direction. + */ + admit(waiter: FleetWaiter): void { + // why: Attempted. If that attempt is refused, `#attempt` decides the rest: one more walk for a + // `noWait` caller (ADR 0009 §5). A target that turned out unreachable re-queues it. A + // candidate the walk rejected as unservable (ADR 0009 §4) is terminal, so the rejection and + // the enqueue below are no-ops on it: `WaitQueue` refuses both for a terminal waiter. + if (this.dispatch(waiter)) return; + if (waiter.options.noWait === true) { + this.options.outcomes.reject( + waiter, + new DispatchError("NO_CAPACITY", "No worker in the fleet can currently serve this request"), + "no-wait", + ); + return; + } + this.options.outcomes.enqueue(waiter); + } + + dispatch(candidate?: FleetWaiter): boolean { + if (this.#dispatchDepth > 0) { + // why: Round 4 review: a nested pass used to be *dropped*. The outer walk iterates a snapshot + // and has already claimed workers, so a waiter freed mid-pass would never be reconsidered + // -- C1's stall, reintroduced silently by the guard meant to prevent recursion. Defer + // instead of dropping: the outer call runs another pass once this one unwinds. + this.#passRequested = true; + return false; + } + this.#dispatchDepth += 1; + try { + const candidateAttempted = this.#dispatchPass(candidate); + while (this.#passRequested) { + this.#passRequested = false; + this.#dispatchPass(); + } + return candidateAttempted; + } finally { + this.#dispatchDepth -= 1; + this.#passRequested = false; + } + } + + noteCreated(waiter: FleetWaiter): void { + this.#createdAt.set(waiter, this.options.clock.now()); + } + + workerReasons(waiter: FleetWaiter): readonly WorkerEntry[] | undefined { + return this.#workerReasons.get(waiter); + } + + /** + * why: `candidate` walks last, because it is the newest: that keeps the single fleet FIFO honest. + * The return says whether it was attempted, which `waiter.state` alone cannot tell `#admit`. + */ + #dispatchPass(candidate?: FleetWaiter): boolean { + // why: a virtual pick holds a worker; a physical pick holds a worker for the same match only. + const claimedThisPass = { physical: new Set(), virtual: new Set() }; + const attempted = new Set(); + const waiters = + candidate === undefined || this.options.queue.isQueued(candidate) + ? this.options.queue.list() + : [...this.options.queue.list(), candidate]; + for (const waiter of waiters) { + // why: The candidate is mid-admission, not `queued`, even when it holds a queue position. + if (waiter !== candidate && waiter.state !== "queued") continue; + if (this.#walkWaiter(waiter, claimedThisPass)) attempted.add(waiter); + } + return attempted.has(candidate); + } + + #walkWaiter( + waiter: FleetWaiter, + claims: { readonly physical: Set; readonly virtual: Set }, + ): boolean { + const isPhysical = lineOf(waiter.request) === "physical"; + const claimedThisPass = isPhysical ? claims.physical : claims.virtual; + const claimKey = (workerId: string): string => + isPhysical ? `${workerId}\u0000${physicalMatchKey(waiter.request)}` : workerId; + const refusals = this.#refusals.cannotServe(waiter); + // why: A worker claimed this pass or one that answered `NO_CAPACITY` is busy, not unable, so it + // stays in the table's view. + const allViews = this.options.views.routingViews(); + const views = allViews.filter((worker) => refusals?.has(worker.id) !== true); + const request = routable(waiter); + const verdict = this.options.routing.assess(request, views); + if (verdict.kind === "reject") { + this.#rejectUnservable(waiter, verdict, refusals); + return false; + } + const reasons = this.#refusals.reasonsFor(waiter, request, allViews, { + claimed: (workerId) => claimedThisPass.has(claimKey(workerId)), + }); + this.#workerReasons.set(waiter, reasons); + const eligibleWorkers = views.filter( + (worker) => + !claimedThisPass.has(claimKey(worker.id)) && !this.#refusals.refused(waiter, worker), + ); + const decision = this.options.routing.select(request, eligibleWorkers); + if (decision === undefined) { + this.#rejectIfEveryWorkerFailedProvisioning(waiter, request, views); + return false; + } + claimedThisPass.add(claimKey(decision.workerId)); + this.#beginAttempt(waiter, decision, forwardedModel(request, eligibleWorkers, decision)); + return true; + } + + // why: a second failed provision walks on like a cannot-serve code, and only when every worker that could take the request failed so does it end with the last answer; a worker held back only for now (full, busy, not ready, unhealthy) could take it later, so it keeps the request queued. + #rejectIfEveryWorkerFailedProvisioning( + waiter: FleetWaiter, + request: RoutableRequest, + views: readonly WorkerView[], + ): void { + const refusals = this.#refusals.refusals(waiter); + if (refusals === undefined) return; + const open = this.options.routing + .explain(request, views) + .filter((entry) => entry.filter === undefined || COULD_TAKE_LATER.has(entry.filter.reason)) + .map((entry) => entry.workerId); + const failed = [...refusals] + .filter(([workerId]) => open.includes(workerId)) + .flatMap(([workerId, refusal]) => + refusal.provisionFailure === undefined + ? [] + : [{ failure: refusal.provisionFailure, workerId }], + ); + const last = failed.at(-1); + if (last === undefined || failed.length < open.length) return; + this.options.outcomes.reject(waiter, last.failure, "worker-failed", { + code: last.failure.code, + worker: last.workerId, + }); + } + + /** why: Ends a request the table of ADR 0009 §4 says no worker can serve, with the code and + * details a worker gives the same request. A request that holds a worker's "cannot serve" + * refusal ends with the last one instead (ADR 0009 §5). The worker only declined the dispatch + * (`lease.declined`), so the end is this gateway's own fact (ADR 0021 §4): `lease.rejected` + * with reason `unresolvable-spec`, whether or not the request had entered the gateway queue. */ + #rejectUnservable( + waiter: FleetWaiter, + verdict: Rejection, + refusals: ReadonlyMap | undefined, + ): void { + const last = refusals === undefined ? undefined : [...refusals.values()].at(-1); + if (last !== undefined) { + this.options.outcomes.reject(waiter, last, "unresolvable-spec"); + return; + } + this.options.outcomes.reject( + waiter, + new DispatchError(verdict.code, verdict.message, verdict.details), + verdict.reason, + ); + } + + /** + * why: H2 (round 2 review): `markProcessing`'s own `false` return -- the waiter is already terminal + * -- used to be ignored, issuing the RPC anyway. Masked in practice by every caller + * (`#admit`/`#dispatch`) only ever reaching this with a live waiter today, but nothing enforced + * that: a grant landing for a waiter that settled a moment earlier would still be indexed by + * `#settleGrant` while `queue.resolve` quietly answers `false`, leaving an orphan lease no + * client holds a reference to release. + */ + #beginAttempt(waiter: FleetWaiter, decision: RoutingDecision, rename: ModelName): void { + if (!this.options.queue.markProcessing(waiter)) return; + void this.#attempt(waiter, decision, rename); + } + + /** + * why: Every exit from this method leaves `waiter` either terminal (`resolve`/`reject`, inside + * `#settleGrant`/the catch below), back in `queued` (`#staleView`'s `#enqueue`), or -- a + * `noWait` waiter refused with `NO_CAPACITY`, or any waiter refused with a cannot-serve code + * before progress (ADR 0009 §5) -- handed back to `#admit`, which leaves it + * terminal or `processing` in a new attempt that these same rules govern -- never + * stuck `processing` with nothing left to drive it, and never in two places disagreeing about + * which. C1 (round 2 review): the first early return used to skip straight to `#staleView` + * without first clearing a separate `#dispatchTargets` mark this method used to set -- the mark + * survived the re-queue, and `#dispatch`'s guard on it then skipped this waiter forever. Since + * `queue.markProcessing`/`#enqueue` (`WaitQueue`'s own state, not a second map this class kept + * beside it) is now the *only* bookkeeping a waiter's in-flight-ness lives in, every branch + * below already leaves it in one of exactly those states on its own -- there is no second + * clear-up step left to forget. + * + * C1 (round 3 review): the two branches that leave `waiter` *terminal* -- `#settleGrant`, and + * the catch block's own `queue.reject` -- also each call `#dispatch()` themselves, right after. + * `#staleView`'s own branches (the unreachable-target check above, the catch's immediate + * `NO_CAPACITY`, and the mismatched-grant retry) do not need to: all already call a worker's `refresh()`, and a real snapshot + * landing for it fires `#onViewsChanged` -> `#dispatch()`. That pass offers the waiter to every + * other worker; it offers it to a worker that answered `NO_CAPACITY` only once that worker's + * view has changed (ADR 0009 §5), so a worker that keeps refusing is asked once per change of + * its state, not on every refresh. A `noWait` waiter refused with `NO_CAPACITY` is not + * re-queued: it gets one more walk here, and is rejected if no worker is picked in it. The + * terminal branches have no such follow-up of their own -- a worker this + * class gave up on (`WORKER_UNREACHABLE`, `INTERNAL`) or was refused for a reason genuinely + * unrelated to capacity (any other worker-side code) may still be exactly the worker a different + * queued waiter needs, and nothing else would ever schedule that second look. + */ + // fallow-ignore-next-line complexity -- one attempt, every exit of which is named in the doc comment above. + async #attempt(waiter: FleetWaiter, decision: RoutingDecision, rename: ModelName): Promise { + const workerId = decision.workerId; + const client = liveClient(this.options.directory.target(workerId)); + if (client === undefined) { + // why: The view named a worker that is no longer reachable by the time we got here -- the same + // "stale, not wrong" story as a worker's own `NO_CAPACITY`, just discovered a step earlier. + this.#staleView(waiter, workerId, { unreachable: true }); + return; + } + + const namespacedRequesterId = `${this.options.leaseIndex.requesterPrefix}${waiter.options.requesterId}`; + let announced = false; + const announceDispatched = (): void => { + if (announced) return; + announced = true; + const createdAt = this.#createdAt.get(waiter); + this.options.facts.emit("request.dispatched", { + ...(waiter.request.model === undefined ? {} : { model: waiter.request.model }), + ...(waiter.request.class === undefined ? {} : { class: waiter.request.class }), + ...(waiter.request.mode === undefined ? {} : { mode: waiter.request.mode }), + ...(lineOf(waiter.request) === "physical" ? { physical: true as const } : {}), + platform: waiter.request.platform, + queuedMs: createdAt === undefined ? 0 : Math.max(0, this.options.clock.now() - createdAt), + reason: decision.reason, + requestId: waiter.id, + requesterId: waiter.options.requesterId, + stage: decision.stage, + workerId, + }); + }; + + // why: P4 (round 3 review): H4's own reasoning applies verbatim here -- + // `FleetForwarding.withLeaseRequestTimeout` settling `WORKER_UNREACHABLE` only stops *this* + // call's returned promise from being awaited + // any further, not the worker's own still-pending `lease.request` RPC, whose `onProgress` + // closure below stays live for as long as the worker keeps pushing progress. Before this, + // a late `progress` push after the timeout had already rejected the waiter still called + // `announceDispatched()` and emitted `request.dispatched` for a request this gateway had + // already told its caller was `WORKER_UNREACHABLE` -- a false past-tense fact against + // `docs/internal/EVENTS.md`'s own "and the worker took it". Same mechanism as `exec`'s `detached`, not + // a second bespoke one: flipped by `FleetForwarding.withLeaseRequestTimeout`'s own `onTimeout` + // hook, checked here before either `announceDispatched()` or `notifyProgress` run. + let detached = false; + let grant: LeaseGrant; + try { + grant = await this.options.forwarding.withLeaseRequestTimeout( + client.requestLease( + { + ...requestedDevice({ ...waiter.request, ...rename }), + requesterId: namespacedRequesterId, + // why: ADR §27a (narrowed, round 3 review, H3): only the worker's own gateway-uplink + // session may set `owner` -- and this RPC always travels over exactly that + // connection (`WorkerLink#connect`, via `acceptUplink` on the worker's own end, see + // `session.isGatewayUplink`'s doc). The worker stores this verbatim as the lease's + // `ownerId` instead of deriving it from the connection -- see + // `daemon/dispatcher.ts`'s `#leaseRequest`. + owner: waiter.options.ownerId, + // why: ADR 0009 §3: a download is never triggered through a gateway. + allowDownload: false, + // why: ADR §12: worker queues never hold gateway traffic. Every dispatch is `noWait` + // regardless of what the original caller asked the gateway for -- the *gateway's + // own* queue is where a "wait" request actually waits. + noWait: true, + ...(waiter.options.ttlMs === undefined ? {} : { ttlMs: waiter.options.ttlMs }), + // why: ADR 0020: sent as the requester sent it. The worker grants exactly this id, and + // its refusal (`LEASE_ID_TAKEN`) is the caller's answer: no other worker is tried. + ...(waiter.options.leaseId === undefined ? {} : { leaseId: waiter.options.leaseId }), + // why: ADR 0021: this gateway's id for the request marks the dispatch a probe. The worker + // declines a probe it refuses (`lease.declined`) rather than rejecting it, and names + // the id on the lease events of the dispatch. + fleetRequestId: waiter.id, + ...(waiter.options.requires === undefined + ? {} + : { requires: [...waiter.options.requires] }), + }, + { + onProgress: (progress) => { + if (detached) return; + // why: ADR §11: "the first `progress` push" is one of the two signals a request is + // this worker's now -- device work having started means it is committed here even + // before the grant itself arrives. + announceDispatched(); + this.options.queue.notifyProgress(waiter, progress); + }, + }, + ), + workerId, + () => { + detached = true; + }, + ); + // why: the grant crossed a wire; a worker must not be able to crash the gateway with an answer without a lease, so it fails this attempt like any other worker failure. + if (typeof (grant as Partial | null)?.lease?.id !== "string") { + throw new DispatchError( + "INTERNAL", + `Worker ${workerId} answered lease.request with a malformed grant`, + { workerId }, + ); + } + } catch (error: unknown) { + // why: P1 (round 2 review): "an *immediate* NO_CAPACITY is the only answer that leaves it + // queued ... the request waits" (§11) -- `announced` is exactly the "work has begun" + // distinction the ADR draws, already maintained above for `request.dispatched`. A worker + // can answer `NO_CAPACITY` *after* pushing progress (its own `#evictManaged` failure path, + // reached from a `noWait` waiter's `#defer`, runs after `provisioning`/`reclaiming`), and + // that is this request's own terminal failure -- not a stale view to retry, because device + // work already started means the request was this worker's (§11's own wording). Re-queuing + // it anyway silently reversed a dispatch the caller had already been told about. + if (isSimlockError(error) && STALE_VIEW_CODES.has(error.code) && !announced) { + this.#refusals.rememberRefusal(waiter, workerId, { + capacityReason: refusedCapacityReason(error.details), + }); + if (waiter.options.noWait === true) { + this.#refreshView(workerId); + // why: ADR 0009 §5: one more walk, with this worker left out until its view changes. A + // `noWait` caller never asked to wait for that change, so `#admit` rejects it if no + // worker is picked. + this.admit(waiter); + return; + } + this.#staleView(waiter, workerId); + return; + } + // why: a provision failure before the worker's progress push did no device work, so another worker may serve it and the walk goes on without this one; after a progress push it falls through to the terminal branch below, as today. + if (isSimlockError(error) && error.code === "PROVISION_FAILED" && !announced) { + this.#refusals.rememberRefusal(waiter, workerId, { + provisionFailure: classifyRelayedError(error, workerId), + }); + this.#refreshView(workerId); + this.admit(waiter); + return; + } + if (isSimlockError(error) && CANNOT_SERVE_CODES.has(error.code) && !announced) { + this.#refusals.rememberCannotServe(waiter, workerId, classifyRelayedError(error, workerId)); + this.#refreshView(workerId, { includeCatalog: true }); + this.admit(waiter); + return; + } + // why: C1 (round 3 review): this waiter's own outcome is terminal, but the worker it just gave + // up (or never actually reached) may still be free for whoever else is queued behind it -- + // and unlike `#staleView`'s branch above, nothing else here schedules another look at them. + // `WORKER_UNREACHABLE`/`INTERNAL` in particular need not produce any worker-side event, so + // without this a waiter passed over in this same pass had nothing left to wake it until the + // next real worker-view change (the 30s refresh tick, at best). + // ADR 0021 §4: this is the request's end, and the worker emits no `lease.rejected` for a + // probe (at most a `lease.declined`; for an unreachable worker or a gateway-side `INTERNAL` + // there is usually none, and a dispatch timeout only stops this gateway waiting: the + // worker's `lease.request` keeps running and can still end in a relayed `lease.declined` + // or a grant), so this gateway records it, with the code the caller got and the worker it + // failed on. + const failure = this.#classifyLeaseRequestError(error, workerId); + this.options.outcomes.reject(waiter, failure, "worker-failed", { + code: failure.code, + worker: workerId, + }); + this.dispatch(); + return; + } + + announceDispatched(); + if (this.options.grants.settle(waiter, workerId, grant) === "retry") { + // why: The worker answered with an id the gateway did not ask for: the attempt failed, and the + // request goes back to the queue as after an unreachable worker. This worker is left out + // until its view changes, so a worker that keeps answering so is not asked in a loop. + this.#refusals.rememberRefusal(waiter, workerId); + this.#staleView(waiter, workerId); + return; + } + // why: C1 (round 3 review): see the catch branch above -- a grant settling this waiter is just as + // much a reason for whoever else is queued to get another look, not only a terminal failure. + this.dispatch(); + } + + /** + * why: a terminal failure past `#attempt`'s `NO_CAPACITY`/stale-view check. A worker never + * rejects a probe: where it answered, it only declined (`lease.declined`, relayed onto this bus with + * `workerId` added by `WorkerLink`), and where it did not (`WORKER_UNREACHABLE`, a gateway-side + * `INTERNAL`) it usually emitted nothing; after a dispatch timeout the worker's request is + * still running and may yet end in a relayed `lease.declined` or a grant, since the timeout + * only stops this gateway waiting. Either way the request's end is this + * gateway's own `lease.rejected` (ADR 0021 §4). The error code is + * preserved so the caller sees what the worker actually said, with three exceptions: + * + * - a transport failure (the uplink itself, not the worker's own answer) -- ADR §28/§29 name + * `WORKER_UNREACHABLE` for that, not whatever the client's own connection loss happens to be + * called (`daemon/dispatch.js`'s `SimlockError.kind` is what tells the two apart); + * - `FleetForwarding.withLeaseRequestTimeout`'s own `DispatchError` (P2, round 2 review) -- + * never a `SimlockError`, since it is raised here, by this gateway's own timer, not by the worker's + * answer (the request itself did reach the worker and keeps running there) -- forwarded as-is; + * - anything else that is neither of those (H1, round 2 review): not a fact about the worker, + * since every real transport failure already arrives as a `kind: "transport"` `SimlockError`, + * so this is a bug in this coordinator's own request-building code. `WORKER_UNREACHABLE` + * used to answer this case too, misreporting a gateway-side crash as "the machine is + * unreachable"; `INTERNAL` is what `FleetForwarding#forwardToWorker`'s matching branch + * answers for the same shape of failure. + */ + #classifyLeaseRequestError(error: unknown, workerId: string): DispatchError { + if (error instanceof DispatchError) return error; + if (isSimlockError(error)) return classifyRelayedError(error, workerId); + return new DispatchError( + "INTERNAL", + `Unexpected error forwarding lease.request to worker ${workerId}`, + { workerId }, + ); + } + + /** why: ADR §11: "an immediate `NO_CAPACITY` is the only answer that leaves it queued ... the + * request waits", because this is a stale view, not a real refusal. A waiter reaches + * this from a target that turned out unreachable, and from a grant whose id was not the one + * asked for (`noWait` included, in both); a `noWait` caller refused with `NO_CAPACITY` is + * settled in `#attempt` instead (ADR 0009 §5). */ + #staleView( + waiter: FleetWaiter, + workerId: string, + options: { readonly unreachable?: boolean } = {}, + ): void { + // why: the walk that picked this worker ran before its answer; the entries must say what is true now. + this.#workerReasons.set( + waiter, + this.#refusals.reasonsFor( + waiter, + routable(waiter), + this.options.views.routingViews(), + options.unreachable === true ? { unreachable: workerId } : {}, + ), + ); + this.options.outcomes.enqueue(waiter); + this.#refreshView(workerId); + } + + #refreshView(workerId: string, options?: { readonly includeCatalog: true }): void { + const target = this.options.directory.target(workerId); + void target?.refresh(options).catch((error: unknown) => { + this.options.logger.debug("Failed to refresh a worker's view after a worker's refusal", { + workerId, + message: error instanceof Error ? error.message : String(error), + }); + }); + } +} diff --git a/src/gateway/fleet-facts.ts b/src/gateway/fleet-facts.ts new file mode 100644 index 000000000..c176b7bfe --- /dev/null +++ b/src/gateway/fleet-facts.ts @@ -0,0 +1,74 @@ +import type { EventBus, EventMap } from "../bus/index.js"; +import type { FleetQueue, FleetWaiter } from "./queue.js"; +import type { Rejection } from "./routing/index.js"; + +export interface FleetFactsOptions { + readonly eventBus: Pick; +} + +export type FleetEventName = + | "lease.requested" + | "lease.queued" + | "lease.rejected" + | "queue.changed" + | "request.dispatched" + | "request.granted"; + +export class FleetFacts { + constructor(private readonly options: FleetFactsOptions) {} + + emit(event: Event, payload: EventMap[Event]): void { + this.options.eventBus.emit(event, payload, "fleet-lease-coordinator"); + } + + rejected( + waiter: FleetWaiter, + reason: EventMap["lease.rejected"]["reason"], + failedOn?: { readonly code: string; readonly worker: string }, + ): void { + this.emit("lease.rejected", { + requestId: waiter.id, + requester: waiter.options.requesterId, + requestSpec: waiter.request, + reason, + // why: `worker`, never `workerId`: that key marks an event relayed from a worker (ADR 0014 §6). + ...(failedOn === undefined ? {} : { code: failedOn.code, worker: failedOn.worker }), + }); + } +} + +export interface FleetOutcomesOptions { + readonly queue: FleetQueue; + readonly facts: FleetFacts; +} + +export class FleetOutcomes { + constructor(private readonly options: FleetOutcomesOptions) {} + + enqueue(waiter: FleetWaiter): void { + const alreadyQueued = this.options.queue.isQueued(waiter); + if (this.options.queue.enqueue(waiter) && !alreadyQueued) { + this.options.facts.emit("lease.queued", { + queuePosition: this.options.queue.positionOf(waiter), + requestId: waiter.id, + }); + } + } + + reject( + waiter: FleetWaiter, + error: Error, + reason: + | Rejection["reason"] + | "no-wait" + | "cancelled" + | "timeout" + | "lease-id-taken" + | "worker-failed", + failedOn?: { readonly code: string; readonly worker: string }, + ): void { + if (this.options.queue.reject(waiter, error)) { + this.options.facts.rejected(waiter, reason, failedOn); + } + } +} diff --git a/src/gateway/fleet-grant.ts b/src/gateway/fleet-grant.ts new file mode 100644 index 000000000..8c3d36eaa --- /dev/null +++ b/src/gateway/fleet-grant.ts @@ -0,0 +1,97 @@ +import type { LeaseGrant } from "../admin/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import type { Logger } from "../ports/index.js"; +import type { FleetForwarding } from "./fleet-forwarding.js"; +import type { FleetFacts, FleetOutcomes } from "./fleet-facts.js"; +import type { FleetLeaseEntry, FleetLeaseIndex } from "./lease-index.js"; +import type { FleetLeaseGrant, FleetQueue, FleetWaiter } from "./queue.js"; + +export interface FleetGrantsOptions { + readonly forwarding: FleetForwarding; + readonly leaseIndex: FleetLeaseIndex; + readonly logger: Logger; + readonly queue: FleetQueue; + readonly facts: FleetFacts; + readonly outcomes: FleetOutcomes; +} + +export class FleetGrants { + constructor(private readonly options: FleetGrantsOptions) {} + + /** + * why: H6 (round 2 review): the entry's `ownerId` comes from `waiter.options.ownerId` -- what this + * gateway itself forwarded as `owner` in the `lease.request` a moment ago -- never from the + * worker's own echo of it (`grant.lease.ownerId`). Everything this gateway authorizes + * afterwards (`lease.renew`/`release`/`device.exec`) keys on the index's `ownerId`, so a worker + * that ignored or rewrote the field would otherwise redefine ownership at the gateway; the + * rebuild path (`FleetLeaseIndex#rebuildFromWorker`) has no better source and must trust the + * worker, but this path already knows the answer. A mismatched echo is logged -- it means + * either a worker bug or something worth knowing about, never silently swallowed. + */ + settle(waiter: FleetWaiter, workerId: string, grant: LeaseGrant): "settled" | "retry" { + // why: ADR 0020: a forwarded `leaseId` is the gateway lease id, bare, and never the worker's echo + // of it -- a worker that granted something else is not believed, and its lease is given back. + const forwardedLeaseId = waiter.options.leaseId; + if (forwardedLeaseId !== undefined && grant.lease.id !== forwardedLeaseId) { + this.options.logger.warn( + "Worker granted a lease ID different from the leaseId the gateway forwarded", + { + forwardedLeaseId, + grantedLeaseId: grant.lease.id, + workerId, + }, + ); + this.options.forwarding.releaseOnWorker(workerId, grant.lease.id); + return "retry"; + } + const gatewayLeaseId = forwardedLeaseId ?? `${workerId}.${grant.lease.id}`; + if (grant.lease.ownerId !== waiter.options.ownerId) { + this.options.logger.warn( + "Worker echoed an ownerId different from the one the gateway forwarded", + { + echoedOwnerId: grant.lease.ownerId, + forwardedOwnerId: waiter.options.ownerId, + gatewayLeaseId, + workerId, + }, + ); + } + const entry: FleetLeaseEntry = { + gatewayLeaseId, + grantedAt: grant.lease.grantedAt, + idChosenByRequester: forwardedLeaseId !== undefined, + ownerId: waiter.options.ownerId, + requesterId: waiter.options.requesterId, + workerId, + workerLeaseId: grant.lease.id, + }; + if (!this.options.leaseIndex.add(entry, waiter.request.platform)) { + // why: The index already routes this bare id to another worker, which got it there first: the + // new worker's lease is given back and the caller is told the id is taken. + this.options.forwarding.releaseOnWorker(workerId, grant.lease.id); + this.options.outcomes.reject(waiter, leaseIdTaken(gatewayLeaseId), "lease-id-taken"); + return "settled"; + } + const fleetGrant: FleetLeaseGrant = { + device: grant.device, + environment: grant.environment, + lease: this.options.forwarding.projectRecord(grant.lease, entry), + timing: grant.timing, + }; + // why: ADR 0021 §4: the grant is the request's end only if the waiter was still open. One that + // landed after a timeout, a cancel or `dispose()` handed nothing to a caller, so it is no fact. + if (this.options.queue.resolve(waiter, fleetGrant)) { + this.options.facts.emit("request.granted", { + leaseId: gatewayLeaseId, + requestId: waiter.id, + worker: workerId, + workerLeaseId: grant.lease.id, + }); + } + return "settled"; + } +} + +export function leaseIdTaken(leaseId: string): DispatchError { + return new DispatchError("LEASE_ID_TAKEN", `lease ID ${leaseId} is already in use`, { leaseId }); +} diff --git a/src/gateway/fleet-index-upkeep.ts b/src/gateway/fleet-index-upkeep.ts new file mode 100644 index 000000000..1bff30049 --- /dev/null +++ b/src/gateway/fleet-index-upkeep.ts @@ -0,0 +1,75 @@ +import type { Clock } from "../ports/index.js"; +import type { EndedLeaseIds } from "./ended-lease-ids.js"; +import type { FleetViews } from "./fleet-ports.js"; +import type { FleetLeaseIndex } from "./lease-index.js"; +import type { WorkerView } from "./worker-registry.js"; + +export interface FleetIndexUpkeepOptions { + readonly clock: Clock; + readonly endedLeaseIds: EndedLeaseIds; + readonly leaseIndex: FleetLeaseIndex; + readonly views: Pick; +} + +export class FleetIndexUpkeep { + #knownWorkerIds = new Set(); + /** why: C1 (round 2 review): the `view.leases` array reference this coordinator last reconciled + * into `leaseIndex` for each worker, keyed by worker id. `onViewsChanged` fires for *every* + * worker-view mutation on the whole registry (`connected`, `refresh`, `disconnected`, + * `setDrained`, `remove`, `pruneExpired` -- see `FleetViews#onViewsChanged`'s own doc), not + * just a change to the worker being reconciled, so calling `rebuildFromWorker` for every view + * on every notification bumped a worker's reconciliation generation without a new snapshot + * ever having arrived for it (see `#onViewsChanged`'s own doc for the eviction this caused). + * `WorkerRegistry` only ever replaces a view's `leases` array when a snapshot naming `leases` + * actually lands for that specific worker (`refresh`'s partial-update contract; `connected` and + * `disconnected` never touch it) -- so this reference is exactly "a new snapshot for this + * worker arrived" and nothing else, with no need to widen `FleetViews` to carry worker + * identity on the notification itself. */ + readonly #lastReconciledLeases = new Map>(); + + constructor(private readonly options: FleetIndexUpkeepOptions) { + options.leaseIndex.onEnded((entry) => { + if (entry.idChosenByRequester === true) { + options.endedLeaseIds.record(entry.gatewayLeaseId, options.clock.now()); + } + }); + } + + /** + * why: §30's reconnect rebuild, and its own upkeep on every worker-view change: reconciles each + * worker's entries against its current view -- adding any gateway-issued lease the index does + * not already know and, a generation later, forgetting one the view has stopped reporting at + * all (see `FleetLeaseIndex#rebuildFromWorker`'s own doc comment, C3 round 2 review) -- forgets + * a worker's entries entirely once its view has disappeared (`worker.remove`, or retention), + * then runs a dispatch pass. + * + * C1 (round 2 review): `rebuildFromWorker` runs *only* for a worker whose `view.leases` + * reference actually changed since the last time this method reconciled it -- see + * `#lastReconciledLeases`'s own doc. Before this fix, every view was reconciled on *every* + * notification regardless of which worker it was about: two view changes on worker B alone + * bumped worker A's reconciliation generation twice against A's own byte-identical, stale + * cached snapshot, which is enough for `rebuildFromWorker`'s two-consecutive-misses rule to + * evict a lease on A that was never actually missing from a real snapshot at all -- silently + * violating ADR §14 (the fleet-wide one-lease check reads this same index) and losing the + * `lease-lost` push `GatewayOwnerRoutedFacts` would otherwise emit for it. + */ + reconcile(): void { + const views = this.options.views.views(); + const currentIds = new Set(views.map((view) => view.id)); + for (const view of views) { + // why: A starting worker's view has no `leases`: nothing was read, so the index keeps what it + // holds for that worker until the worker answers `running`. + if (view.leases === undefined) continue; + if (this.#lastReconciledLeases.get(view.id) === view.leases) continue; + this.options.leaseIndex.rebuildFromWorker(view.id, view.leases, view.devices); + this.#lastReconciledLeases.set(view.id, view.leases); + } + for (const workerId of this.#knownWorkerIds) { + if (!currentIds.has(workerId)) { + this.options.leaseIndex.forgetWorker(workerId); + this.#lastReconciledLeases.delete(workerId); + } + } + this.#knownWorkerIds = currentIds; + } +} diff --git a/src/gateway/fleet-request-shape.ts b/src/gateway/fleet-request-shape.ts new file mode 100644 index 000000000..8737810e3 --- /dev/null +++ b/src/gateway/fleet-request-shape.ts @@ -0,0 +1,53 @@ +import { requestedClass, type Platform } from "../contract/index.js"; +import { lineOf, type FleetWaiter } from "./queue.js"; +import { matchRequest, type RoutableRequest, type RoutingDecision } from "./routing.js"; +import type { WorkerView } from "./worker-registry.js"; + +/** why: A class request has no model to send: the worker applies its own rules to the class. */ +export function forwardedModel( + request: RoutableRequest, + views: readonly WorkerView[], + decision: RoutingDecision, +): ModelName { + if (request.model === undefined || request.physical === true) return {}; + const view = views.find((worker) => worker.id === decision.workerId); + return { model: (view === undefined ? undefined : matchRequest(view, request)) ?? request.model }; +} + +export interface ModelName { + readonly model?: string; +} + +// why: keyed as the core's `fitsPhysical` compares: platform, model in any letter case or else the class (`phone` when absent), and the OS text. +export function physicalMatchKey(request: FleetWaiter["request"]): string { + const target = + request.model === undefined + ? `class:${requestedClass({ class: request.class })}` + : `model:${request.model.toLowerCase()}`; + return [request.platform, target, request.osVersion ?? ""].join("\u0000"); +} + +// why: a request field that is absent stays absent in the routable request (never `undefined`), as `exactOptionalPropertyTypes` needs. +function present(fields: T): { [K in keyof T]?: Exclude } { + return Object.fromEntries(Object.entries(fields).filter(([, value]) => value !== undefined)) as { + [K in keyof T]?: Exclude; + }; +} + +export function routable(waiter: FleetWaiter): RoutableRequest { + const { request, options } = waiter; + return { + platform: request.platform as Platform, + ...present({ + model: request.model, + class: request.class, + osVersion: request.osVersion, + imageTag: request.imageTag, + baseline: request.baseline, + mode: request.mode, + requires: options.requires, + except: request.except, + }), + ...(lineOf(request) === "physical" ? { physical: true as const } : {}), + }; +} From 0f4434712197d473c574a7c4f7c74326d432a7c3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:21:45 +0100 Subject: [PATCH 18/83] docs(manuals): point at the parts that now hold the operations, dispatcher handlers, client and config checks --- src/auth/AGENTS.md | 2 +- src/contract/AGENTS.md | 8 ++++---- src/core/capacity/AGENTS.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/auth/AGENTS.md b/src/auth/AGENTS.md index f67e24e4a..153ac45d3 100644 --- a/src/auth/AGENTS.md +++ b/src/auth/AGENTS.md @@ -39,4 +39,4 @@ Socket: 1. A directory under `strategies/`, and one line in `strategies/index.ts`: its place in the list is its priority. 2. A fixture in `strategies/conformance.test.ts`. -3. If it owns a config key under `http`: the mirror in the contract's `configSchema`. `src/core/config.ts` validates and defaults the key from the definition. +3. If it owns a config key under `http`: the mirror in the contract's `configSchema`. `src/core/config-validators.ts` validates and `config-defaults.ts` defaults the key from the definition. diff --git a/src/contract/AGENTS.md b/src/contract/AGENTS.md index a3ba490a8..d5ee04b02 100644 --- a/src/contract/AGENTS.md +++ b/src/contract/AGENTS.md @@ -9,7 +9,7 @@ Shapes: - Public types are `z.infer`red from these schemas, never written a second time. - The daemon parses every handler result through the output schema; a core field the schema does not declare is stripped, never sent. - `config.get` carries a driver's own keys under `ios` and `android`, and a module's under `capabilities`, as sent: a new one is no shape change. -- `lease.request` holds `except` (a slim iOS simulator lease keeping slim categories running, ADR 0066) to one object-level refinement with no path (`exceptRefusal`, `operations.ts`), so every transport answers `BAD_REQUEST` with the message as written: 1 to 16 names matching `EXCEPT_NAME_PATTERN`, none twice, never with `physical`, a platform but iOS or `mode: full`. The worker's default mode and a baseline's mode are checked in leasing. +- `lease.request` holds `except` (a slim iOS simulator lease keeping slim categories running, ADR 0066) to one object-level refinement with no path (`exceptRefusal`, `operations-lease-request.ts`), so every transport answers `BAD_REQUEST` with the message as written: 1 to 16 names matching `EXCEPT_NAME_PATTERN`, none twice, never with `physical`, a platform but iOS or `mode: full`. The worker's default mode and a baseline's mode are checked in leasing. - `lease.request`'s `metadata` (ADR 0074 §3) is held to one object-level refinement too (`refuseBadMetadata`): the same words on every transport. A gateway hands it to its admission command alone; nothing else may read it. - Wire input is a claim: every field a gateway parses from a worker is bounded here (`fit*`, `CONSOLE_URL_MAX_LENGTH`, `relay.ts`). - One OS grammar (`os-range.ts`), one duration parser (`duration.ts`), one usage window (`usageWindow`, `usage-window.ts`), one events page query (`eventsPageQuery`, `events-page.ts`). Nobody else parses. @@ -43,11 +43,11 @@ Errors and views: ## Adding an operation -1. `operations.ts`: `defineOperation`, with `authorize` when it acts on one lease. Add it to `OPERATIONS`. +1. The `operations-*.ts` file of its family (lease request, leases, admin, fleet): `defineOperation` (`operation-definition.ts`), with `authorize` when it acts on one lease. Add it to `OPERATIONS` in `operations.ts`. 2. `operations.test.ts`: a row in the role matrix and the effect matrix. 3. `protocol.ts`: `PROTOCOL_VERSION_RANGE` + 1. -4. Handlers in `src/daemon/dispatcher.ts` and `src/gateway/dispatcher.ts`. A gateway-only operation answers `UNSUPPORTED_IN_WORKER_MODE` on a worker; the reverse, `UNSUPPORTED_IN_GATEWAY_MODE`. -5. Client: a method on `SimlockClient` or `SimlockAdminClient` (`src/simlock-client/client.ts`), types in `src/simlock-client/types.ts`, re-exported from `src/client/index.ts` or `src/admin/index.ts`. +4. Handlers in `src/daemon/dispatcher.ts` (the table; the handler goes in its family's `dispatcher-*.ts` part) and `src/gateway/dispatcher.ts`. A gateway-only operation answers `UNSUPPORTED_IN_WORKER_MODE` on a worker; the reverse, `UNSUPPORTED_IN_GATEWAY_MODE`. +5. Client: a method on `SimlockClient` or `SimlockAdminClient` (declared in `src/simlock-client/client-api.ts`, implemented in `client-impl.ts`), types in `src/simlock-client/types.ts`, re-exported from `src/client/index.ts` or `src/admin/index.ts`. 6. Frontends as needed: CLI, HTTP, MCP only for an agent session's need. 7. A new error code: `ErrorDetailsMap` and `ERROR_TABLE` (`errors.ts`). 8. Docs: the website pages that describe it, at least [errors.mdx](../../website/src/content/docs/docs/reference/errors.mdx) (documentation rule 9). diff --git a/src/core/capacity/AGENTS.md b/src/core/capacity/AGENTS.md index 2440bbd74..03db1c439 100644 --- a/src/core/capacity/AGENTS.md +++ b/src/core/capacity/AGENTS.md @@ -16,7 +16,7 @@ Does: how many virtual devices may exist and run at once — limits, RAM budget, - Over the RAM limit (sizes raised by a restart), every create and every boot that adds RAM is refused. Nothing is stopped or reclaimed for it. - Release is idempotent. Every take and release calls the observer. - `CapacityObserver.changed()` runs after every registry commit and reservation change, and emits `capacity.changed` only when the payload differs from the last one. It is silent until `start()` in `core.converge`, which emits once. -- Top-level `limits` and `ramBudget` are folded into `capacity.config` under `resource`, and ignored with a warning under `fixed` (`core/config.ts`). +- Top-level `limits` and `ramBudget` are folded into `capacity.config` under `resource`, and ignored with a warning under `fixed` (`core/config-role-checks.ts`). - Adding a strategy is a directory under `strategies/` and one line in its `index.ts`. ## Known gaps From ad4205844165b2a7c1fd4b4e78bf2de9e5f86a28 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:21:08 +0100 Subject: [PATCH 19/83] test(leasing): split create-leasing.test.ts by topic --- .../create-leasing-baseline-pools.test.ts | 76 + .../create-leasing-capacity-events.test.ts | 147 + .../create-leasing-fresh-identity.test.ts | 371 ++ .../create-leasing-gateway-and-retry.test.ts | 399 ++ .../create-leasing-grant-and-queue.test.ts | 717 +++ src/leasing/create-leasing-lease-id.test.ts | 333 ++ .../create-leasing-max-per-requester.test.ts | 591 ++ src/leasing/create-leasing-ram-budget.test.ts | 205 + src/leasing/create-leasing-rejections.test.ts | 233 + ...create-leasing-restart-and-classes.test.ts | 187 + .../create-leasing-startup-reclaim.test.ts | 241 + .../create-leasing-waiting-reasons.test.ts | 294 + src/leasing/create-leasing-warm-pool.test.ts | 562 ++ .../create-leasing-warm-targets.test.ts | 278 + .../create-leasing-wiring-and-logger.test.ts | 193 + src/leasing/create-leasing.test.ts | 4982 +---------------- src/leasing/test-create-leasing.ts | 236 + src/source-hygiene.test.ts | 1 - 18 files changed, 5070 insertions(+), 4976 deletions(-) create mode 100644 src/leasing/create-leasing-baseline-pools.test.ts create mode 100644 src/leasing/create-leasing-capacity-events.test.ts create mode 100644 src/leasing/create-leasing-fresh-identity.test.ts create mode 100644 src/leasing/create-leasing-gateway-and-retry.test.ts create mode 100644 src/leasing/create-leasing-grant-and-queue.test.ts create mode 100644 src/leasing/create-leasing-lease-id.test.ts create mode 100644 src/leasing/create-leasing-max-per-requester.test.ts create mode 100644 src/leasing/create-leasing-ram-budget.test.ts create mode 100644 src/leasing/create-leasing-rejections.test.ts create mode 100644 src/leasing/create-leasing-restart-and-classes.test.ts create mode 100644 src/leasing/create-leasing-startup-reclaim.test.ts create mode 100644 src/leasing/create-leasing-waiting-reasons.test.ts create mode 100644 src/leasing/create-leasing-warm-pool.test.ts create mode 100644 src/leasing/create-leasing-warm-targets.test.ts create mode 100644 src/leasing/create-leasing-wiring-and-logger.test.ts create mode 100644 src/leasing/test-create-leasing.ts diff --git a/src/leasing/create-leasing-baseline-pools.test.ts b/src/leasing/create-leasing-baseline-pools.test.ts new file mode 100644 index 000000000..c01c1ab6f --- /dev/null +++ b/src/leasing/create-leasing-baseline-pools.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { request, createHarness, seedReady } from "./test-create-leasing.js"; + +describe("createLeasing: a baseline is a pool key", () => { + type Harness = Awaited>; + const plain = { ...request, physical: false } as const; + const copyOf = (name: string, version: number) => + ({ ...plain, baseline: { name, version } }) as const; + const named = { baseline: "checkout", platform: "ios" } as const; + + // why: the engine reads a baseline from `baselines` and copies the source the driver holds, so a row and a saved source stand in for an operator's save. + async function saveBaseline(harness: Harness, name: string, version: number): Promise { + const driverDevice = await harness.driver.provision(plain); + const ready = await harness.driver.makeReady(driverDevice, { + mode: "full", + purpose: "prepare", + }); + const saved = await harness.driver.saveSource({ ...ready, physical: false }); + harness.state.database + .prepare( + `INSERT INTO baselines + (name, version, platform, model, os_version, mode, saved_at, source_id) + VALUES (?, ?, 'ios', 'iPhone 16', '26.5', 'full', '2026-11-03T00:00:00.000Z', ?)`, + ) + .run(name, version, saved.sourceId); + } + + it.each([ + { + label: "an idle baseline device and a plain request", + idle: copyOf("checkout", 1), + ask: request, + granted: undefined, + reused: false, + }, + { + label: "an idle plain device and a request naming a baseline", + idle: plain, + ask: named, + granted: { name: "checkout", version: 2 }, + reused: false, + }, + { + label: "an idle device of another baseline", + idle: copyOf("login", 2), + ask: named, + granted: { name: "checkout", version: 2 }, + reused: false, + }, + { + label: "an idle device of an older version", + idle: copyOf("checkout", 1), + ask: named, + granted: { name: "checkout", version: 2 }, + reused: false, + }, + { + label: "an idle device of the baseline's current version (control)", + idle: copyOf("checkout", 2), + ask: named, + granted: { name: "checkout", version: 2 }, + reused: true, + }, + ])("never grants a device across pools: $label", async ({ ask, granted, idle, reused }) => { + const harness = await createHarness(); + await saveBaseline(harness, "checkout", 1); + await saveBaseline(harness, "checkout", 2); + await saveBaseline(harness, "login", 1); + const device = await seedReady(harness, idle); + + const grant = await harness.engine.request(ask, { ownerId: "o", requesterId: "r" }); + + expect(grant.device.id === device.id).toBe(reused); + expect(grant.device.spec.baseline).toEqual(granted); + }); +}); diff --git a/src/leasing/create-leasing-capacity-events.test.ts b/src/leasing/create-leasing-capacity-events.test.ts new file mode 100644 index 000000000..14acdaa65 --- /dev/null +++ b/src/leasing/create-leasing-capacity-events.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "vitest"; +import { buildCapacityFigures } from "../core/index.js"; +import { capacityChangedPayload } from "../core/testing.js"; +import { type EventMap } from "../bus/index.js"; +import { request, createHarness, seedReady, flush } from "./test-create-leasing.js"; + +describe("createLeasing: capacity.changed and queue.changed", () => { + type CapacityEvent = EventMap["capacity.changed"]; + + function capacityEvents(harness: Awaited>): CapacityEvent[] { + return harness.bus + .replay() + .filter((event) => event.event === "capacity.changed") + .map((event) => event.payload as CapacityEvent); + } + + function queueEvents(harness: Awaited>): number[] { + return harness.bus + .replay() + .filter((event) => event.event === "queue.changed") + .map((event) => (event.payload as EventMap["queue.changed"]).depth); + } + + function currentFigures(harness: Awaited>): CapacityEvent { + return capacityChangedPayload( + buildCapacityFigures(harness.registry.snapshot.virtualDevices, harness.engine), + ); + } + + it("emits capacity.changed and queue.changed once when the daemon has started, and none before", async () => { + const harness = await createHarness(); + await seedReady(harness); + expect([capacityEvents(harness), queueEvents(harness)]).toEqual([[], []]); + + await harness.engine.convergeRunningCapacity(); + + expect(capacityEvents(harness)).toEqual([currentFigures(harness)]); + expect(capacityEvents(harness)[0]).toMatchObject({ global: { warm: 1 }, ios: { warm: 1 } }); + expect(queueEvents(harness)).toEqual([0]); + }); + + it("emits capacity.changed after a provision commits, with the figures status.get reports at that moment", async () => { + const harness = await createHarness(); + await harness.engine.convergeRunningCapacity(); + const before = capacityEvents(harness).length; + const seen: Array<{ event: CapacityEvent; figures: CapacityEvent }> = []; + harness.bus.subscribe("capacity.changed", (envelope) => + seen.push({ event: envelope.payload, figures: currentFigures(harness) }), + ); + + await harness.registry.registerDevice({ + driverData: { fakeDeviceId: "sim-1" }, + driverDeviceId: "sim-1", + provisionDuration: 0, + spec: { ...request, physical: false }, + }); + + expect(capacityEvents(harness)).toHaveLength(before + 1); + expect(seen).toEqual([{ event: currentFigures(harness), figures: currentFigures(harness) }]); + expect(seen[0]?.event.ramBudget?.usedBytes).toBeGreaterThan(0); + }); + + it("emits capacity.changed when a provisioning reservation is taken, before any registry commit, and when it is released", async () => { + const harness = await createHarness(); + await harness.engine.convergeRunningCapacity(); + const seen: Array<{ reserved: number; devices: number }> = []; + harness.bus.subscribe("capacity.changed", (envelope) => + seen.push({ + devices: harness.registry.snapshot.devices.length, + reserved: envelope.payload.ios.reserved, + }), + ); + harness.driver.hangMakeReady(); + + const pending = harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + await flush(); + expect(seen[0]).toEqual({ devices: 0, reserved: 1 }); + harness.driver.releaseMakeReady(); + await pending; + await flush(); + + expect(seen.at(-1)?.reserved).toBe(0); + expect(capacityEvents(harness).at(-1)).toEqual(currentFigures(harness)); + }); + + it("emits capacity.changed after a device is deleted and after a lease release destroys its device", async () => { + const harness = await createHarness(); + const grant = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + await harness.engine.convergeRunningCapacity(); + expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 1 } }); + + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 0 } }); + + const warm = await seedReady(harness); + expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 1, running: 1 } }); + + await harness.registry.markDeviceMissing(warm.id, "test"); + expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 0 } }); + expect(capacityEvents(harness).at(-1)).toEqual(currentFigures(harness)); + }); + + it("emits no capacity.changed when a commit leaves the capacity figures unchanged", async () => { + const harness = await createHarness(); + await seedReady(harness); + await harness.engine.convergeRunningCapacity(); + const before = capacityEvents(harness).length; + + await harness.registry.createLeaseRequest({ + ownerId: "a", + request, + requesterId: "a", + }); + await harness.registry.markForeignStateDetected(harness.registry.snapshot.devices[0]!.id, 1); + + expect(capacityEvents(harness)).toHaveLength(before); + }); + + it("never emits two consecutive capacity.changed events with equal figures across a whole lease", async () => { + const harness = await createHarness(); + await harness.engine.convergeRunningCapacity(); + const grant = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + + const events = capacityEvents(harness); + expect(events.length).toBeGreaterThan(3); + events.slice(1).forEach((event, index) => expect(event).not.toEqual(events[index])); + }); + + it("carries ramBudget under the resource strategy and omits it under the fixed strategy", async () => { + const resource = await createHarness(); + await resource.engine.convergeRunningCapacity(); + const fixed = await createHarness({ + capacity: { strategy: "fixed", config: { maxRunning: 2 } }, + }); + await fixed.engine.convergeRunningCapacity(); + + expect(capacityEvents(resource)[0]?.ramBudget).toEqual({ + limitBytes: expect.any(Number), + usedBytes: 0, + }); + expect(capacityEvents(fixed)).toHaveLength(1); + expect(capacityEvents(fixed)[0]).not.toHaveProperty("ramBudget"); + }); +}); diff --git a/src/leasing/create-leasing-fresh-identity.test.ts b/src/leasing/create-leasing-fresh-identity.test.ts new file mode 100644 index 000000000..eac153045 --- /dev/null +++ b/src/leasing/create-leasing-fresh-identity.test.ts @@ -0,0 +1,371 @@ +import { describe, expect, it } from "vitest"; +import { DriverCrashError, type Registry } from "../core/index.js"; +import { testStateOf } from "../core/testing.js"; +import { + FakeDriver, + loadTestRegistry, + openTestState, + testComponentWiring, +} from "../core/testing.js"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, FakeSystemStats } from "../ports/index.js"; +import { createTestEngine } from "./testing.js"; +import { gibibyte, request, config, createHarness, flush } from "./test-create-leasing.js"; + +describe("createLeasing fresh lease identity (#75)", () => { + const freshIos = { android: "reusable", ios: "fresh" } as const; + + function driverDeviceIdOf( + harness: { readonly registry: Registry }, + deviceId: string, + ): string | undefined { + return harness.registry.snapshot.devices.find((device) => device.id === deviceId) + ?.driverDeviceId; + } + + function destroyedDriverDeviceIds(driver: FakeDriver): string[] { + return driver.calls + .filter((call) => call.operation === "destroy") + .map((call) => (call.arguments[0] as { readonly deviceId: string }).deviceId); + } + + it("gives two sequential fresh leases for one shape different driver device ids", async () => { + const harness = await createHarness({ identity: freshIos }); + + const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + const second = await harness.engine.request(request, { + ownerId: "second", + requesterId: "second", + }); + + expect(second.device.id).not.toBe(first.device.id); + expect(driverDeviceIdOf(harness, second.device.id)).not.toBe( + driverDeviceIdOf(harness, first.device.id), + ); + expect(harness.driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("takes a released fresh device to deleted and destroys its driver device, without an erase", async () => { + const harness = await createHarness({ identity: freshIos }); + const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + const driverDeviceId = driverDeviceIdOf(harness, granted.device.id); + + await harness.engine.release(granted.lease.id, "explicit"); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: granted.device.id, state: "deleted" }, + ]); + expect(destroyedDriverDeviceIds(harness.driver)).toEqual([driverDeviceId]); + expect(harness.driver.calls.map((call) => call.operation)).not.toContain("reclaim"); + expect(harness.bus.replay().filter((event) => event.event === "device.deleted")).toMatchObject([ + { payload: { deviceId: granted.device.id, initiator: "lease-end" } }, + ]); + expect(harness.bus.replay().map((event) => event.event)).not.toContain("device.reclaimed"); + }); + + it("deletes the device of an expired fresh lease", async () => { + const harness = await createHarness({ identity: freshIos, lease: { defaultTtlMs: 10 } }); + const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + + harness.clock.advance(10); + await flush(); + await harness.engine.settle(); + + expect(harness.bus.replay().map((event) => event.event)).toContain("lease.expired"); + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: granted.device.id, state: "deleted" }, + ]); + expect(destroyedDriverDeviceIds(harness.driver)).toHaveLength(1); + }); + + it("the health monitor clears a device's recovery mark only once the decision section is free", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 14_400_000 } }); + const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + await harness.registry.markRecoveryAttempt(granted.device.id, harness.clock.now()); + let open!: () => void; + const held = harness.decisions.run( + () => + new Promise((resolve) => { + open = resolve; + }), + ); + + harness.engine.healthMonitor.start(); + harness.clock.advance(30_000); + await flush(); + expect(harness.registry.snapshot.devices[0]?.recoveringSince).toBeDefined(); + open(); + await held; + await flush(); + harness.engine.healthMonitor.dispose(); + + expect(harness.registry.snapshot.devices[0]?.recoveringSince).toBeUndefined(); + }); + + it("deletes the device of a recovery-driven lease termination", async () => { + const harness = await createHarness({ + identity: freshIos, + lease: { defaultTtlMs: 14_400_000 }, + }); + const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + harness.driver.setManagedReality({ devices: [], processes: [] }); + + harness.engine.healthMonitor.start(); + for (let tick = 0; tick < 3; tick += 1) { + harness.clock.advance(30_000); + await flush(); + } + await harness.engine.settle(); + harness.engine.healthMonitor.dispose(); + + expect(harness.bus.replay().filter((event) => event.event === "lease.released")).toMatchObject([ + { payload: { leaseId: granted.lease.id, reason: "device-lost" } }, + ]); + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: granted.device.id, state: "deleted" }, + ]); + }); + + it("quarantines a fresh device whose delete fails, keeps it ungrantable, and reports strategy delete", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("destroy", 1, new DriverCrashError("delete exploded")); + const harness = await createHarness({ + driver, + identity: freshIos, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 3, + }, + }); + const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); + + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: first.device.id, state: "quarantined" }, + ]); + expect( + harness.bus.replay().filter((event) => event.event === "device.purge-failed"), + ).toMatchObject([ + { + payload: { + attemptedStrategy: "delete", + deviceId: first.device.id, + error: "DriverCrashError: delete exploded", + leaseId: first.lease.id, + }, + }, + ]); + const second = await harness.engine.request(request, { + noWait: true, + ownerId: "second", + requesterId: "second", + }); + expect(second.device.id).not.toBe(first.device.id); + }); + + it("still deletes a device created under fresh after the configuration changes to reusable", async () => { + const state = await openTestState(); + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + let nextId = 1; + const idGenerator = { generate: () => `${nextId++}` }; + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const systemStats = new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }); + const beforeRegistry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator, + leaseIdentity: freshIos, + state, + }); + const before = createTestEngine({ + clock, + ...testComponentWiring({ clock, drivers: [driver], eventBus: bus, registry: beforeRegistry }), + config: config({ identity: freshIos }), + drivers: [driver], + eventBus: bus, + idGenerator, + registry: beforeRegistry, + state: testStateOf(beforeRegistry), + systemStats, + }); + const granted = await before.request(request, { ownerId: "a", requesterId: "a" }); + before.dispose(); + + const reusable = { android: "reusable", ios: "reusable" } as const; + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator, + leaseIdentity: reusable, + state, + }); + const after = createTestEngine({ + ...testComponentWiring({ + clock: clock, + drivers: [driver], + eventBus: bus, + registry: registry, + }), + clock, + config: config({ identity: reusable, defaultTtlMs: 14_400_000 }), + drivers: [driver], + eventBus: bus, + idGenerator, + registry, + state: testStateOf(registry), + systemStats, + }); + await after.convergeRunningCapacity(); + await after.release(granted.lease.id, "explicit"); + await after.settle(); + after.dispose(); + + expect(registry.snapshot.devices).toMatchObject([ + { id: granted.device.id, leaseIdentity: "fresh", state: "deleted" }, + ]); + expect(driver.calls.map((call) => call.operation)).not.toContain("reclaim"); + }); + + it("destroys a device created under reusable too, and grants the next requester a new device", async () => { + const harness = await createHarness(); + const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); + + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + const second = await harness.engine.request(request, { + ownerId: "second", + requesterId: "second", + }); + + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: first.device.id, leaseIdentity: "reusable", state: "deleted" }, + { id: second.device.id, leaseIdentity: "reusable", state: "leased" }, + ]); + expect(second.device.id).not.toBe(first.device.id); + const operations = harness.driver.calls.map((call) => call.operation); + expect(operations).not.toContain("reclaim"); + expect(operations.filter((operation) => operation === "destroy")).toHaveLength(1); + }); + + async function restartAfterCrash(crashedIn: "reclaiming" | "shutdown") { + const state = await openTestState(); + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + let nextId = 1; + const idGenerator = { generate: () => `${nextId++}` }; + // why: One driver across both processes: the simulator outlives the daemon. + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const crashed = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator, + leaseIdentity: freshIos, + state, + }); + const driverDevice = await driver.makeReady( + await driver.provision({ ...request, physical: false }), + { + mode: "full", + purpose: "prepare", + }, + ); + const device = await crashed.registerDevice({ + driverData: driverDevice.driverData, + driverDeviceId: driverDevice.deviceId, + provisionDuration: 0, + spec: { ...request, physical: false }, + }); + await crashed.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await crashed.createLease({ + deviceId: device.id, + ownerId: "gone", + requesterId: "gone", + ttlDeadline: 60_000, + ttlMs: 60_000, + }); + await crashed.beginRelease(lease.id); + if (crashedIn === "shutdown") { + await driver.shutdown(driverDevice); + await crashed.completeReclaimWithoutPurge(device.id); + } + + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator, + leaseIdentity: freshIos, + state, + }); + const engine = createTestEngine({ + ...testComponentWiring({ + clock: clock, + drivers: [driver], + eventBus: bus, + registry: registry, + }), + clock, + config: config({ identity: freshIos }), + drivers: [driver], + eventBus: bus, + idGenerator, + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + }); + const operationsBeforeStart = driver.calls.length; + return { + device, + driver, + engine, + operationsSinceStart: () => + driver.calls.slice(operationsBeforeStart).map((call) => call.operation), + registry, + }; + } + + it.each([ + { + crashedIn: "reclaiming" as const, + operations: ["listManaged", "shutdown", "destroy", "listSources"], + title: "while it was reclaiming", + }, + { + crashedIn: "shutdown" as const, + operations: ["listManaged", "destroy", "listSources"], + title: "after its shutdown commit and before its delete", + }, + ])( + "deletes a fresh device on the next start when the daemon crashed $title", + async ({ crashedIn, operations }) => { + const restarted = await restartAfterCrash(crashedIn); + expect(restarted.registry.snapshot.devices).toMatchObject([{ state: crashedIn }]); + + await restarted.engine.convergeRunningCapacity(); + + expect(restarted.registry.snapshot.devices).toMatchObject([ + { id: restarted.device.id, state: "deleted" }, + ]); + expect(restarted.operationsSinceStart()).toEqual(operations); + }, + ); +}); diff --git a/src/leasing/create-leasing-gateway-and-retry.test.ts b/src/leasing/create-leasing-gateway-and-retry.test.ts new file mode 100644 index 000000000..525b78885 --- /dev/null +++ b/src/leasing/create-leasing-gateway-and-retry.test.ts @@ -0,0 +1,399 @@ +import { describe, expect, it } from "vitest"; +import { type LeaseGrant, DriverCrashError } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { type EventMap } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { request, config, createHarness, flush, settledOrPending } from "./test-create-leasing.js"; +import { NoCapacityError, ProvisionFailedError, QueueTimeoutError } from "./index.js"; + +describe("createLeasing: a gateway dispatch is a probe (ADR 0021)", () => { + type Harness = Awaited>; + + const holder = { ownerId: "holder", requesterId: "holder" }; + const probeOf = (requesterId: string, extra: Record = {}) => ({ + fleetRequestId: "req_gw1", + noWait: true, + ownerId: requesterId, + requesterId, + ...extra, + }); + + function payloads< + Name extends + | "lease.declined" + | "lease.rejected" + | "lease.requested" + | "lease.queued" + | "lease.granted", + >(harness: Harness, name: Name): Array { + return harness.bus + .replay() + .filter((event) => event.event === name) + .map((event) => event.payload as EventMap[Name]); + } + + async function admittedId(harness: Harness, requesterId: string): Promise { + return harness.registry.leaseRequests().find((record) => record.requesterId === requesterId) + ?.id; + } + + interface Declined { + readonly harness: Harness; + readonly payload: unknown; + readonly check?: () => void; + } + + const declinedCases: ReadonlyArray Promise]> = [ + [ + "no-wait", + async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + await expect(harness.engine.request(request, probeOf("gw:one"))).rejects.toBeInstanceOf( + NoCapacityError, + ); + return { + harness, + payload: { + fleetRequestId: "req_gw1", + reason: "no-wait", + requestId: await admittedId(harness, "gw:one"), + requestSpec: request, + requester: "gw:one", + }, + }; + }, + ], + [ + "unresolvable-spec", + async () => { + const harness = await createHarness(); + const odd = { ...request, model: "iPhone 99 Imaginary", osVersion: ">=18" }; + await settledOrPending(harness.engine.request(odd, probeOf("gw:odd"))); + return { + harness, + payload: expect.objectContaining({ + fleetRequestId: "req_gw1", + reason: "unresolvable-spec", + requestSpec: odd, + }), + }; + }, + ], + [ + "already-leased", + async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.hangMakeReady(); + const harness = await createHarness({ driver }); + void harness.engine.request(request, probeOf("gw:one")).catch(() => undefined); + await expect + .poll(() => + harness.registry.leaseRequests().find((record) => record.requesterId === "gw:one"), + ) + .toMatchObject({ state: "open" }); + await expect(harness.engine.request(request, probeOf("gw:one"))).rejects.toMatchObject({ + name: "RequesterAlreadyLeasedError", + }); + return { + harness, + payload: { + fleetRequestId: "req_gw1", + reason: "already-leased", + requestId: expect.stringMatching(/^req_/), + requestSpec: request, + requester: "gw:one", + }, + }; + }, + ], + [ + "lease-id-taken", + async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }, + }); + await harness.engine.request(request, { ...holder, leaseId: "myid" }); + await settledOrPending( + harness.engine.request(request, probeOf("gw:two", { leaseId: "myid" })), + ); + return { + harness, + payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "lease-id-taken" }), + }; + }, + ], + [ + "lease-ended", + async () => { + const harness = await createHarness(); + await harness.engine.request(request, { ...holder, leaseId: "myid" }); + await harness.engine.release("myid", "explicit"); + await expect( + harness.engine.request(request, probeOf("gw:two", { leaseId: "myid" })), + ).rejects.toMatchObject({ code: "LEASE_ENDED" }); + return { + harness, + payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "lease-ended" }), + }; + }, + ], + [ + "boot-timeout", + async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("makeReady", 1, new Error("boot failed")); + const harness = await createHarness({ driver }); + const stages: string[] = []; + await settledOrPending( + harness.engine.request(request, { + ...probeOf("gw:slow"), + onProgress: (progress) => stages.push(progress.stage), + }), + ); + return { + check: () => expect(stages.length).toBeGreaterThan(0), + harness, + payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "boot-timeout" }), + }; + }, + ], + [ + "daemon-restarted", + async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.hangMakeReady(); + const before = await createHarness({ driver }); + void before.engine.request(request, probeOf("gw:open")).catch(() => undefined); + await expect + .poll(() => + before.registry.leaseRequests().find((record) => record.requesterId === "gw:open"), + ) + .toMatchObject({ fleetRequestId: "req_gw1", state: "open" }); + const requestId = await admittedId(before, "gw:open"); + const after = await createHarness({ state: before.state }); + await after.engine.convergeRunningCapacity(); + return { + harness: after, + payload: { + fleetRequestId: "req_gw1", + reason: "daemon-restarted", + requestId, + requestSpec: request, + requester: "gw:open", + }, + }; + }, + ], + ]; + + it.each(declinedCases)( + "emits one lease.declined with its fleetRequestId and reason %s, and no lease.rejected, for a probe refused that way", + async (_reason, run) => { + const { check, harness, payload } = await run(); + + expect(payloads(harness, "lease.declined")).toEqual([payload]); + expect(payloads(harness, "lease.rejected")).toEqual([]); + check?.(); + }, + ); + + it("still emits lease.rejected no-wait, and no lease.declined, for a local noWait request with no room", async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + + await expect( + harness.engine.request(request, { noWait: true, ownerId: "local", requesterId: "local" }), + ).rejects.toBeInstanceOf(NoCapacityError); + + expect(payloads(harness, "lease.rejected")).toEqual([ + expect.objectContaining({ reason: "no-wait", requester: "local" }), + ]); + expect(payloads(harness, "lease.declined")).toEqual([]); + }); + + it("carries a probe's fleetRequestId on its lease.requested and lease.granted, and none on a local request's", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 2, + }, + }); + + await harness.engine.request(request, probeOf("gw:one")); + await harness.engine.request(request, { ownerId: "local", requesterId: "local" }); + + const [requestedProbe, requestedLocal] = payloads(harness, "lease.requested"); + const [grantedProbe, grantedLocal] = payloads(harness, "lease.granted"); + expect(requestedProbe).toMatchObject({ fleetRequestId: "req_gw1", requester: "gw:one" }); + expect(grantedProbe).toMatchObject({ fleetRequestId: "req_gw1", requester: "gw:one" }); + expect(requestedLocal).toMatchObject({ requester: "local" }); + expect(requestedLocal).not.toHaveProperty("fleetRequestId"); + expect(grantedLocal).toMatchObject({ requester: "local" }); + expect(grantedLocal).not.toHaveProperty("fleetRequestId"); + }); + + it.each([ + { name: "a number", maxPerRequester: undefined }, + { name: "a per-platform object", maxPerRequester: { ios: 1, android: 1 } }, + ])( + "a worker grants a probe for a requester that already holds a lease from an earlier probe, lease.maxPerRequester being $name", + async ({ maxPerRequester }) => { + const harness = await createHarness({ + ...(maxPerRequester === undefined ? {} : { lease: { maxPerRequester } }), + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }, + }); + const first = await harness.engine.request(request, probeOf("gw:one")); + + const second = await harness.engine.request(request, probeOf("gw:one")); + + expect(second.lease.id).not.toBe(first.lease.id); + expect( + harness.registry.snapshot.leases.filter((lease) => lease.requesterId === "gw:one"), + ).toHaveLength(2); + }, + ); + + it("declines, rather than rejects, a request carrying fleetRequestId whose wait in the queue times out", async () => { + // why: The dispatcher only lets a probe in with noWait, so none is ever queued; this reaches the + // queue's timeout site directly, which applies the same rule as every other. + const harness = await createHarness(); + await harness.engine.request(request, holder); + const waiting = harness.engine.request(request, { + ...probeOf("gw:late", { noWait: false }), + timeoutMs: 10, + }); + await flush(); + harness.clock.advance(10); + + expect(await settledOrPending(waiting)).toBeInstanceOf(QueueTimeoutError); + + expect(payloads(harness, "lease.declined")).toEqual([ + expect.objectContaining({ fleetRequestId: "req_gw1", reason: "timeout" }), + ]); + expect(payloads(harness, "lease.rejected")).toEqual([]); + }); + + describe("a provision that fails twice", () => { + function failingTwice() { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("provision", 1, new DriverCrashError("simulator exited")); + driver.failOn("provision", 2, new DriverCrashError("simulator exited again")); + return driver; + } + + it("declines a probe with reason no-wait, answers PROVISION_FAILED with the driver's message, and emits no lease.queued", async () => { + const harness = await createHarness({ driver: failingTwice() }); + + const outcome = await settledOrPending(harness.engine.request(request, probeOf("gw:one"))); + + expect(outcome).toBeInstanceOf(ProvisionFailedError); + expect(outcome).toMatchObject({ message: "simulator exited again" }); + expect(payloads(harness, "lease.declined")).toEqual([ + expect.objectContaining({ fleetRequestId: "req_gw1", reason: "no-wait" }), + ]); + expect(payloads(harness, "lease.queued")).toEqual([]); + expect(payloads(harness, "lease.rejected")).toEqual([]); + }); + + it("rejects a local request in the same case with PROVISION_FAILED, lists nothing and emits no lease.queued", async () => { + const harness = await createHarness({ driver: failingTwice() }); + + const outcome = await settledOrPending( + harness.engine.request(request, { ownerId: "local", requesterId: "local" }), + ); + + expect(outcome).toBeInstanceOf(ProvisionFailedError); + expect(outcome).toMatchObject({ message: "simulator exited again" }); + expect(harness.engine.waitingRequests()).toEqual([]); + expect(payloads(harness, "lease.queued")).toEqual([]); + expect(payloads(harness, "lease.declined")).toEqual([]); + }); + }); +}); + +describe("createLeasing after a quarantine retry (ADR 0022 §2)", () => { + it("grants a request that waits at capacity a new simulator once the retry has deleted the quarantined one", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + const harness = await createHarness({ driver }); + const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); + const waiting = harness.engine.request(request, { ownerId: "second", requesterId: "second" }); + await flush(); + + harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); + await flush(); + + const granted = (await settledOrPending(waiting)) as LeaseGrant; + expect(granted).toMatchObject({ lease: { requesterId: "second" } }); + expect(granted.device.id).not.toBe(first.device.id); + expect( + harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, + ).toBe("deleted"); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("reports a warm target of one for the quarantined simulator's spec as ready 1 once the retry has deleted it and the pool has created a new one", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + const harness = await createHarness({ + driver, + warmPool: { targets: [{ count: 1, model: "iPhone 16", osVersion: "26.5", platform: "ios" }] }, + }); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await harness.engine.settle(); + const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); + expect(harness.engine.core.warmPoolReader.figures().targets[0]?.ready).toBe(0); + + harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); + await flush(); + await harness.engine.settle(); + + expect( + harness.bus + .replay() + .filter((event) => event.event === "device.deleted") + .map((event) => (event.payload as { deviceId: string }).deviceId), + ).toEqual([first.device.id]); + expect( + harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, + ).toBe("deleted"); + expect(harness.engine.core.warmPoolReader.figures().targets).toMatchObject([ + { count: 1, ready: 1 }, + ]); + }); +}); diff --git a/src/leasing/create-leasing-grant-and-queue.test.ts b/src/leasing/create-leasing-grant-and-queue.test.ts new file mode 100644 index 000000000..854677c04 --- /dev/null +++ b/src/leasing/create-leasing-grant-and-queue.test.ts @@ -0,0 +1,717 @@ +import { describe, expect, it } from "vitest"; +import { + BootTimeoutError, + type DeviceSpec, + type DriverEstimate, + DriverCrashError, +} from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { + request, + createHarness, + seedReady, + flush, + settledOrPending, +} from "./test-create-leasing.js"; +import { NoCapacityError, QueueTimeoutError, RequestCancelledError } from "./index.js"; + +describe("createLeasing", () => { + it("shuts down an unleased ready device over maxRunning at startup with initiator warm-pool, never a leased one, and is idempotent", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 1 }, + maxRunning: 1, + }, + }); + const leasedDevice = await seedReady(harness); + const unleasedDevice = await seedReady(harness); + await harness.registry.createLease({ + deviceId: leasedDevice.id, + requesterId: "active", + ownerId: "active", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + + await harness.engine.convergeRunningCapacity(); + await harness.engine.settle(); + await harness.engine.convergeRunningCapacity(); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices).toMatchObject([ + { id: leasedDevice.id, state: "leased" }, + { id: unleasedDevice.id, state: "shutdown" }, + ]); + expect(harness.driver.calls.filter((call) => call.operation === "shutdown")).toHaveLength(1); + expect(harness.engine.runningCapacity.global.overLimit).toBe(false); + expect(harness.bus.replay().filter((event) => event.event === "device.shutdown")).toMatchObject( + [{ payload: { deviceId: unleasedDevice.id, initiator: "warm-pool" } }], + ); + }); + + it("retains in-limit warm devices at startup and never boots shutdown inventory", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 2, + }, + }); + const warm = await seedReady(harness); + const shutdown = await seedReady(harness); + await harness.registry.transitionDevice(shutdown.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: shutdown.id, initiator: "test" }, + }); + const callsBefore = harness.driver.calls.length; + + await harness.engine.convergeRunningCapacity(); + + expect(harness.registry.snapshot.devices.find((item) => item.id === warm.id)?.state).toBe( + "ready", + ); + expect(harness.registry.snapshot.devices.find((item) => item.id === shutdown.id)?.state).toBe( + "shutdown", + ); + expect(harness.driver.calls.slice(callsBefore).map((call) => call.operation)).toEqual([ + "listManaged", + "listSources", + ]); + }); + + it("shuts an orphaned reclaiming device down and destroys it through its driver", async () => { + const harness = await createHarness(); + const device = await seedReady(harness); + const lease = await harness.registry.createLease({ + deviceId: device.id, + requesterId: "former", + ownerId: "former", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await harness.registry.beginRelease(lease.id); + + await harness.engine.convergeRunningCapacity(); + + expect(harness.registry.snapshot.devices[0]?.state).toBe("deleted"); + const operations = harness.driver.calls.map((call) => call.operation); + expect(operations).toContain("shutdown"); + expect(operations).toContain("destroy"); + }); + + it("reports unavoidable leased running overage", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 1 }, + maxRunning: 1, + }, + }); + for (const requesterId of ["one", "two"]) { + const device = await seedReady(harness); + await harness.registry.createLease({ + deviceId: device.id, + ownerId: requesterId, + requesterId, + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + } + + await harness.engine.convergeRunningCapacity(); + + expect(harness.driver.calls.filter((call) => call.operation === "shutdown")).toHaveLength(0); + expect(harness.engine.runningCapacity.global.overLimit).toBe(true); + await expect( + harness.engine.request(request, { + noWait: true, + ownerId: "three", + requesterId: "three", + }), + ).rejects.toBeInstanceOf(NoCapacityError); + }); + + it("holds one running reservation across provision and readiness", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + platform: "ios", + }); + driver.hangMakeReady(); + const harness = await createHarness({ driver }); + + const first = harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const second = harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + await flush(); + + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); + expect(harness.engine.runningCapacity.global.reserved).toBe(1); + expect(harness.engine.queueDepth).toBe(1); + + driver.releaseMakeReady(); + const grant = await first; + await harness.engine.release(grant.lease.id, "explicit"); + await expect(second).resolves.toMatchObject({ + lease: { ownerId: "agent-2", requesterId: "agent-2" }, + }); + }); + + it("starts exactly one of two existing shutdown devices at running capacity one", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const harness = await createHarness({ driver }); + for (let index = 0; index < 2; index += 1) { + const device = await seedReady(harness); + await harness.registry.transitionDevice(device.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: device.id, initiator: "test" }, + }); + } + const seededBoots = driver.calls.filter((call) => call.operation === "makeReady").length; + driver.hangMakeReady(); + + void harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + void harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + await flush(); + + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength( + seededBoots + 1, + ); + expect(harness.engine.queueDepth).toBe(1); + driver.releaseMakeReady(); + }); + + it("serializes simultaneous requests so one device of capacity starts exactly one provision", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { provision: 20 }, + platform: "ios", + }); + const harness = await createHarness({ driver }); + + const first = harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const second = harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + await flush(); + + expect(harness.engine.queueDepth).toBe(1); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + clock.advance(20); + await first; + await flush(); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + + const released = (await first).device.id; + await harness.engine.release((await first).lease.id, "explicit"); + await flush(); + clock.advance(20); + const granted = await second; + expect(granted).toMatchObject({ lease: { ownerId: "agent-2", requesterId: "agent-2" } }); + expect(granted.device.id).not.toBe(released); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("grants an existing matching ready device and does not match a different spec", async () => { + const harness = await createHarness(); + const matching = await seedReady(harness); + await seedReady(harness, { + physical: false, + model: "iPhone SE", + osVersion: "26.5", + platform: "ios", + }); + const progress: string[] = []; + const provisions = () => + harness.driver.calls.filter((call) => call.operation === "provision").length; + const provisionedBefore = provisions(); + + const grant = await harness.engine.request(request, { + onProgress: (update) => progress.push(update.stage), + requesterId: "agent-1", + ownerId: "agent-1", + }); + + expect(grant.device.id).toBe(matching.id); + expect(progress).toEqual([]); + expect(provisions()).toBe(provisionedBefore); + }); + + it("provisions and makes a device ready when capacity is available, returning progress estimates", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + estimateMs: { boot: 20, provision: 10, reclaim: 34 }, + platform: "ios", + }); + const harness = await createHarness({ driver }); + + const progress: string[] = []; + const grant = await harness.engine.request(request, { + onProgress: (update) => progress.push(update.stage), + requesterId: "agent-1", + ownerId: "agent-1", + }); + + // why: `estimatedReclaimMs` is the one figure describing work still ahead of the holder -- + // what releasing this device will cost -- rather than work already done to grant it. + expect(grant).toMatchObject({ + device: { spec: request, state: "leased" }, + timing: { + estimatedBootMs: 20, + estimatedProvisionMs: 10, + estimatedReadyMs: 30, + estimatedReclaimMs: 34, + }, + }); + expect(driver.calls.map((call) => call.operation)).toEqual([ + "resolveSpec", + "provision", + "makeReady", + ]); + }); + + it("prices the eventual reclaim even when a warm device is granted with no work at all", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + estimateMs: { boot: 20, provision: 10, reclaim: 34 }, + platform: "ios", + }); + const harness = await createHarness({ driver }); + await seedReady(harness); + + const grant = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + + expect(grant.timing).toEqual({ + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 34, + }); + }); + + it("reports only the selected work as it begins, quoting a shut-down device's own prepare-boot figure", async () => { + const clock = new FakeClock(1_000); + // why: A real driver quotes a persisted-baseline boot below its cold figure; the fake does so only + // for an estimate that names the device and the prepare purpose. + class DeviceQuotingDriver extends FakeDriver { + override estimate(estimate: DriverEstimate, spec: DeviceSpec): number { + if (estimate.operation === "boot" && estimate.purpose === "prepare") { + return estimate.device === undefined ? 99 : 4; + } + return super.estimate(estimate, spec); + } + } + const driver = new DeviceQuotingDriver({ + availableOsVersions: ["26.5"], + clock, + estimateMs: { boot: 20, provision: 10, reclaim: 15 }, + platform: "ios", + }); + const harness = await createHarness({ driver, idleShutdownAfterMs: 0 }); + const provisioned: string[] = []; + + const provisionedGrant = await harness.engine.request(request, { + onProgress: (progress) => provisioned.push(progress.stage), + requesterId: "provisioned", + ownerId: "provisioned", + }); + expect(provisioned).toEqual(["provisioning", "booting"]); + await harness.engine.release(provisionedGrant.lease.id, "explicit"); + await harness.engine.settle(); + + // why: a released virtual device is destroyed, so the shut-down device is seeded. + const shutdown = await seedReady(harness); + await harness.registry.transitionDevice(shutdown.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: shutdown.id, initiator: "test" }, + }); + const booted: string[] = []; + const bootEtas: (number | undefined)[] = []; + const bootedGrant = await harness.engine.request(request, { + onProgress: (progress) => { + booted.push(progress.stage); + bootEtas.push(progress.stage === "booting" ? progress.etaMs : undefined); + }, + requesterId: "shutdown", + ownerId: "shutdown", + }); + expect(booted).toEqual(["booting"]); + expect(bootEtas).toEqual([4]); + expect(bootedGrant.timing.estimatedBootMs).toBe(4); + }); + + it("grants a shut-down device with the cold boot figure when its own boot estimate rejects", async () => { + const clock = new FakeClock(1_000); + class RejectingDeviceEstimateDriver extends FakeDriver { + override estimate(estimate: DriverEstimate, spec: DeviceSpec): number { + if (estimate.operation === "boot" && estimate.device !== undefined) { + return Promise.reject(new Error("emulator -version failed")) as unknown as number; + } + return super.estimate(estimate, spec); + } + } + const driver = new RejectingDeviceEstimateDriver({ + availableOsVersions: ["26.5"], + clock, + estimateMs: { boot: 20, provision: 10, reclaim: 15 }, + platform: "ios", + }); + const harness = await createHarness({ driver, idleShutdownAfterMs: 0 }); + const shutdown = await seedReady(harness); + await harness.registry.transitionDevice(shutdown.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: shutdown.id, initiator: "test" }, + }); + + const granted = harness.engine + .request(request, { requesterId: "b", ownerId: "b" }) + .then((grant) => grant.timing.estimatedBootMs); + // why: Without the fallback the waiter hangs, so the race turns that hang into a named failure. + const outcome = await Promise.race([ + granted, + new Promise((resolve) => setTimeout(() => resolve("still pending"), 1_000)), + ]); + + expect(outcome).toBe(20); + }); + + it("reports queue insertion without speculative work and isolates callback failures", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const harness = await createHarness({ driver }); + const holder = await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + const progress: string[] = []; + const queued = harness.engine.request(request, { + onProgress: (update) => progress.push(update.stage), + requesterId: "queued", + ownerId: "queued", + }); + await flush(); + expect(progress).toEqual(["queued"]); + + await expect( + harness.engine.request(request, { + noWait: true, + onProgress: () => progress.push("unexpected"), + requesterId: "no-wait", + ownerId: "no-wait", + }), + ).rejects.toBeInstanceOf(NoCapacityError); + expect(progress).toEqual(["queued"]); + + await harness.engine.release(holder.lease.id, "explicit"); + await queued; + + const callbackFailure = await createHarness(); + await expect( + callbackFailure.engine.request(request, { + onProgress: () => { + throw new Error("client disconnected"); + }, + requesterId: "throwing-callback", + ownerId: "throwing-callback", + }), + ).resolves.toMatchObject({ device: { state: "leased" } }); + expect(callbackFailure.driver.calls.map((call) => call.operation)).toContain("makeReady"); + }); + + it("cancels a queued request through the QueueControl facade and frees the requester for a later grant", async () => { + const harness = await createHarness(); + const holder = await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + const queued = harness.engine.request(request, { + ownerId: "queued", + requesterId: "queued", + }); + await flush(); + + await expect(harness.engine.cancelPending("nobody")).resolves.toBe("not-found"); + await expect(harness.engine.cancelPending("queued")).resolves.toBe("cancelled"); + await expect(queued).rejects.toBeInstanceOf(RequestCancelledError); + + await harness.engine.release(holder.lease.id, "explicit"); + await expect( + harness.engine.request(request, { ownerId: "queued", requesterId: "queued" }), + ).resolves.toMatchObject({ lease: { ownerId: "queued", requesterId: "queued" } }); + }); + + it("stamps queue.changed with the wait-queue module when a request queues and when it leaves", async () => { + const harness = await createHarness(); + const modules: { depth: number; module: string }[] = []; + harness.bus.subscribe("queue.changed", (envelope) => + modules.push({ depth: envelope.payload.depth, module: envelope.module }), + ); + const holder = await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + const queued = harness.engine.request(request, { + ownerId: "queued", + requesterId: "queued", + }); + await flush(); + expect(modules).toEqual([{ depth: 1, module: "wait-queue" }]); + + await harness.engine.release(holder.lease.id, "explicit"); + await queued; + expect(modules).toEqual([ + { depth: 1, module: "wait-queue" }, + { depth: 0, module: "wait-queue" }, + ]); + }); + + it("queues at capacity in FIFO order across three waiters", async () => { + const harness = await createHarness(); + const first = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + // why: Each waiter records its grant and hands the device straight back, so the queue drains in + // whatever order it serves and the assertion below sees that order rather than hanging. + const granted: string[] = []; + const waiters = ["agent-2", "agent-3", "agent-4"].map((requesterId) => + harness.engine.request(request, { ownerId: requesterId, requesterId }).then(async (grant) => { + granted.push(grant.lease.requesterId); + await harness.engine.release(grant.lease.id, "explicit"); + }), + ); + await flush(); + + await harness.engine.release(first.lease.id, "explicit"); + await Promise.all(waiters); + + expect(granted).toEqual(["agent-2", "agent-3", "agent-4"]); + }); + + it("wakes exactly the queue head on release and grants it a new device, not the destroyed one", async () => { + const harness = await createHarness(); + const first = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const second = harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + const third = harness.engine.request(request, { + ownerId: "agent-3", + requesterId: "agent-3", + }); + await flush(); + + await harness.engine.release(first.lease.id, "explicit"); + const secondGrant = await second; + await flush(); + + expect(secondGrant.device.id).not.toBe(first.device.id); + expect(harness.registry.snapshot.leases).toHaveLength(1); + expect(harness.registry.snapshot.leases[0]?.requesterId).toBe("agent-2"); + expect(harness.driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + await harness.engine.release(secondGrant.lease.id, "explicit"); + await expect(third).resolves.toMatchObject({ + lease: { ownerId: "agent-3", requesterId: "agent-3" }, + }); + }); + + it("provisions a new device for the queue head instead of deadlocking at capacity", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const harness = await createHarness({ driver }); + const first = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const queued = harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + await flush(); + + await harness.engine.release(first.lease.id, "explicit"); + + const granted = await queued; + expect(granted.device.id).not.toBe(first.device.id); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2); + }); + + it("rejects a timed-out queue entry and skips it on a later release", async () => { + const harness = await createHarness(); + const first = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const progress: string[] = []; + const timedOut = harness.engine.request(request, { + onProgress: (update) => progress.push(update.stage), + requesterId: "agent-2", + ownerId: "agent-2", + timeoutMs: 10, + }); + const next = harness.engine.request(request, { + ownerId: "agent-3", + requesterId: "agent-3", + }); + await flush(); + + // why: Each outcome settles against a flush rather than an await, so a waiter left pending + // fails the assertion that names it instead of hanging the test. + harness.clock.advance(10); + expect(await settledOrPending(timedOut)).toBeInstanceOf(QueueTimeoutError); + expect(progress).toEqual(["queued"]); + await harness.engine.release(first.lease.id, "explicit"); + + expect(await settledOrPending(next)).toMatchObject({ + lease: { ownerId: "agent-3", requesterId: "agent-3" }, + }); + }); + + it("expires an un-renewed lease at its deadline and serves the queue with its device", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); + await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + const queued = harness.engine.request(request, { + ownerId: "agent-2", + requesterId: "agent-2", + }); + await flush(); + + harness.clock.advance(9); + await flush(); + expect(await settledOrPending(queued)).toBe("still pending"); + + harness.clock.advance(1); + await flush(); + await expect(queued).resolves.toMatchObject({ + lease: { ownerId: "agent-2", requesterId: "agent-2" }, + }); + }); + + it("renews via the engine, re-arming the expiry timer at the new deadline", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); + const granted = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + + harness.clock.advance(6); + const renewed = await harness.engine.renew(granted.lease.id, 20); + expect(renewed).toMatchObject({ lastRenewedAt: 1_006, ttlMs: 20, ttlDeadline: 1_026 }); + expect(harness.bus.replay().find((event) => event.event === "lease.renewed")?.payload).toEqual({ + leaseId: granted.lease.id, + newDeadline: 1_026, + }); + + // why: The grant-time deadline (1_010) has now passed, but the renewal re-armed the expiry + // timer at the new one, so the lease must still be alive. + harness.clock.advance(4); + await flush(); + expect(harness.registry.snapshot.leases).toHaveLength(1); + + harness.clock.advance(16); + await flush(); + expect(harness.registry.snapshot.leases).toHaveLength(0); + }); + + it("grants at the request's own ttlMs, and re-applies that width on a body-less renew", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); + const granted = await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + ttlMs: 40, + }); + expect(granted.lease).toMatchObject({ ttlMs: 40, ttlDeadline: 1_040 }); + + harness.clock.advance(5); + // why: No TTL named: the lease keeps its own 40ms width rather than snapping back to the + // 10ms `lease.defaultTtlMs` (ADR 0004 §4). + await expect(harness.engine.renew(granted.lease.id)).resolves.toMatchObject({ + ttlMs: 40, + ttlDeadline: 1_045, + }); + }); + + it("retries after a provision failure without leaking capacity", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("provision", 1, new DriverCrashError("simulator exited")); + const harness = await createHarness({ driver }); + + // why: A request that never retried would sit pending; settling against a flush names that. + const grant = await settledOrPending( + harness.engine.request(request, { ownerId: "agent-1", requesterId: "agent-1" }), + ); + + expect(grant).toMatchObject({ device: { state: "leased" } }); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + expect( + harness.registry.snapshot.devices.filter((device) => device.state !== "deleted"), + ).toHaveLength(1); + }); + + it("destroys a registered device and returns BootTimeoutError when readiness fails", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("makeReady", 1, new Error("boot failed")); + const harness = await createHarness({ driver }); + + await expect( + harness.engine.request(request, { ownerId: "agent-1", requesterId: "agent-1" }), + ).rejects.toBeInstanceOf(BootTimeoutError); + + expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); + expect(driver.calls.map((call) => call.operation)).toContain("destroy"); + }); + + it("emits committed happy-path facts in lifecycle order", async () => { + const harness = await createHarness(); + + await harness.engine.request(request, { + ownerId: "agent-1", + requesterId: "agent-1", + }); + + expect(harness.bus.replay().map((event) => event.event)).toEqual([ + "lease.requested", + "device.provisioned", + "device.ready", + "lease.granted", + ]); + }); +}); diff --git a/src/leasing/create-leasing-lease-id.test.ts b/src/leasing/create-leasing-lease-id.test.ts new file mode 100644 index 000000000..7e07061bf --- /dev/null +++ b/src/leasing/create-leasing-lease-id.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, flush, settledOrPending } from "./test-create-leasing.js"; + +describe("createLeasing a lease ID chosen by the requester", () => { + const roomy = { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }; + + function asking(requesterId: string, leaseId?: string, more: Record = {}) { + return { + ownerId: requesterId, + requesterId, + ...(leaseId === undefined ? {} : { leaseId }), + ...more, + }; + } + + async function withWaiting(requesterId: string, leaseId: string) { + const harness = await createHarness(); + await harness.engine.request(request, asking("holder")); + void harness.engine.request(request, asking(requesterId, leaseId)).catch(() => undefined); + await expect + .poll(() => + harness.registry.leaseRequests().find((record) => record.requesterId === requesterId), + ) + .toMatchObject({ state: "open" }); + return harness; + } + + it("a lease request with leaseId gets a lease with exactly that ID", async () => { + const harness = await createHarness(); + + const grant = await harness.engine.request(request, asking("agent-1", "ad-7f3a")); + + expect(grant.lease.id).toBe("ad-7f3a"); + expect(grant.lease).toMatchObject({ idChosenByRequester: true }); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["ad-7f3a"]); + }); + + it("a lease request without leaseId gets an lse_ ID as today", async () => { + const harness = await createHarness(); + + const grant = await harness.engine.request(request, asking("agent-1")); + + expect(grant.lease.id).toMatch(/^lse_/); + expect(grant.lease).toMatchObject({ idChosenByRequester: false }); + }); + + it("renew and release by a caller-chosen ID work", async () => { + const harness = await createHarness(); + await harness.engine.request(request, asking("agent-1", "ad-7f3a")); + + await expect(harness.engine.renew("ad-7f3a", 5_000)).resolves.toMatchObject({ + id: "ad-7f3a", + ttlMs: 5_000, + }); + await harness.engine.release("ad-7f3a", "explicit"); + + expect(harness.registry.snapshot.leases).toEqual([]); + expect( + harness.bus.replay().find((event) => event.event === "lease.released")?.payload, + ).toMatchObject({ + leaseId: "ad-7f3a", + }); + }); + + it("MyID and myid are two different IDs", async () => { + const harness = await createHarness({ limits: roomy }); + + const first = await harness.engine.request(request, asking("agent-1", "myid")); + const second = await harness.engine.request(request, asking("agent-2", "MyID")); + + expect([first.lease.id, second.lease.id]).toEqual(["myid", "MyID"]); + }); + + it("a second request for an ID held by an active lease fails with LEASE_ID_TAKEN", async () => { + const harness = await createHarness({ limits: roomy }); + await harness.engine.request(request, asking("agent-1", "myid")); + + await expect(harness.engine.request(request, asking("agent-2", "myid"))).rejects.toMatchObject({ + leaseId: "myid", + message: "lease ID myid is already in use", + name: "LeaseIdTakenError", + }); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["myid"]); + }); + + it("a second request for an ID held by a waiting request fails with LEASE_ID_TAKEN", async () => { + const harness = await withWaiting("agent-2", "myid"); + + const clash = harness.engine.request(request, asking("agent-3", "myid")); + void clash.catch(() => undefined); + + // why: Settled at once: a request that was let in would wait for the one device instead. + await expect(settledOrPending(clash)).resolves.toMatchObject({ + leaseId: "myid", + name: "LeaseIdTakenError", + }); + }); + + it("a requester that already holds a lease gets REQUESTER_ALREADY_LEASED, not LEASE_ID_TAKEN, even when it repeats its own lease ID", async () => { + const harness = await createHarness({ limits: roomy }); + const first = await harness.engine.request(request, asking("agent-1", "myid")); + + await expect(harness.engine.request(request, asking("agent-1", "myid"))).rejects.toMatchObject({ + existingLeaseId: first.lease.id, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("a LEASE_ID_TAKEN refusal emits lease.rejected with reason lease-id-taken", async () => { + const harness = await createHarness({ limits: roomy }); + await harness.engine.request(request, asking("agent-1", "myid")); + + await harness.engine.request(request, asking("agent-2", "myid")).catch(() => undefined); + + const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); + expect(rejected.map((event) => event.payload)).toEqual([ + { + reason: "lease-id-taken", + requestId: expect.stringMatching(/^req_/), + requester: "agent-2", + requestSpec: request, + }, + ]); + expect(rejected.map((event) => event.module)).toEqual(["lease-acquisition-coordinator"]); + }); + + it("a caller re-sending a chosen lease id whose lease ended within the window gets LEASE_ENDED with the id and when it ended", async () => { + type Harness = Awaited>; + const dayMs = 24 * 60 * 60_000; + const freshDriver = () => + new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const cases: readonly { + readonly name: string; + readonly endIt: (harness: Harness, id: string) => Promise; + readonly outcome: "ended" | "granted" | "taken"; + readonly endedAt?: number; + /** why: the first lease asks for no id (`generated`), or for one the retry does not use (`other`). */ + readonly asked?: "generated" | "other"; + /** why: rows left in `ended_lease_ids` once the retry is answered. */ + readonly rows?: number; + }[] = [ + { + name: "released, and another lease commits after it", + endIt: async (harness) => { + await harness.engine.release("myid", "explicit"); + harness.clock.advance(5_000); + await harness.engine.request(request, asking("agent-3", "other")); + return harness; + }, + outcome: "ended", + endedAt: 1_000, + }, + { + name: "expired", + endIt: async (harness) => { + harness.clock.advance(100_000); + await flush(); + return harness; + }, + outcome: "ended", + endedAt: 101_000, + }, + { + name: "ended by nuke", + endIt: async (harness) => { + await harness.engine.nuke(false); + return harness; + }, + outcome: "ended", + endedAt: 1_000, + }, + { + name: "ended at a daemon start", + endIt: async (harness) => { + const restarted = await createHarness({ driver: freshDriver(), state: harness.state }); + await restarted.engine.convergeRunningCapacity(); + return restarted; + }, + outcome: "ended", + endedAt: 1_000, + }, + { + name: "after a daemon restart", + endIt: async (harness) => { + await harness.engine.release("myid", "explicit"); + return createHarness({ driver: freshDriver(), state: harness.state }); + }, + outcome: "ended", + endedAt: 1_000, + }, + { + name: "past the window", + endIt: async (harness) => { + await harness.engine.release("myid", "explicit"); + harness.clock.advance(dayMs + 1); + return harness; + }, + outcome: "granted", + rows: 0, + }, + { + name: "a lease with a generated id", + asked: "generated", + endIt: async (harness, id) => { + await harness.engine.release(id, "explicit"); + return harness; + }, + outcome: "granted", + rows: 0, + }, + { + name: "never used", + asked: "other", + endIt: async (harness) => harness, + outcome: "granted", + }, + { name: "still held", endIt: async (harness) => harness, outcome: "taken" }, + ]; + + for (const { name, asked, endIt, outcome, endedAt, rows } of cases) { + const first = await createHarness({ lease: { defaultTtlMs: 100_000 }, limits: roomy }); + const granted = await first.engine.request( + request, + asking("agent-1", asked === "generated" ? undefined : asked === "other" ? "other" : "myid"), + ); + const harness = await endIt(first, granted.lease.id); + const again = asked === "other" ? "myid" : granted.lease.id; + harness.clock.advance(1_000); + + const retry = harness.engine.request(request, asking("agent-2", again)); + + if (outcome === "ended") { + await expect(retry, name).rejects.toMatchObject({ + code: "LEASE_ENDED", + details: { endedAt, leaseId: "myid" }, + message: `Lease myid ended at ${new Date(endedAt ?? 0).toISOString()}; choose a new lease id`, + }); + } else if (outcome === "taken") { + await expect(retry, name).rejects.toMatchObject({ name: "LeaseIdTakenError" }); + } else { + await expect(retry, name).resolves.toMatchObject({ lease: { id: again } }); + } + if (rows !== undefined) { + expect( + harness.state.database.prepare("SELECT COUNT(*) AS n FROM ended_lease_ids").get(), + name, + ).toEqual({ n: rows }); + } + } + }); + + it("lease.requested and lease.rejected requestSpec, and the status waiting spec, carry no leaseId", async () => { + const harness = await withWaiting("agent-2", "myid"); + await settledOrPending(harness.engine.request(request, asking("agent-3", "myid"))); + + const requested = harness.bus.replay().filter((event) => event.event === "lease.requested"); + const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); + expect(requested.map((event) => event.payload)).toContainEqual( + expect.objectContaining({ requester: "agent-2", requestSpec: request }), + ); + expect( + rejected.map((event) => (event.payload as { requestSpec: unknown }).requestSpec), + ).toEqual([request]); + expect(harness.engine.waitingRequests().map((entry) => entry.spec)).toEqual([request]); + expect( + harness.registry.leaseRequests().find((record) => record.requesterId === "agent-2")?.request, + ).toEqual(request); + }); + + it("a retry with the same idempotency key and the same leaseId replays the first answer", async () => { + const harness = await createHarness(); + const asker = asking("agent-1", "myid", { idempotencyKey: "key-1" }); + const first = await harness.engine.request(request, asker); + + const retry = await harness.engine.request(request, asker); + + expect(retry).toEqual(first); + expect(retry.lease.id).toBe("myid"); + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); + + it.each([ + ["a different leaseId", "other"], + ["no leaseId", undefined], + ])( + "a retry with the same idempotency key and %s fails with IDEMPOTENCY_CONFLICT", + async (_name, retryId) => { + const harness = await createHarness(); + await harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })); + + await expect( + harness.engine.request(request, asking("agent-1", retryId, { idempotencyKey: "key-1" })), + ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); + }, + ); + + it("a retry with the same idempotency key that adds a leaseId the first call did not have fails with IDEMPOTENCY_CONFLICT", async () => { + const harness = await createHarness(); + const first = await harness.engine.request( + request, + asking("agent-1", undefined, { idempotencyKey: "key-1" }), + ); + + await expect( + harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })), + ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); + await expect( + harness.engine.request(request, asking("agent-1", undefined, { idempotencyKey: "key-1" })), + ).resolves.toEqual(first); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual([first.lease.id]); + }); + + it("after a daemon restart, an ID held only by a waiting request is accepted again", async () => { + const before = await withWaiting("agent-2", "myid"); + // why: Started again with room to spare, so the only thing that could refuse "myid" is the ID. + const after = await createHarness({ state: before.state, limits: roomy }); + await after.engine.convergeRunningCapacity(); + + await expect(after.engine.request(request, asking("agent-3", "myid"))).resolves.toMatchObject({ + lease: { id: "myid" }, + }); + }); +}); diff --git a/src/leasing/create-leasing-max-per-requester.test.ts b/src/leasing/create-leasing-max-per-requester.test.ts new file mode 100644 index 000000000..4382b9461 --- /dev/null +++ b/src/leasing/create-leasing-max-per-requester.test.ts @@ -0,0 +1,591 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { request, androidRequest, createHarness, settledOrPending } from "./test-create-leasing.js"; + +describe("createLeasing: a requester may hold up to lease.maxPerRequester leases (ADR 0023)", () => { + const roomy = { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 4, maxRunning: 4 }, + maxRunning: 5, + }; + const twoDevices = { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 3, + }; + const asking = (requesterId: string, more: Record = {}) => ({ + ownerId: requesterId, + requesterId, + ...more, + }); + const atLimit = (held: number, limit: number, leaseId: string) => + `Requester agent-1 already holds ${String(held)} leases and lease.maxPerRequester is ${String(limit)}; release one (\`simlock release ${leaseId}\`) before requesting another device`; + const waitingMessage = + "Requester agent-1 already has a waiting request; wait for it to be granted or cancel it before requesting another device"; + + it("with no lease.maxPerRequester, a requester holding a lease is refused with today's message, naming that lease, and is granted another once it releases it", async () => { + const harness = await createHarness({ limits: roomy }); + const first = await harness.engine.request(request, asking("agent-1")); + + const refusal = await harness.engine + .request(request, asking("agent-1")) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: first.lease.id, + message: `Requester agent-1 already holds lease ${first.lease.id}; release it (\`simlock release ${first.lease.id}\`) before requesting another device`, + name: "RequesterAlreadyLeasedError", + }); + + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + await expect(harness.engine.request(request, asking("agent-1"))).resolves.toMatchObject({ + lease: { ownerId: "agent-1", requesterId: "agent-1" }, + }); + }); + + it("with no lease.maxPerRequester, a requester with a waiting request is refused with today's message", async () => { + const harness = await createHarness(); + await harness.engine.request(request, asking("holder")); + void harness.engine.request(request, asking("agent-1")).catch(() => undefined); + await expect + .poll(() => + harness.registry.leaseRequests().find((record) => record.requesterId === "agent-1"), + ) + .toMatchObject({ state: "open" }); + + const refusal = await harness.engine + .request(request, asking("agent-1")) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: "Requester already has a lease or pending request: agent-1", + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with lease.maxPerRequester: 2, one requester holds two leases at once", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + + const first = await harness.engine.request(request, asking("agent-1")); + const second = await harness.engine.request(request, asking("agent-1")); + + expect(second.lease.id).not.toBe(first.lease.id); + expect( + harness.registry.snapshot.leases.filter((lease) => lease.requesterId === "agent-1"), + ).toHaveLength(2); + }); + + it("with lease.maxPerRequester: 2, a requester holding two leases is refused, the message naming the limit and the earlier-granted lease", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + await harness.engine.request(request, asking("agent-1", { leaseId: "zeta" })); + harness.clock.advance(5); + await harness.engine.request(request, asking("agent-1", { leaseId: "alpha" })); + + const refusal = await harness.engine + .request(request, asking("agent-1")) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: "zeta", + message: atLimit(2, 2, "zeta"), + name: "RequesterAlreadyLeasedError", + requesterId: "agent-1", + }); + }); + + it("with lease.maxPerRequester: 3, a requester holding one lease and one waiting request is refused with the waiting-request message", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 3 }, limits: twoDevices }); + await harness.engine.request(request, asking("agent-1")); + await harness.engine.request(request, asking("other")); + void harness.engine.request(request, asking("agent-1")).catch(() => undefined); + await expect + .poll(() => + harness.registry + .leaseRequests() + .filter((record) => record.requesterId === "agent-1" && record.state === "open"), + ) + .toHaveLength(1); + + const refusal = await harness.engine + .request(request, asking("agent-1")) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: waitingMessage, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with lease.maxPerRequester per platform, a requester holding an iOS lease is granted an Android one", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: { ios: 1, android: 1 } }, + limits: roomy, + }); + const first = await harness.engine.request(request, asking("agent-1")); + + const second = await harness.engine.request(androidRequest, asking("agent-1")); + + expect(first.device.spec.platform).toBe("ios"); + expect(second.device.spec.platform).toBe("android"); + }); + + it.each([ + { + name: "limit 1 on iOS", + limit: { ios: 1, android: 1 }, + held: [request], + next: request, + message: (ids: string[]) => + `Requester agent-1 already holds iOS lease ${ids[0]}; release it (\`simlock release ${ids[0]}\`) before requesting another iOS device`, + }, + { + name: "an Android platform the object leaves out, so its limit is 1", + limit: { ios: 2 }, + held: [androidRequest], + next: androidRequest, + message: (ids: string[]) => + `Requester agent-1 already holds Android lease ${ids[0]}; release it (\`simlock release ${ids[0]}\`) before requesting another Android device`, + }, + { + name: "two iOS leases at an iOS limit of 2", + limit: { ios: 2, android: 1 }, + held: [request, request], + next: request, + message: (ids: string[]) => + `Requester agent-1 already holds 2 iOS leases and lease.maxPerRequester.ios is 2; release one (\`simlock release ${ids[0]}\`) before requesting another iOS device`, + }, + ])( + "with lease.maxPerRequester per platform, a requester at its limit on a platform is refused with that platform's message naming its earliest lease ($name)", + async ({ limit, held, next, message }) => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: limit }, + limits: roomy, + }); + const ids: string[] = []; + for (const spec of held) { + const grant = await harness.engine.request(spec, asking("agent-1")); + ids.push(grant.lease.id); + harness.clock.advance(5); + } + + const refusal = await harness.engine + .request(next, asking("agent-1")) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: ids[0], + message: message(ids), + name: "RequesterAlreadyLeasedError", + }); + }, + ); + + it.each([ + { + name: "an Android request while an iOS one waits is admitted", + limit: { ios: 1, android: 1 }, + iosDevices: 1, + next: androidRequest, + refusal: undefined, + }, + { + name: "an iOS request while an iOS one waits is refused", + limit: { ios: 1, android: 1 }, + iosDevices: 1, + next: request, + refusal: + "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", + }, + { + name: "an iOS request holding one iOS lease and waiting for another under an iOS limit of 2 is refused", + limit: { ios: 2 }, + iosDevices: 2, + next: request, + holdsIos: true, + refusal: + "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", + }, + ])( + "with lease.maxPerRequester per platform, a requester with a waiting request is refused another on that platform only ($name)", + async ({ limit, iosDevices, next, refusal, holdsIos }) => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: limit }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: iosDevices, maxRunning: iosDevices }, + maxRunning: iosDevices + 1, + }, + }); + await harness.engine.request(request, asking("holder")); + await harness.engine.request(androidRequest, asking("holder")); + if (holdsIos === true) await harness.engine.request(request, asking("agent-1")); + void harness.engine.request(request, asking("agent-1")).catch(() => undefined); + const openOf = (requesterId: string) => + harness.registry + .leaseRequests() + .filter((record) => record.requesterId === requesterId && record.state === "open"); + await expect.poll(() => openOf("agent-1")).toHaveLength(1); + + const outcome = harness.engine.request(next, asking("agent-1")).catch((e: unknown) => e); + + await harness.engine.settle(); + const state = await settledOrPending(outcome); + if (refusal === undefined) { + expect(state).toBe("still pending"); + expect(openOf("agent-1")).toHaveLength(2); + } else { + expect(state).toMatchObject({ message: refusal, name: "RequesterAlreadyLeasedError" }); + } + }, + ); + + it.each([ + { + name: "2, waiting for iOS, then Android: admitted", + limit: 2, + granted: [ + ["holder-1", request], + ["holder-2", androidRequest], + ], + waits: request, + next: androidRequest, + refusal: undefined, + }, + { + name: "2, waiting for iOS, then iOS: refused", + limit: 2, + granted: [ + ["holder-1", request], + ["holder-2", androidRequest], + ], + waits: request, + next: request, + refusal: waitingMessage, + }, + { + name: "1, waiting for iOS, then Android: refused", + limit: 1, + granted: [ + ["holder-1", request], + ["holder-2", androidRequest], + ], + waits: request, + next: androidRequest, + refusal: "Requester already has a lease or pending request: agent-1", + }, + { + name: "2, holding iOS and waiting for Android, then iOS: refused", + limit: 2, + granted: [ + ["agent-1", request], + ["holder-2", androidRequest], + ], + waits: androidRequest, + next: request, + refusal: waitingMessage, + }, + ] as const)( + "with a numeric lease.maxPerRequester, waiting and held requests share the limit across platforms ($name)", + async ({ limit, granted, waits, next, refusal }) => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: limit }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + }); + for (const [requesterId, spec] of granted) { + await harness.engine.request(spec, asking(requesterId)); + } + void harness.engine.request(waits, asking("agent-1")).catch(() => undefined); + await expect + .poll(() => + harness.registry + .leaseRequests() + .filter((record) => record.requesterId === "agent-1" && record.state === "open"), + ) + .toHaveLength(1); + + const outcome = harness.engine.request(next, asking("agent-1")).catch((e: unknown) => e); + + await harness.engine.settle(); + const state = await settledOrPending(outcome); + if (refusal === undefined) { + expect(state).toBe("still pending"); + } else { + expect(state).toMatchObject({ message: refusal, name: "RequesterAlreadyLeasedError" }); + } + }, + ); + + it("with lease.maxPerRequester per platform, a gateway probe is refused only by a waiting probe on its own platform", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: { ios: 1, android: 1 } }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + }); + await harness.engine.request(request, asking("holder-1")); + await harness.engine.request(androidRequest, asking("holder-2")); + const probe = (spec: typeof request | typeof androidRequest, id: string) => + harness.engine + .request(spec, asking("gw:1:agent-1", { fleetRequestId: id })) + .catch((e: unknown) => e); + void probe(request, "fleet_1"); + await expect + .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) + .toHaveLength(1); + + const android2 = probe(androidRequest, "fleet_2"); + const ios2 = probe(request, "fleet_3"); + + await expect(ios2).resolves.toMatchObject({ + message: expect.stringContaining("already has a waiting iOS request"), + name: "RequesterAlreadyLeasedError", + }); + await expect(Promise.race([android2, Promise.resolve("waiting")])).resolves.toBe("waiting"); + }); + + it.each([ + { + name: "Android", + waiting: ["ios", "android"], + cancel: "android", + result: "cancelled", + left: ["ios"], + }, + { + name: "none", + waiting: ["ios", "android"], + cancel: undefined, + result: "cancelled", + left: ["android"], + }, + { + name: "iOS with only Android waiting", + waiting: ["android"], + cancel: "ios", + result: "not-found", + left: ["android"], + }, + ] as const)( + "cancelPending names the platform of the waiting request to cancel, and the earliest without one ($name)", + async ({ waiting, cancel, result, left }) => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: { ios: 1, android: 1 } }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + }); + await harness.engine.request(request, asking("holder-1")); + await harness.engine.request(androidRequest, asking("holder-2")); + const specs = { android: androidRequest, ios: request }; + for (const [index, platform] of waiting.entries()) { + void harness.engine.request(specs[platform], asking("agent-1")).catch(() => undefined); + await expect + .poll(() => + harness.registry + .leaseRequests() + .filter((record) => record.requesterId === "agent-1" && record.state === "open"), + ) + .toHaveLength(index + 1); + } + + await harness.engine.settle(); + await expect(harness.engine.cancelPending("agent-1", cancel)).resolves.toBe(result); + + expect( + harness.registry + .leaseRequests() + .filter((record) => record.requesterId === "agent-1" && record.state === "open") + .map((record) => record.request.platform), + ).toEqual(left); + }, + ); + + it("pendingRequestOwner names the owner of the waiting request of the platform it names, and of the earliest without one", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + lease: { maxPerRequester: { ios: 1, android: 1 } }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + }); + await harness.engine.request(request, asking("holder-1")); + await harness.engine.request(androidRequest, asking("holder-2")); + void harness.engine + .request(request, { ownerId: "ios-owner", requesterId: "agent-1" }) + .catch(() => undefined); + await expect + .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) + .toHaveLength(1); + void harness.engine + .request(androidRequest, { ownerId: "android-owner", requesterId: "agent-1" }) + .catch(() => undefined); + await expect + .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) + .toHaveLength(2); + + expect(harness.engine.pendingRequestOwner("agent-1", "android")).toBe("android-owner"); + expect(harness.engine.pendingRequestOwner("agent-1", "ios")).toBe("ios-owner"); + expect(harness.engine.pendingRequestOwner("agent-1")).toBe("ios-owner"); + }); + + it("a refusal at the lease limit emits lease.rejected with reason already-leased", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + await harness.engine.request(request, asking("agent-1")); + await harness.engine.request(request, asking("agent-1")); + + await harness.engine.request(request, asking("agent-1")).catch(() => undefined); + + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload), + ).toEqual([ + { + reason: "already-leased", + requestId: expect.stringMatching(/^req_/), + requester: "agent-1", + requestSpec: request, + }, + ]); + }); + + it("releasing one of two leases lets the requester's next request through", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + const first = await harness.engine.request(request, asking("agent-1")); + await harness.engine.request(request, asking("agent-1")); + await expect(harness.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ + name: "RequesterAlreadyLeasedError", + }); + + await harness.engine.release(first.lease.id, "explicit"); + + await expect(harness.engine.request(request, asking("agent-1"))).resolves.toMatchObject({ + lease: { id: expect.stringMatching(/^lse_/) }, + }); + }); + + it("at the lease limit, a repeat with a used idempotencyKey gets its stored answer", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + const first = await harness.engine.request( + request, + asking("agent-1", { idempotencyKey: "k1" }), + ); + await harness.engine.request(request, asking("agent-1", { idempotencyKey: "k2" })); + + const repeat = await harness.engine.request( + request, + asking("agent-1", { idempotencyKey: "k1" }), + ); + + expect(repeat).toEqual(first); + }); + + it("at the lease limit, a request naming a taken lease id gets REQUESTER_ALREADY_LEASED", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + await harness.engine.request(request, asking("agent-2", { leaseId: "taken" })); + await harness.engine.request(request, asking("agent-1")); + await harness.engine.request(request, asking("agent-1")); + + await expect( + harness.engine.request(request, asking("agent-1", { leaseId: "taken" })), + ).rejects.toMatchObject({ name: "RequesterAlreadyLeasedError" }); + }); + + it("after a restart with lease.maxPerRequester lowered from 3 to 1, a requester's three running leases are all kept, and its next request is refused naming the earliest one", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const before = await createHarness({ + driver, + lease: { maxPerRequester: 3 }, + limits: roomy, + }); + const earliest = await before.engine.request(request, asking("agent-1")); + before.clock.advance(5); + await before.engine.request(request, asking("agent-1")); + await before.engine.request(request, asking("agent-1")); + + const after = await createHarness({ + driver, + state: before.state, + lease: { maxPerRequester: 1 }, + limits: roomy, + }); + await after.engine.convergeRunningCapacity(); + + expect( + after.registry.snapshot.leases.filter((lease) => lease.requesterId === "agent-1"), + ).toHaveLength(3); + await expect(after.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ + existingLeaseId: earliest.lease.id, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("a worker still refuses a local request of a requester at its limit while probes of other requesters are granted", async () => { + const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); + await harness.engine.request(request, asking("agent-1")); + await harness.engine.request(request, asking("agent-1")); + + await expect(harness.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ + name: "RequesterAlreadyLeasedError", + }); + await expect( + harness.engine.request(request, { + fleetRequestId: "req_gw1", + noWait: true, + ownerId: "gw:two", + requesterId: "gw:two", + }), + ).resolves.toMatchObject({ lease: { id: expect.stringMatching(/^lse_/) } }); + }); +}); diff --git a/src/leasing/create-leasing-ram-budget.test.ts b/src/leasing/create-leasing-ram-budget.test.ts new file mode 100644 index 000000000..6b8ca8b1a --- /dev/null +++ b/src/leasing/create-leasing-ram-budget.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "vitest"; +import { type CapacityLimits } from "../core/testing.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { gibibyte, request, createHarness, seedReady } from "./test-create-leasing.js"; +import { NoCapacityError } from "./index.js"; + +describe("createLeasing RAM budget by mode", () => { + const slimRequest = { ...request, mode: "slim" } as const; + const roomy: CapacityLimits = { + android: { maxDevices: 4, maxRunning: 4 }, + ios: { maxDevices: 8, maxRunning: 8 }, + maxRunning: 12, + }; + /** why: With 9 GiB of RAM the budget is 5 GiB: one full iOS device (3 GiB) and two slim ones fit. */ + const ramBudget = { + androidBytesPerDevice: 4 * gibibyte, + iosBytesPerDevice: 3 * gibibyte, + iosSlimBytesPerDevice: gibibyte, + }; + + function operations(driver: FakeDriver, operation: string): number { + return driver.calls.filter((call) => call.operation === operation).length; + } + + it("refuses a full spec, which a slim request on a runtime that cannot be slimmed resolves to, before it is provisioned, and counts it full after it boots", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.4", "26.5"], + clock, + platform: "ios", + slimmableOsVersions: ["26.4"], + }); + const harness = await createHarness({ + driver, + limits: roomy, + ramBudget, + totalRamBytes: 9 * gibibyte, + }); + + const unslimmable = await harness.engine.request(slimRequest, { + ownerId: "unslimmable", + requesterId: "unslimmable", + }); + expect(unslimmable.device.mode).toBe("full"); + expect(harness.engine.ramBudget?.usedBytes).toBe(3 * gibibyte); + + // why: 3 GiB used of 5: the next device needs 3 more and is refused before any provision. + const provisions = operations(driver, "provision"); + await expect( + harness.engine.request(slimRequest, { noWait: true, ownerId: "next", requesterId: "next" }), + ).rejects.toBeInstanceOf(NoCapacityError); + expect(operations(driver, "provision")).toBe(provisions); + + // why: Control: had the first device come up slim, 1 GiB would be used and the next would fit. + const control = await createHarness({ + driver: new FakeDriver({ + availableOsVersions: ["26.4"], + clock: new FakeClock(1_000), + platform: "ios", + slimmableOsVersions: ["26.4"], + }), + limits: roomy, + ramBudget, + totalRamBytes: 9 * gibibyte, + }); + const slimmable = { ...slimRequest, osVersion: "26.4" }; + await control.engine.request(slimmable, { ownerId: "first", requesterId: "first" }); + await expect( + control.engine.request(slimmable, { noWait: true, ownerId: "next", requesterId: "next" }), + ).resolves.toMatchObject({ device: { mode: "slim" } }); + }); + + it("counts a slim-spec device quarantined from provisioning at the full size", async () => { + const harness = await createHarness({ limits: roomy, ramBudget, totalRamBytes: 9 * gibibyte }); + const registered = await harness.registry.registerDevice({ + driverData: { fakeDeviceId: "driver-stalled" }, + driverDeviceId: "driver-stalled", + provisionDuration: 0, + spec: { ...slimRequest, physical: false }, + }); + const quarantined = await harness.registry.enterQuarantine(registered.id, 60_000, { + kind: "purge-failed", + }); + + expect(quarantined.state).toBe("quarantined"); + expect(harness.engine.ramBudget?.usedBytes).toBe(ramBudget.iosBytesPerDevice); + }); + + it("keeps the lease of a device planned as slim whose driver reports full, shuts nothing down, and counts it full", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + mode: "full", + platform: "ios", + slimmableOsVersions: ["26.5"], + }); + const harness = await createHarness({ + driver, + limits: roomy, + ramBudget, + totalRamBytes: 9 * gibibyte, + }); + + const leased = await harness.engine.request(slimRequest, { ownerId: "a", requesterId: "a" }); + + expect(leased.device.mode).toBe("full"); + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual([leased.lease.id]); + expect(operations(driver, "shutdown")).toBe(0); + expect(harness.engine.ramBudget).toEqual({ + limitBytes: 5 * gibibyte, + overLimit: false, + usedBytes: 3 * gibibyte, + }); + }); + + it("over its limit after a restart with larger sizes, creates no device in either mode and boots no shut-down slim device, but grants an idle one; releasing the idle one destroys it and brings it back under", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + knownModels: ["iPhone 16", "iPhone 16 Pro"], + availableOsVersions: ["26.4", "26.5"], + clock, + platform: "ios", + slimmableOsVersions: ["26.4", "26.5"], + }); + const small = { ...ramBudget, iosSlimBytesPerDevice: 0.5 * gibibyte }; + const before = await createHarness({ + driver, + idleShutdownAfterMs: 0, + limits: roomy, + ramBudget: small, + totalRamBytes: 9 * gibibyte, + }); + const idleSpec = { ...slimRequest, osVersion: "26.4" }; + const held = await before.engine.request(slimRequest, { ownerId: "held", requesterId: "held" }); + // why: a released virtual device is destroyed, so the idle devices are seeded rather than released. + const toShutDown = await seedReady(before, { ...slimRequest, physical: false }, "slim"); + const idle = await seedReady(before, { ...idleSpec, physical: false }, "slim"); + await before.engine.cleanup.execute({ + action: "shutdown", + reason: "test", + rule: "test", + target: toShutDown.id, + }); + expect(before.engine.ramBudget?.usedBytes).toBe(1 * gibibyte); + + const after = await createHarness({ + driver, + state: before.state, + idleShutdownAfterMs: 0, + limits: roomy, + ramBudget: { ...ramBudget, iosSlimBytesPerDevice: 3 * gibibyte }, + totalRamBytes: 9 * gibibyte, + }); + await after.engine.convergeRunningCapacity(); + expect(after.engine.ramBudget).toEqual({ + limitBytes: 5 * gibibyte, + overLimit: true, + usedBytes: 6 * gibibyte, + }); + + const provisions = operations(driver, "provision"); + const boots = operations(driver, "makeReady"); + for (const [owner, wanted] of [ + ["new-full", request], + ["new-slim", { ...slimRequest, model: "iPhone 16 Pro" }], + ["boot-slim", slimRequest], + ] as const) { + await expect( + after.engine.request(wanted, { noWait: true, ownerId: owner, requesterId: owner }), + ).rejects.toBeInstanceOf(NoCapacityError); + } + expect(operations(driver, "provision")).toBe(provisions); + expect(operations(driver, "makeReady")).toBe(boots); + expect(after.registry.snapshot.devices.find((d) => d.id === toShutDown.id)?.state).toBe( + "shutdown", + ); + + const reused = await after.engine.request(idleSpec, { + noWait: true, + ownerId: "reuse", + requesterId: "reuse", + }); + expect(reused.device.id).toBe(idle.id); + expect(after.registry.snapshot.leases.map((lease) => lease.deviceId)).toContain(held.device.id); + expect(after.engine.ramBudget?.overLimit).toBe(true); + + await after.engine.release(reused.lease.id, "explicit"); + await after.engine.settle(); + await after.engine.cleanup.execute({ + action: "destroy", + reason: "test", + rule: "test", + target: toShutDown.id, + }); + await after.engine.settle(); + expect(operations(driver, "makeReady")).toBe(boots); + expect(after.engine.ramBudget).toEqual({ + limitBytes: 5 * gibibyte, + overLimit: false, + usedBytes: 3 * gibibyte, + }); + }); +}); diff --git a/src/leasing/create-leasing-rejections.test.ts b/src/leasing/create-leasing-rejections.test.ts new file mode 100644 index 000000000..e977e7f2e --- /dev/null +++ b/src/leasing/create-leasing-rejections.test.ts @@ -0,0 +1,233 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { type EventMap } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, flush, settledOrPending } from "./test-create-leasing.js"; +import { QueueTimeoutError } from "./index.js"; + +describe("createLeasing: lease.rejected names its request", () => { + type Reason = Exclude; + type Rejection = EventMap["lease.rejected"]; + type Harness = Awaited>; + + interface Outcome { + readonly harness: Harness; + readonly requester: string; + readonly requestId?: string | undefined; + } + + const holder = { ownerId: "holder", requesterId: "holder" }; + + async function admittedId(harness: Harness, requesterId: string): Promise { + return harness.registry.leaseRequests().find((record) => record.requesterId === requesterId) + ?.id; + } + + /** why: One case per reason in `EventMap`, so a reason added there without a case fails typecheck. */ + const cases: Record Promise) | "gateway"> = { + timeout: async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + const timedOut = harness.engine.request(request, { + ownerId: "late", + requesterId: "late", + timeoutMs: 10, + }); + await flush(); + harness.clock.advance(10); + await settledOrPending(timedOut); + return { harness, requestId: await admittedId(harness, "late"), requester: "late" }; + }, + "no-wait": async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + await settledOrPending( + harness.engine.request(request, { noWait: true, ownerId: "now", requesterId: "now" }), + ); + return { harness, requestId: await admittedId(harness, "now"), requester: "now" }; + }, + "unresolvable-spec": async () => { + const harness = await createHarness(); + await settledOrPending( + harness.engine.request( + { ...request, osVersion: "1.0" }, + { ownerId: "odd", requesterId: "odd" }, + ), + ); + return { harness, requestId: await admittedId(harness, "odd"), requester: "odd" }; + }, + "no-worker": "gateway", + "worker-failed": "gateway", + "already-leased": async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + await settledOrPending(harness.engine.request(request, holder)); + return { harness, requester: "holder" }; + }, + "lease-id-taken": async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 4, + }, + }); + await harness.engine.request(request, { ...holder, leaseId: "myid" }); + await settledOrPending( + harness.engine.request(request, { + leaseId: "myid", + ownerId: "clash", + requesterId: "clash", + }), + ); + return { harness, requester: "clash" }; + }, + "lease-ended": async () => { + const harness = await createHarness(); + await harness.engine.request(request, { ...holder, leaseId: "myid" }); + await harness.engine.release("myid", "explicit"); + await settledOrPending( + harness.engine.request(request, { + leaseId: "myid", + ownerId: "again", + requesterId: "again", + }), + ); + return { harness, requester: "again" }; + }, + "boot-timeout": async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("makeReady", 1, new Error("boot failed")); + const harness = await createHarness({ driver }); + await settledOrPending( + harness.engine.request(request, { ownerId: "slow", requesterId: "slow" }), + ); + return { harness, requestId: await admittedId(harness, "slow"), requester: "slow" }; + }, + killed: async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + const queued = harness.engine.request(request, { ownerId: "next", requesterId: "next" }); + void queued.catch(() => undefined); + await flush(); + await harness.engine.nuke(false); + return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; + }, + updating: async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + void harness.engine + .request(request, { ownerId: "next", requesterId: "next" }) + .catch(() => undefined); + await flush(); + await harness.engine.leasing.holdForUpdate(); + return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; + }, + cancelled: async () => { + const harness = await createHarness(); + await harness.engine.request(request, holder); + void harness.engine + .request(request, { ownerId: "next", requesterId: "next" }) + .catch(() => undefined); + await flush(); + await harness.engine.cancelPending("next"); + return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; + }, + "daemon-restarted": async () => { + const before = await createHarness(); + await before.engine.request(request, holder); + void before.engine + .request(request, { ownerId: "agent", requesterId: "agent" }) + .catch(() => undefined); + await expect.poll(() => admittedId(before, "agent")).toEqual(expect.stringMatching(/^req_/)); + const requestId = await admittedId(before, "agent"); + const harness = await createHarness({ state: before.state }); + await harness.engine.convergeRunningCapacity(); + return { harness, requestId, requester: "agent" }; + }, + }; + + const workerReasons = (Object.keys(cases) as Reason[]).filter( + (reason) => cases[reason] !== "gateway", + ); + + it.each(workerReasons)( + "includes the request id and the requester when it rejects with %s", + async (reason) => { + const run = cases[reason]; + if (run === "gateway") throw new Error(`${reason} is the gateway's`); + + const { harness, requestId, requester } = await run(); + + const rejections = harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload as Rejection) + .filter((payload) => payload.reason === reason); + expect(rejections).toHaveLength(1); + expect(rejections[0]).toMatchObject({ + requestId: requestId ?? expect.stringMatching(/^req_/), + requester, + }); + }, + ); +}); + +describe("createLeasing queue timeout and failure storage", () => { + const android = { + physical: false, + model: "Pixel 9", + osVersion: "35", + platform: "android", + } as const; + + it("emits lease.rejected for a timed-out waiter under the wait queue's module, then lets the request behind it through", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const droid = new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }); + const harness = await createHarness({ driver: ios, drivers: [ios, droid] }); + await harness.engine.request(request, { ownerId: "holder", requesterId: "holder" }); + const blocking = harness.engine.request(request, { + ownerId: "head", + requesterId: "head", + timeoutMs: 10, + }); + const behind = harness.engine.request(android, { ownerId: "behind", requesterId: "behind" }); + await flush(); + // why: The android request could be served now, but it waits behind the iOS one at the head. + expect(await settledOrPending(behind)).toBe("still pending"); + + harness.clock.advance(10); + + expect(await settledOrPending(blocking)).toBeInstanceOf(QueueTimeoutError); + expect(await settledOrPending(behind)).toMatchObject({ lease: { requesterId: "behind" } }); + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => ({ + module: event.module, + reason: (event.payload as { reason: string }).reason, + })), + ).toEqual([{ module: "wait-queue", reason: "timeout" }]); + }); + + it("stores a failed request as INTERNAL with the error's own message when no classifier was given", async () => { + const harness = await createHarness(); + + await expect( + harness.engine.request(android, { ownerId: "agent", requesterId: "agent" }), + ).rejects.toThrow("No driver registered for platform: android"); + + expect(harness.registry.leaseRequests()).toMatchObject([ + { + failure: { code: "INTERNAL", message: "No driver registered for platform: android" }, + state: "failed", + }, + ]); + }); +}); diff --git a/src/leasing/create-leasing-restart-and-classes.test.ts b/src/leasing/create-leasing-restart-and-classes.test.ts new file mode 100644 index 000000000..530558dcd --- /dev/null +++ b/src/leasing/create-leasing-restart-and-classes.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from "vitest"; +import { UnknownLeaseError } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, seedReady, flush } from "./test-create-leasing.js"; + +describe("createLeasing restart recovery of stored lease requests", () => { + async function restartWithAgentQueued(idempotencyKey?: string) { + const before = await createHarness(); + await before.engine.request(request, { ownerId: "holder", requesterId: "holder" }); + void before.engine + .request(request, { + ownerId: "agent", + requesterId: "agent", + ...(idempotencyKey === undefined ? {} : { idempotencyKey }), + }) + .catch(() => undefined); + await expect + .poll(() => before.registry.leaseRequests().find((record) => record.requesterId === "agent")) + .toMatchObject({ state: "open" }); + + const after = await createHarness({ state: before.state }); + await after.engine.convergeRunningCapacity(); + return after; + } + + it.each([{ idempotencyKey: "key-1" as string | undefined }, { idempotencyKey: undefined }])( + "settles a request still open at a restart as failed and says the daemon restarted, whether it carried an idempotency key or not ($idempotencyKey)", + async ({ idempotencyKey }) => { + const after = await restartWithAgentQueued(idempotencyKey); + + expect( + after.registry.leaseRequests().find((record) => record.requesterId === "agent"), + ).toMatchObject({ + failure: { code: "INTERNAL", message: expect.stringContaining("daemon restarted") }, + state: "failed", + }); + expect( + after.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload), + ).toEqual([ + { + reason: "daemon-restarted", + requestId: expect.stringMatching(/^req_/), + requester: "agent", + requestSpec: request, + }, + ]); + if (idempotencyKey !== undefined) { + await expect( + after.engine.request(request, { + idempotencyKey, + ownerId: "agent", + requesterId: "agent", + }), + ).rejects.toMatchObject({ message: expect.stringContaining("daemon restarted") }); + } + }, + ); +}); + +describe("createLeasing a grant and its request's result", () => { + const asker = { idempotencyKey: "key-1", ownerId: "agent", requesterId: "agent" }; + + it("answers a repeat under the same key with the lease after the daemon stopped between the grant and the request book's settle write", async () => { + const before = await createHarness(); + await seedReady(before); + vi.spyOn(before.registry, "settleLeaseRequest").mockRejectedValue(new Error("disk gone")); + const grant = await before.engine.request(request, asker); + await flush(); + + const after = await createHarness({ driver: before.driver, state: before.state }); + await after.engine.convergeRunningCapacity(); + const repeat = await after.engine.request(request, asker); + + expect(repeat).toEqual(grant); + expect(repeat.lease.id).toBe(grant.lease.id); + expect(after.registry.leaseRequests()).toMatchObject([{ state: "granted" }]); + expect(after.registry.snapshot.leases).toHaveLength(1); + }); + + it("answers a repeat under the same key in the same process with the lease", async () => { + const harness = await createHarness(); + await seedReady(harness); + const grant = await harness.engine.request(request, asker); + await flush(); + + const repeat = await harness.engine.request(request, asker); + expect(repeat).toEqual(grant); + expect(repeat.lease.id).toBe(grant.lease.id); + expect(harness.registry.leaseRequests()).toMatchObject([{ grant, state: "granted" }]); + }); + + it("answers a repeat of a granted request whose lease was released since with the lease as recorded and the device as it is now, and a renew of that lease with UnknownLeaseError", async () => { + const harness = await createHarness(); + await seedReady(harness); + const grant = await harness.engine.request(request, asker); + await flush(); + await harness.engine.release(grant.lease.id, "explicit"); + await harness.engine.settle(); + + const repeat = await harness.engine.request(request, asker); + + expect(repeat.lease.id).toBe(grant.lease.id); + expect(grant.device.state).toBe("leased"); + expect(repeat).toEqual({ ...grant, device: harness.registry.snapshot.devices[0] }); + expect(repeat.device.state).toBe("deleted"); + await expect(harness.engine.renew(grant.lease.id)).rejects.toBeInstanceOf(UnknownLeaseError); + }); +}); + +describe("createLeasing class requests", () => { + it("creates the first model on the class's preference list the catalog lists, for a request naming a class", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + knownModels: ["iPhone 15", "iPhone 16"], + modelClasses: { "iPhone 15": "phone", "iPhone 16": "phone" }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 16", "iPhone 15"] } }, + }); + + const granted = await harness.engine.request( + { class: "phone", platform: "ios" }, + { ownerId: "agent", requesterId: "agent" }, + ); + + expect(granted.device.spec.model).toBe("iPhone 16"); + }); + + it("creates the first listed model that pairs with the requested OS version, not one that pairs with another", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["18.4", "26.5"], + clock: new FakeClock(1_000), + knownModels: ["iPhone 16", "iPhone 17"], + modelClasses: { "iPhone 16": "phone", "iPhone 17": "phone" }, + modelRuntimes: { "iPhone 16": ["18.4", "26.5"], "iPhone 17": ["26.5"] }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 16"] } }, + }); + + const granted = await harness.engine.request( + { class: "phone", osVersion: "18.4", platform: "ios" }, + { ownerId: "agent", requesterId: "agent" }, + ); + + expect(granted.device.spec).toMatchObject({ model: "iPhone 16", osVersion: "18.4" }); + }); + + it("refuses a class request naming an OS that is not installed as RUNTIME_MISSING without calling the installer", async () => { + const asked: unknown[] = []; + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + knownModels: ["iPhone 17"], + modelClasses: { "iPhone 17": "phone" }, + platform: "ios", + }); + const harness = await createHarness({ + components: { + claimProvision: () => () => undefined, + install: async (call) => { + asked.push(call); + return { outcome: "installed", version: "18.4" }; + }, + }, + driver, + modelPreferences: { ios: { phone: ["iPhone 17"] } }, + }); + + await expect( + harness.engine.request( + { class: "phone", osVersion: "18.4", platform: "ios" }, + { allowDownload: true, ownerId: "agent", requesterId: "agent" }, + ), + ).rejects.toMatchObject({ downloadable: false, name: "RuntimeMissingError" }); + expect(asked).toEqual([]); + }); +}); diff --git a/src/leasing/create-leasing-startup-reclaim.test.ts b/src/leasing/create-leasing-startup-reclaim.test.ts new file mode 100644 index 000000000..6bb364bac --- /dev/null +++ b/src/leasing/create-leasing-startup-reclaim.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it } from "vitest"; +import { type DriverDevice } from "../core/index.js"; +import { testStateOf } from "../core/testing.js"; +import { + FakeDriver, + loadTestRegistry, + openTestState, + testComponentWiring, +} from "../core/testing.js"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, FakeSystemStats } from "../ports/index.js"; +import { createTestEngine } from "./testing.js"; +import { gibibyte, request, config, createHarness, flush } from "./test-create-leasing.js"; + +// why: #43: startup convergence used to await each orphaned held lease's device reclaim +// inline (an erase measured ~34s for one simulator), so N orphaned leases cost N +// serial erases before any other request could be served. These cover the shape +// that replaced it: the lease is released registry-only on the convergence path, +// and its reclaim proceeds in the background. +import { NoCapacityError } from "./index.js"; + +describe("createLeasing startup reclaim backgrounding (#43)", () => { + it("converges without waiting for an in-flight destroy, and a fresh request is served immediately after", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + // why: Matches the issue's measured single-simulator-erase cost -- the old inline + // await would have forced convergence to sit through this. + latencyMs: { destroy: 34_000 }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 2, + }, + }); + // why: under ADR 0004 convergence itself releases nothing -- there is no orphan sweep left -- + // so the in-flight destroy it must not block on is produced the way one actually arises: + // a release commits the registry half and hands the 34s destroy off to the background. + const released = await harness.engine.request(request, { + ownerId: "previous-holder", + requesterId: "previous-holder", + }); + await harness.engine.release(released.lease.id, "explicit"); + + const convergeStartedAt = harness.clock.now(); + await harness.engine.convergeRunningCapacity(); + // why: the clock never had to move for convergence to finish: it did not sit through + // the 34s destroy above. + expect(harness.clock.now()).toBe(convergeStartedAt); + expect( + harness.registry.snapshot.devices.find((device) => device.id === released.device.id)?.state, + ).toBe("shutdown"); + expect(harness.registry.snapshot.leases).toEqual([]); + + // why: a fresh request is servable right away -- capacity allows a second device, and the + // one still mid-destroy never blocks it. Still 0ms elapsed. + const granted = await harness.engine.request(request, { + requesterId: "new-agent", + ownerId: "new-agent", + }); + expect(granted.device.id).not.toBe(released.device.id); + expect(harness.clock.now()).toBe(convergeStartedAt); + }); + + it("never grants a device whose background destroy is still in flight", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { destroy: 34_000 }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 1, + }, + }); + const released = await harness.engine.request(request, { + ownerId: "previous-holder", + requesterId: "previous-holder", + }); + await harness.engine.release(released.lease.id, "explicit"); + + await harness.engine.convergeRunningCapacity(); + expect(harness.registry.snapshot.devices).toMatchObject([{ state: "shutdown" }]); + + // why: a device that ended a lease is never granted again, even to reboot it. With maxDevices:1 + // there is also no room to provision a second device, so the request is refused rather than + // handed the one still mid-destroy. + await expect( + harness.engine.request(request, { + noWait: true, + ownerId: "new-agent", + requesterId: "new-agent", + }), + ).rejects.toBeInstanceOf(NoCapacityError); + + clock.advance(34_000); + await flush(); + expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); + + // why: once the destroy finishes the slot is free again, and a new device is provisioned for it. + const granted = await harness.engine.request(request, { + ownerId: "new-agent", + requesterId: "new-agent", + }); + expect(granted.device.id).not.toBe(released.device.id); + }); + + it("recovers a background destroy interrupted by a daemon crash on the next start", async () => { + const state = await openTestState(); + let nextId = 1; + const idGenerator = { generate: () => `${nextId++}` }; + const systemStats = () => + new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }); + // why: the physical device survives a daemon restart even though the daemon's + // in-memory state does not -- represented here by one FakeDriver instance + // shared across both simulated processes below. + // why: the first destroy never returns, as one in a crashed daemon never does. + class CrashingDestroyDriver extends FakeDriver { + #destroys = 0; + override async destroy(device: DriverDevice): Promise { + this.#destroys += 1; + if (this.#destroys === 1) return new Promise(() => {}); + return super.destroy(device); + } + } + const driver = new CrashingDestroyDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + + const clock1 = new FakeClock(1_000); + const bus1 = new EventBus(clock1); + const registry1 = await loadTestRegistry({ + clock: clock1, + eventBus: bus1, + idGenerator, + state, + }); + const engine1 = createTestEngine({ + ...testComponentWiring({ + clock: clock1, + drivers: [driver], + eventBus: bus1, + registry: registry1, + }), + clock: clock1, + config: config(), + drivers: [driver], + eventBus: bus1, + idGenerator, + registry: registry1, + state: testStateOf(registry1), + systemStats: systemStats(), + }); + const granted = await engine1.request(request, { + ownerId: "previous-holder", + requesterId: "previous-holder", + }); + // why: the holder released, so the destroy behind it is running in the background... + await engine1.release(granted.lease.id, "explicit"); + + // why: ...convergence leaves it alone (it is claimed by this process)... + await engine1.convergeRunningCapacity(); + expect(registry1.snapshot.devices).toMatchObject([ + { id: granted.device.id, state: "shutdown" }, + ]); + // why: ...and the process is abandoned before the driver's 34s destroy resolves -- + // exactly like a crashed daemon. Nothing here ever settles that promise. + + // why: next start: a fresh process reads the same persisted registry (state.db + // survives the crash) and reconnects to the same physical driver, but its + // in-memory claim tracking starts empty -- there is no live claim for this + // device, because the process that held it is gone. + const clock2 = new FakeClock(clock1.now()); + const bus2 = new EventBus(clock2); + const registry2 = await loadTestRegistry({ + clock: clock2, + eventBus: bus2, + idGenerator, + state, + }); + const engine2 = createTestEngine({ + ...testComponentWiring({ + clock: clock2, + drivers: [driver], + eventBus: bus2, + registry: registry2, + }), + clock: clock2, + config: config(), + drivers: [driver], + eventBus: bus2, + idGenerator, + registry: registry2, + state: testStateOf(registry2), + systemStats: systemStats(), + }); + + await engine2.convergeRunningCapacity(); + await flush(); + + // why: recovered by destroying the spent device again, never by reclaiming it. + expect(registry2.snapshot.devices).toMatchObject([{ id: granted.device.id, state: "deleted" }]); + expect(driver.calls.filter((call) => call.operation === "reclaim")).toHaveLength(0); + expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); + }); + + it("leaves no armed timer behind for a lease it is disposed with", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 900_000 } }); + await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + expect(harness.clock.pendingTimerCount).toBeGreaterThan(0); + + harness.engine.dispose(); + + // why: A real `setTimeout` holds the event loop open, so a timer surviving disposal is a + // daemon that has decided to stop and cannot: `daemon stop` used to hang for the whole + // of an outstanding lease's TTL. Under ADR 0004 that is every lease, since a stop + // releases none of them. + expect(harness.clock.pendingTimerCount).toBe(0); + // why: The lease itself is untouched -- disposal cancels a timer, it does not expire a + // lease, which is what lets a lease survive the restart with its deadline intact. + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); +}); diff --git a/src/leasing/create-leasing-waiting-reasons.test.ts b/src/leasing/create-leasing-waiting-reasons.test.ts new file mode 100644 index 000000000..c962ca83f --- /dev/null +++ b/src/leasing/create-leasing-waiting-reasons.test.ts @@ -0,0 +1,294 @@ +import { describe, expect, it, vi } from "vitest"; +import { DriverCrashError } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, seedReady, flush } from "./test-create-leasing.js"; +import { DaemonUpdatingError, ProvisionFailedError, RequestCancelledError } from "./index.js"; + +describe("createLeasing: a waiting request says why it is not placed (#636)", () => { + const runningLimit = { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 1 }, + maxRunning: 2, + }; + const asking = (requesterId: string, more: Record = {}) => ({ + ownerId: requesterId, + requesterId, + ...more, + }); + const listing = (harness: Awaited>) => + harness.engine.waitingRequests().map((entry) => ({ + reason: (entry as { reason?: string }).reason, + requesterId: entry.requesterId, + stage: entry.stage, + })); + + it("lists a queued request at the iOS running limit as platform-running-limit and one behind it as behind-earlier-request, and stores no reason", async () => { + const harness = await createHarness({ limits: runningLimit }); + await harness.engine.request(request, asking("holder")); + void harness.engine.request(request, asking("head")); + void harness.engine.request(request, asking("behind")); + await flush(); + + expect(listing(harness)).toEqual([ + { reason: "platform-running-limit", requesterId: "head", stage: "queued" }, + { reason: "behind-earlier-request", requesterId: "behind", stage: "queued" }, + ]); + const stored = harness.registry.leaseRequests().find((record) => record.requesterId === "head"); + expect(stored).toBeDefined(); + expect(stored).not.toHaveProperty("reason"); + }); + + it("follows the latest attempt: when the head is granted, the request behind it lists its own new reason", async () => { + const harness = await createHarness({ limits: runningLimit }); + const holder = await harness.engine.request(request, asking("holder")); + const head = harness.engine.request(request, asking("head")); + void harness.engine.request(request, asking("behind")); + await flush(); + + await harness.engine.release(holder.lease.id, "explicit"); + await head; + await harness.engine.settle(); + + expect(listing(harness)).toEqual([ + { reason: "platform-running-limit", requesterId: "behind", stage: "queued" }, + ]); + }); + + it("cancelling the queued head makes the request behind it plan again and list its own reason", async () => { + const harness = await createHarness({ limits: runningLimit }); + await harness.engine.request(request, asking("holder")); + const head = harness.engine.request(request, asking("head")); + void harness.engine.request(request, asking("behind")); + await flush(); + + await expect(harness.engine.cancelPending("head")).resolves.toBe("cancelled"); + await expect(head).rejects.toBeInstanceOf(RequestCancelledError); + await flush(); + + expect(listing(harness)).toEqual([ + { reason: "platform-running-limit", requesterId: "behind", stage: "queued" }, + ]); + }); + + it("plans again at once after a first provisioning failure, and a success then grants", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("provision", 1, new DriverCrashError("simulator exited")); + const harness = await createHarness({ driver }); + + const granted = await harness.engine.request(request, asking("retried")); + + expect(granted.lease.id).toBeDefined(); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + expect(listing(harness)).toEqual([]); + }); + + it("rejects a request whose provision failed twice with PROVISION_FAILED, lists nothing for it, and the request behind it plans again", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { provision: 50 }, + platform: "ios", + }); + driver.failOn("provision", 1, new DriverCrashError("simulator exited")); + driver.failOn("provision", 2, new DriverCrashError("simulator exited again")); + const harness = await createHarness({ driver }); + const rejected = harness.engine + .request(request, asking("failing")) + .catch((error: unknown) => error); + await flush(); + clock.advance(50); + await flush(); + void harness.engine.request(request, asking("meanwhile")); + await flush(); + expect(listing(harness)).toEqual([ + { reason: undefined, requesterId: "failing", stage: "starting" }, + { reason: "device-limit", requesterId: "meanwhile", stage: "queued" }, + ]); + expect(harness.engine.waitingRequests()[0]).not.toHaveProperty("reason"); + + clock.advance(50); + await flush(); + + expect(await rejected).toBeInstanceOf(ProvisionFailedError); + expect(await rejected).toMatchObject({ message: "simulator exited again" }); + expect(listing(harness)).toEqual([ + { reason: undefined, requesterId: "meanwhile", stage: "starting" }, + ]); + }); + + it("refuses a queued request when the update hold begins and still grants the one that is provisioning", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { provision: 50 }, + platform: "ios", + }); + const harness = await createHarness({ driver, limits: runningLimit }); + const provisioning = harness.engine + .request(request, asking("provisioning")) + .catch((error: unknown) => error); + await flush(); + const queued = harness.engine + .request(request, asking("queued")) + .catch((error: unknown) => error); + await flush(); + expect(listing(harness)).toEqual([ + { reason: undefined, requesterId: "provisioning", stage: "starting" }, + { reason: "platform-running-limit", requesterId: "queued", stage: "queued" }, + ]); + + await harness.engine.leasing.holdForUpdate(); + clock.advance(50); + await flush(); + + expect(await queued).toBeInstanceOf(DaemonUpdatingError); + expect(await provisioning, "the request that was provisioning").toMatchObject({ + lease: { id: expect.any(String) }, + }); + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload as { requester: string; reason: string }) + .map(({ requester, reason }) => [requester, reason]), + ).toEqual([["queued", "updating"]]); + }); + + it("refuses the retry of a provision that fails while the update hold is on, with DAEMON_UPDATING and no second provision", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { provision: 50 }, + platform: "ios", + }); + driver.failOn("provision", 1, new DriverCrashError("simulator exited")); + const harness = await createHarness({ driver, limits: runningLimit }); + const failing = harness.engine + .request(request, asking("failing")) + .catch((error: unknown) => error); + await flush(); + + await harness.engine.leasing.holdForUpdate(); + clock.advance(50); + await flush(); + clock.advance(50); + await flush(); + + expect(await failing).toBeInstanceOf(DaemonUpdatingError); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload as { requester: string; reason: string }) + .map(({ requester, reason }) => [requester, reason]), + ).toEqual([["failing", "updating"]]); + }); + + it("refuses a request that is still resolving its spec when the update hold begins", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { resolveSpec: 50 }, + platform: "ios", + }); + const harness = await createHarness({ driver, limits: runningLimit }); + const resolving = harness.engine + .request(request, asking("resolving")) + .catch((error: unknown) => error); + await flush(); + + await harness.engine.leasing.holdForUpdate(); + clock.advance(50); + await flush(); + + expect(await resolving, "the request that was resolving").toBeInstanceOf(DaemonUpdatingError); + expect(driver.calls.map((call) => call.operation)).not.toContain("provision"); + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.rejected") + .map((event) => event.payload as { requester: string; reason: string }) + .map(({ requester, reason }) => [requester, reason]), + ).toEqual([["resolving", "updating"]]); + }); + + it("lists a request that waits for a device the warm pool is booting as waiting-for-device", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + knownModels: ["iPhone 16"], + availableOsVersions: ["26.5"], + clock, + latencyMs: { makeReady: 50 }, + platform: "ios", + }); + const harness = await createHarness({ + startPool: true, + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 2, + }, + }); + const granting = harness.engine.request(request, asking("a")); + await flush(); + clock.advance(50); + const first = await granting; + await harness.engine.release(first.lease.id, "explicit"); + await vi.waitFor(() => + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), + ); + + void harness.engine.request(request, asking("b")); + await flush(); + + expect(listing(harness)).toEqual([ + { reason: "waiting-for-device", requesterId: "b", stage: "queued" }, + ]); + }); + + it("lists a request whose eviction failed as the capacity reason that asked for it", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("shutdown", 1, new DriverCrashError("cannot stop victim")); + const harness = await createHarness({ driver, limits: runningLimit }); + await seedReady(harness); + + void harness.engine.request({ ...request, model: "iPhone SE" }, asking("new-spec")); + await flush(); + + expect(listing(harness)).toEqual([ + { reason: "platform-running-limit", requesterId: "new-spec", stage: "queued" }, + ]); + }); + + it("lists a request whose only match is being destroyed as the capacity reason its plan gave", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { destroy: 20 }, + platform: "ios", + }); + const harness = await createHarness({ driver }); + const first = await harness.engine.request(request, asking("first")); + await harness.engine.release(first.lease.id, "explicit"); + + void harness.engine.request(request, asking("second")); + await flush(); + + expect(listing(harness)).toEqual([ + { reason: "device-limit", requesterId: "second", stage: "queued" }, + ]); + }); +}); diff --git a/src/leasing/create-leasing-warm-pool.test.ts b/src/leasing/create-leasing-warm-pool.test.ts new file mode 100644 index 000000000..7ab94e97b --- /dev/null +++ b/src/leasing/create-leasing-warm-pool.test.ts @@ -0,0 +1,562 @@ +import { describe, expect, it, vi } from "vitest"; +import { CleanupReaper, DriverCrashError } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock, JsonLinesLogger, MemoryLogSink, MemoryFilesystem } from "../ports/index.js"; +import { + gibibyte, + request, + config, + createHarness, + seedReady, + flush, + settledOrPending, +} from "./test-create-leasing.js"; + +describe("createLeasing warm pool", () => { + const androidSpec = { + physical: false, + model: "Pixel 9", + osVersion: "36", + platform: "android", + } as const; + + it("shuts down one of three ready devices with initiator warm-pool after daemon.started when maxRunning is 2", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 2 }, + maxRunning: 2, + }, + }); + await seedReady(harness); + await seedReady(harness); + await seedReady(harness); + + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await harness.engine.settle(); + + const states = harness.registry.snapshot.devices.map((item) => item.state); + expect(states.filter((state) => state === "ready")).toHaveLength(2); + expect(states.filter((state) => state === "shutdown")).toHaveLength(1); + const shutdowns = harness.bus.replay().filter((event) => event.event === "device.shutdown"); + expect(shutdowns).toHaveLength(1); + expect(shutdowns[0]).toMatchObject({ payload: { initiator: "warm-pool" } }); + }); + + it("logs a warm pool shutdown that fails under the warm pool's own module and leaves the device ready", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + driver.failOn("shutdown", 1, new DriverCrashError("cannot stop it")); + const sink = new MemoryLogSink(); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 1 }, + maxRunning: 1, + }, + logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), + }); + await seedReady(harness); + await seedReady(harness); + + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await harness.engine.settle(); + + expect(sink.records.filter((record) => record.level === "warn")).toMatchObject([ + { fields: { step: "shutdown" }, module: expect.stringContaining("warm-pool") }, + ]); + expect(harness.registry.snapshot.devices.map((item) => item.state)).toEqual(["ready", "ready"]); + }); + + it("a nuke that deletes devices is not undone by the warm pool booting a released device back", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const harness = await createHarness({ driver }); + await harness.engine.request(request, { ownerId: "held", requesterId: "held" }); + + await harness.engine.nuke(true); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); + }); + + it("triggers no pool pass from a fact before daemon.started, and ends with one of two ready devices shut down at one running slot", async () => { + const harness = await createHarness({ + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 1 }, + maxRunning: 1, + }, + }); + await seedReady(harness); + await seedReady(harness); + + harness.bus.emit( + "device.deleted", + { deviceId: "x", initiator: "test", physical: false }, + "test", + ); + await harness.engine.settle(); + expect(harness.registry.snapshot.devices.map((item) => item.state)).toEqual(["ready", "ready"]); + + await harness.engine.convergeRunningCapacity(); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices.map((item) => item.state).sort()).toEqual([ + "ready", + "shutdown", + ]); + }); + + it("cancels the pool's tick when the engine is disposed", async () => { + const harness = await createHarness(); + await harness.engine.convergeRunningCapacity(); + expect(harness.clock.pendingTimerCount).toBe(1); + + harness.engine.dispose(); + + expect(harness.clock.pendingTimerCount).toBe(0); + }); + + it("a nuke without --delete-devices leaves the released devices destroyed, not booted back by the pool", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const harness = await createHarness({ driver }); + await harness.engine.convergeRunningCapacity(); + const held = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + + await harness.engine.nuke(false); + await harness.engine.settle(); + // why: Still inside idle.shutdownAfterMs of the forced release, and a fact that triggers a pass. + harness.bus.emit( + "cleanup.executed", + { action: "shutdown", reason: "test", ruleName: "test", target: held.device.id }, + "test", + ); + await harness.engine.settle(); + + expect( + harness.registry.snapshot.devices.find((item) => item.id === held.device.id)?.state, + ).toBe("deleted"); + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); + }); + + it("accepts a new request again once a nuke has finished", async () => { + const harness = await createHarness(); + await harness.engine.request(request, { ownerId: "held", requesterId: "held" }); + + await harness.engine.nuke(false); + + await expect( + harness.engine.request(request, { ownerId: "after", requesterId: "after" }), + ).resolves.toMatchObject({ lease: { requesterId: "after" } }); + }); + + /** + * why: a released virtual device is destroyed, never booted back, so the device the warm pool + * boots is a shut-down one that never ended a lease, under a warm target of one. + * Its boot is held on the clock, with room for a second device. + */ + async function shutdownDeviceBooting(failBoot = false) { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + knownModels: ["iPhone 16"], + availableOsVersions: ["26.5"], + clock, + latencyMs: { makeReady: 50 }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 2, + }, + warmPool: { + targets: [{ count: 1, model: "iPhone 16", osVersion: "26.5", platform: "ios" }], + }, + }); + const seeding = seedReady(harness); + await flush(); + clock.advance(50); + const seeded = await seeding; + await harness.registry.transitionDevice(seeded.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: seeded.id, initiator: "test" }, + }); + if (failBoot) driver.failOn("makeReady", 2, new Error("boom")); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await vi.waitFor(() => + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), + ); + return { clock, driver, seeded, harness }; + } + + it("a nuke waits for a warm pool boot already in flight, then deletes the device", async () => { + const { clock, driver, harness } = await shutdownDeviceBooting(); + + const nuking = harness.engine.nuke(true); + await flush(); + clock.advance(50); + await nuking; + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); + expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); + }); + + describe("a request for a device on its way", () => { + it("a request arriving during a warm-pool boot that serves it is granted that device when the boot settles, without a device provisioned for it", async () => { + const { clock, driver, seeded, harness } = await shutdownDeviceBooting(); + + const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); + await flush(); + expect(await settledOrPending(second)).toBe("still pending"); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + + clock.advance(50); + await flush(); + // why: The pool's kick after its boot woke the request: it is granted without another clock tick. + expect(await settledOrPending(second)).not.toBe("still pending"); + const granted = await second; + + // why: the granted device leaves the warm target short, so the pool provisions a + // replacement of its own; none was provisioned for the request, which got the booting one. + expect(granted.device.id).toBe(seeded.id); + expect(harness.bus.replay().filter((event) => event.event === "device.ready")).toHaveLength( + 2, + ); + }); + + async function twoRoomsHarness(warmPoolEnabled: boolean) { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { makeReady: 50 }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 3, + }, + warmPoolEnabled, + }); + return { clock, driver, harness }; + } + + it("a request arriving while another request is creating its own device provisions its own device rather than waiting for that one", async () => { + const { clock, driver, harness } = await twoRoomsHarness(true); + + const first = harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + await vi.waitFor(() => + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1), + ); + const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); + await flush(); + // why: Before either boot ends: the second request did not wait for the first one's device. + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + clock.advance(50); + await flush(); + clock.advance(50); + const granted = await Promise.all([first, second]); + + expect(granted[0].device.id).not.toBe(granted[1].device.id); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("a request arriving while another request is booting a shut-down device provisions its own device rather than waiting for that one", async () => { + const { clock, driver, harness } = await twoRoomsHarness(false); + // why: a released virtual device is destroyed, so the shut-down device is seeded. + const seeding = seedReady(harness); + await flush(); + clock.advance(50); + const held = await seeding; + await harness.registry.transitionDevice(held.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: held.id, initiator: "test" }, + }); + + const booting = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); + await vi.waitFor(() => + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), + ); + const other = harness.engine.request(request, { ownerId: "c", requesterId: "c" }); + await flush(); + // why: Before either boot ends: the other request did not wait for the one booting. + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + clock.advance(50); + await flush(); + clock.advance(50); + const granted = await Promise.all([booting, other]); + + expect(granted[0].device.id).toBe(held.id); + expect(granted[1].device.id).not.toBe(held.id); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("under noWait a request arriving during a warm-pool boot provisions its own device", async () => { + const { clock, driver, seeded, harness } = await shutdownDeviceBooting(); + + const second = harness.engine.request(request, { + noWait: true, + ownerId: "b", + requesterId: "b", + }); + await flush(); + clock.advance(50); + const granted = await second; + + expect(granted.device.id).not.toBe(seeded.id); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("a request waiting on a warm-pool boot that fails re-plans on the kick and boots the device the failed boot left", async () => { + const { clock, driver, seeded, harness } = await shutdownDeviceBooting(true); + + const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); + await flush(); + clock.advance(50); + await flush(); + // why: The kick from the failed boot woke the request: it has started booting the device itself. + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(3); + clock.advance(50); + const granted = await second; + await harness.engine.settle(); + + // why: The failed boot left the device `shutdown` and unclaimed, so the request boots it itself. + expect(granted.device.id).toBe(seeded.id); + expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(3); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); + }); + }); + + it("a release at the running cap with a class request waiting provisions a new device for it, the released one being destroyed", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.0"], + clock, + knownModels: ["iPhone 15", "iPhone 17"], + modelClasses: { "iPhone 15": "phone", "iPhone 17": "phone" }, + platform: "ios", + }); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 1 }, + maxRunning: 1, + }, + modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 15"] } }, + }); + const iphone15 = await seedReady(harness, { + physical: false, + model: "iPhone 15", + osVersion: "26.0", + platform: "ios", + }); + const holder = await harness.engine.request( + { model: "iPhone 15", osVersion: "26.0", platform: "ios" }, + { ownerId: "holder", requesterId: "holder" }, + ); + expect(holder.device.id).toBe(iphone15.id); + const waiter = harness.engine.request( + { class: "phone", platform: "ios" }, + { ownerId: "waiter", requesterId: "waiter" }, + ); + await flush(); + expect(harness.engine.queueDepth).toBe(1); + + await harness.engine.release(holder.lease.id, "explicit"); + const granted = await waiter; + await harness.engine.settle(); + + expect(granted.device.id).not.toBe(iphone15.id); + expect(granted.device.spec.model).toBe("iPhone 17"); + expect(harness.registry.snapshot.devices.find((item) => item.id === iphone15.id)?.state).toBe( + "deleted", + ); + expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); + }); + + it("destroys a released iOS device instead of booting it back when the running limit has room", async () => { + const driver = new FakeDriver({ + knownModels: ["iPhone 16"], + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const harness = await createHarness({ driver, startPool: true }); + const first = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); + + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + + expect( + harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, + ).toBe("deleted"); + const releasedDriverId = harness.registry.snapshot.devices.find( + (item) => item.id === first.device.id, + )?.driverDeviceId; + expect( + driver.calls.filter( + (call) => + call.operation === "makeReady" && + (call.arguments[0] as { readonly deviceId: string }).deviceId === releasedDriverId, + ), + ).toHaveLength(1); + }); + + it("with warmPool disabled a released fake iOS device and a released fake Android device are both deleted, with no reclaim", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const harness = await createHarness({ + driver: ios, + drivers: [ios, android], + limits: { + android: { maxDevices: 2, maxRunning: 2 }, + ios: { maxDevices: 2, maxRunning: 2 }, + maxRunning: 4, + }, + warmPoolEnabled: false, + }); + const iosLease = await harness.engine.request(request, { ownerId: "i", requesterId: "i" }); + const androidLease = await harness.engine.request(androidSpec, { + ownerId: "a", + requesterId: "a", + }); + + await harness.engine.release(iosLease.lease.id, "explicit"); + await harness.engine.release(androidLease.lease.id, "explicit"); + await harness.engine.settle(); + + const state = (id: string) => + harness.registry.snapshot.devices.find((item) => item.id === id)?.state; + expect(state(iosLease.device.id)).toBe("deleted"); + expect(state(androidLease.device.id)).toBe("deleted"); + const events = harness.bus.replay(); + expect(events.filter((event) => event.event === "device.reclaimed")).toEqual([]); + expect( + events.filter((event) => event.event === "device.deleted").map((event) => event.payload), + ).toMatchObject([ + { deviceId: iosLease.device.id, initiator: "lease-end" }, + { deviceId: androidLease.device.id, initiator: "lease-end" }, + ]); + }); +}); + +describe("createLeasing queued device cleanup", () => { + it("keeps an old shut-down device ready across reaper passes while its requester waits behind the queue head", async () => { + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + knownModels: ["iPhone 16"], + platform: "ios", + }); + const android = new FakeDriver({ + availableOsVersions: ["35"], + clock: new FakeClock(1_000), + platform: "android", + }); + const harness = await createHarness({ + driver, + drivers: [driver, android], + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 3, + }, + }); + const other = { model: "Pixel 9", osVersion: "35", platform: "android" } as const; + await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + ttlMs: 3_600_000, + }); + // why: a released virtual device is destroyed, so the old shut-down device is seeded. + const previous = await seedReady(harness, { ...other, physical: false }, "full", android); + await harness.registry.transitionDevice(previous.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: previous.id, initiator: "test" }, + }); + harness.clock.advance(10_001); + await harness.engine.convergeRunningCapacity(); + const pending = ["head", "behind"].map((requesterId, index) => + harness.engine + .request(index === 0 ? request : other, { + ownerId: requesterId, + requesterId, + }) + .catch((error: unknown) => error), + ); + await flush(); + await harness.engine.core.passWarmPool(); + await harness.engine.settle(); + const state = () => + harness.registry.snapshot.devices.find((device) => device.id === previous.id)?.state; + expect(harness.engine.waitingRequests().map((entry) => entry.requesterId)).toEqual([ + "head", + "behind", + ]); + expect(state()).toBe("ready"); + const reaper = new CleanupReaper({ + clock: harness.clock, + config: { ...config(), diskPressure: { freeBytesThreshold: 10 * gibibyte } }, + eventBus: harness.bus, + executor: harness.engine.cleanup, + filesystem: new MemoryFilesystem(), + registry: harness.registry, + targetedDevices: () => harness.engine.core.targetedDevices(), + }); + try { + for (let pass = 0; pass < 2; pass += 1) { + await reaper.run(); + await harness.engine.settle(); + } + expect( + harness.bus + .replay() + .filter( + (event) => + event.event === "device.shutdown" && + event.payload.deviceId === previous.id && + event.payload.initiator === "cleanup-reaper", + ), + "a device booted for a queued requester must not be reaped", + ).toEqual([]); + expect(state()).toBe("ready"); + await harness.engine.cancelPending("behind"); + expect( + [...(await harness.engine.core.targetedDevices())], + "cancelling the waiter removes cleanup protection immediately", + ).not.toContain(previous.id); + } finally { + reaper.dispose(); + await harness.engine.cancelPending("head"); + await harness.engine.cancelPending("behind"); + await Promise.all(pending); + harness.engine.dispose(); + } + }); +}); diff --git a/src/leasing/create-leasing-warm-targets.test.ts b/src/leasing/create-leasing-warm-targets.test.ts new file mode 100644 index 000000000..60a78638e --- /dev/null +++ b/src/leasing/create-leasing-warm-targets.test.ts @@ -0,0 +1,278 @@ +import { describe, expect, it, vi } from "vitest"; +import { DriverCrashError, type LeaseProgress } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, seedReady, flush } from "./test-create-leasing.js"; + +describe("createLeasing warm targets", () => { + const target = { + count: 2, + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + } as const; + const roomy = { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 4, maxRunning: 4 }, + maxRunning: 4, + }; + const eventsNamed = (harness: Awaited>, name: string) => + harness.bus.replay().filter((event) => event.event === name); + + async function drain( + harness: Awaited>, + driverClock?: FakeClock, + stepMs = 50, + ) { + for (let round = 0; round < 40; round += 1) { + await flush(); + harness.clock.advance(stepMs); + driverClock?.advance(stepMs); + } + await flush(); + await harness.engine.settle(); + } + + it("after daemon.started with a target of two and no devices, creates two devices one after the other, the second only after the first is ready", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { makeReady: 50 }, + platform: "ios", + }); + const harness = await createHarness({ driver, limits: roomy, warmPool: { targets: [target] } }); + await harness.engine.convergeRunningCapacity(); + + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await flush(); + // why: The first is mid-boot: created, not yet ready, and no second one started. + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); + expect(eventsNamed(harness, "device.ready")).toHaveLength(0); + await drain(harness, clock); + + const order = harness.bus + .replay() + .map((event) => event.event) + .filter((name) => name === "device.provisioned" || name === "device.ready"); + expect(order).toEqual([ + "device.provisioned", + "device.ready", + "device.provisioned", + "device.ready", + ]); + expect(harness.registry.snapshot.devices.map((device) => device.state)).toEqual([ + "ready", + "ready", + ]); + }); + + it("on a graceful drain finishes the creation in flight and creates no further device for the target", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { makeReady: 50 }, + platform: "ios", + }); + const harness = await createHarness({ driver, limits: roomy, warmPool: { targets: [target] } }); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await flush(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); + + const draining = harness.engine.core.drain(); + await drain(harness, clock); + await draining; + + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); + expect(harness.registry.snapshot.devices.map((device) => device.state)).toEqual(["ready"]); + }); + + it("creates no device for a target once the warm pool is closed for a stop", async () => { + const harness = await createHarness({ limits: roomy, warmPool: { targets: [target] } }); + await harness.engine.convergeRunningCapacity(); + harness.engine.core.closeWarmPool(); + + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + + expect(eventsNamed(harness, "device.provisioned")).toEqual([]); + }); + + it("grants the first lease of the targeted kind from a ready device with no booting stage, then keeps the count by creating another", async () => { + const harness = await createHarness({ limits: roomy, warmPool: { targets: [target] } }); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + const progress: LeaseProgress[] = []; + + const granted = await harness.engine.request(request, { + onProgress: (update) => progress.push(update), + ownerId: "agent", + requesterId: "agent", + ttlMs: 3_600_000, + }); + await drain(harness); + + expect(progress.map((update) => update.stage)).not.toContain("booting"); + expect(progress.map((update) => update.stage)).not.toContain("provisioning"); + expect(eventsNamed(harness, "lease.granted")[0]).toMatchObject({ payload: { source: "warm" } }); + const states = harness.registry.snapshot.devices.map((device) => device.state).sort(); + expect(states).toEqual(["leased", "ready", "ready"]); + expect(granted.device.spec).toMatchObject({ model: "iPhone 16", osVersion: "26.5" }); + }); + + it("makes one creation attempt per allowed attempt when every boot fails, and recovers when one succeeds", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("makeReady", 1, new DriverCrashError("no boot")); + driver.failOn("makeReady", 2, new DriverCrashError("no boot")); + const harness = await createHarness({ + driver, + limits: roomy, + warmPool: { targets: [{ ...target, count: 1 }] }, + }); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await harness.engine.settle(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); + + harness.clock.advance(60_000 - 1); + await harness.engine.settle(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); + harness.clock.advance(1); + await harness.engine.settle(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); + + harness.clock.advance(2 * 60_000 - 1); + await harness.engine.settle(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); + harness.clock.advance(1); + await harness.engine.settle(); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(3); + expect( + harness.registry.snapshot.devices.filter((device) => device.state === "ready"), + ).toHaveLength(1); + }); + + it("with a target naming an OS that is not installed, starts no install", async () => { + const install = vi.fn(async () => ({ outcome: "installed" as const, version: "27.0" })); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const harness = await createHarness({ + components: { claimProvision: () => () => undefined, install }, + driver, + limits: roomy, + warmPool: { targets: [{ ...target, osVersion: "27.0" }] }, + }); + await harness.engine.convergeRunningCapacity(); + + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + + expect(install).not.toHaveBeenCalled(); + expect(eventsNamed(harness, "component.install-started")).toEqual([]); + expect(driver.calls.filter((call) => call.operation === "installComponent")).toEqual([]); + expect(harness.registry.snapshot.devices).toEqual([]); + }); + + it("under fresh identity, creates a device before any lease, and after that lease ends and the spent device is deleted, creates the next", async () => { + const harness = await createHarness({ + identity: { android: "reusable", ios: "fresh" }, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + warmPool: { targets: [{ ...target, count: 1 }] }, + }); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + const [first] = harness.registry.snapshot.devices; + expect(harness.registry.snapshot.devices).toHaveLength(1); + expect(first).toMatchObject({ leaseIdentity: "fresh", state: "ready" }); + + const granted = await harness.engine.request(request, { + ownerId: "a", + requesterId: "a", + ttlMs: 3_600_000, + }); + expect(granted.device.id).toBe(first?.id); + await harness.engine.release(granted.lease.id, "explicit"); + await drain(harness); + + const devices = harness.registry.snapshot.devices.filter( + (device) => device.state !== "deleted", + ); + expect( + harness.registry.snapshot.devices.find((device) => device.id === first?.id), + ).toMatchObject({ state: "deleted" }); + expect(devices).toHaveLength(1); + expect(devices[0]).toMatchObject({ leaseIdentity: "fresh", state: "ready" }); + expect(devices[0]?.id).not.toBe(first?.id); + expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); + }); + + it("shuts a never-leased ready device no target counts down with initiator warm-pool once it has been idle past idle.shutdownAfterMs", async () => { + const harness = await createHarness({ limits: roomy }); + const device = await seedReady(harness); + await harness.engine.convergeRunningCapacity(); + + // why: a pass runs on a trigger, and the periodic tick is 30 s away: a daemon.started fact is one + // at the threshold and one millisecond past it. + harness.clock.advance(10_000); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( + "ready", + ); + expect(eventsNamed(harness, "device.shutdown")).toEqual([]); + + harness.clock.advance(1); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + + expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( + "shutdown", + ); + expect(eventsNamed(harness, "device.shutdown")).toMatchObject([ + { payload: { deviceId: device.id, initiator: "warm-pool" } }, + ]); + }); + + it("keeps a never-leased ready device a target counts past idle.shutdownAfterMs", async () => { + const harness = await createHarness({ + limits: roomy, + warmPool: { targets: [{ ...target, count: 1 }] }, + }); + const device = await seedReady(harness); + await harness.engine.convergeRunningCapacity(); + harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); + await drain(harness); + + harness.clock.advance(5 * 60_000); + await harness.engine.settle(); + + expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( + "ready", + ); + expect(eventsNamed(harness, "device.shutdown")).toEqual([]); + // why: The reaper reads the same set through the core, so it leaves the device alone too. + expect([...(await harness.engine.core.targetedDevices())]).toEqual([device.id]); + }); +}); + +describe("createLeasing's port for the warm pool", () => { + it("answers each platform's configured default mode, and full where none is configured", async () => { + const { engine } = await createHarness({ defaultModes: { ios: "slim" } }); + + const { defaultMode } = engine.leasing.corePorts.warmPoolDemand; + + expect([defaultMode("ios"), defaultMode("android")]).toEqual(["slim", "full"]); + }); +}); diff --git a/src/leasing/create-leasing-wiring-and-logger.test.ts b/src/leasing/create-leasing-wiring-and-logger.test.ts new file mode 100644 index 000000000..3215517bd --- /dev/null +++ b/src/leasing/create-leasing-wiring-and-logger.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it, vi } from "vitest"; +import { DriverCrashError } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock, JsonLinesLogger, type Logger, MemoryLogSink } from "../ports/index.js"; +import { createLeasing, NoCapacityError, LeaseHealthMonitor } from "./index.js"; +import { request, config, createHarness, seedReady, flush } from "./test-create-leasing.js"; + +describe("createLeasing logger wiring", () => { + function debugLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { + const sink = new MemoryLogSink(); + return { logger: new JsonLinesLogger({ clock: new FakeClock(0), level: "debug", sink }), sink }; + } + + it("createCore hands its logger to the quarantine coordinator: a failed retry is logged", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + driver.failOn("destroy", 2, new DriverCrashError("retry exploded")); + const { logger, sink } = debugLogger(); + const harness = await createHarness({ driver, logger }); + const first = await harness.engine.request(request, { + ownerId: "first", + requesterId: "first", + }); + await harness.engine.release(first.lease.id, "explicit"); + await harness.engine.settle(); + expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); + + harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); + await flush(); + + expect( + sink.records.filter((record) => record.message === "quarantine delete retry failed"), + ).toMatchObject([ + { + level: "warn", + module: "daemon.quarantine-coordinator", + fields: { + attempts: 1, + deviceId: first.device.id, + error: "DriverCrashError: retry exploded", + step: "delete", + }, + }, + ]); + }); + + it("createLeasing hands its logger to the acquisition coordinator: a failed eviction is logged", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("shutdown", 1, new DriverCrashError("cannot stop victim")); + const { logger, sink } = debugLogger(); + const harness = await createHarness({ + driver, + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 2, maxRunning: 1 }, + maxRunning: 1, + }, + logger, + }); + const victim = await seedReady(harness); + + await expect( + harness.engine.request( + { ...request, model: "iPhone SE" }, + { noWait: true, ownerId: "new-spec", requesterId: "new-spec" }, + ), + ).rejects.toBeInstanceOf(NoCapacityError); + + expect( + sink.records.filter((record) => record.message === "shutting down an eviction target failed"), + ).toMatchObject([ + { + level: "warn", + module: "daemon.lease-acquisition-coordinator", + fields: { deviceId: victim.id, requesterId: "new-spec", step: "shutdown" }, + }, + ]); + }); + + it("createLeasing hands its logger to the lease release coordinator: a background reclaim that cannot commit is logged", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { reclaim: 20 }, + platform: "ios", + }); + const { logger, sink } = debugLogger(); + const harness = await createHarness({ driver, logger }); + const first = await harness.engine.request(request, { + ownerId: "first", + requesterId: "first", + }); + await harness.engine.release(first.lease.id, "explicit"); + // why: The release has committed and the erase is still running. A disk that refuses the write + // makes the reclaim's own commit throw, with no caller left to reject to. + vi.spyOn(harness.state, "transaction").mockImplementation(() => { + throw new Error("EIO"); + }); + + clock.advance(20); + await harness.engine.settle(); + + expect( + sink.records.filter((record) => record.message === "background reclaim failed"), + ).toMatchObject([ + { + level: "error", + module: "daemon.lease-release-coordinator", + fields: { deviceId: first.device.id, leaseId: first.lease.id }, + }, + ]); + }); +}); + +describe("createLeasing wiring", () => { + const components = { + claimProvision: () => () => undefined, + install: async () => ({ outcome: "installed" as const, version: "26.5" }), + }; + + it("wires the leased-device health monitor, and leaves it out when told to", async () => { + const harness = await createHarness(); + const without = createLeasing({ + clock: harness.clock, + components, + config: config(), + core: harness.engine.core, + eventBus: harness.bus, + healthMonitor: false, + idGenerator: { generate: () => "1" }, + }); + + expect(harness.engine.leasing.healthMonitor).toBeInstanceOf(LeaseHealthMonitor); + expect(without.healthMonitor).toBeUndefined(); + }); + + it("hands core a leaseExpirer that expires through leasing: doctor --fix ends a lease past its deadline", async () => { + const harness = await createHarness(); + const grant = await harness.engine.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + // why: No timer may expire it first: only doctor's finding does. + harness.engine.dispose(); + harness.clock.advance(10 * 60_000 * 60); + + await harness.engine.core.doctor.reconcile({ fix: true }); + + expect(harness.registry.snapshot.leases.map((lease) => lease.id)).not.toContain(grant.lease.id); + expect(harness.bus.replay().map((event) => event.event)).toContain("lease.expired"); + }); + + it("disposes the physical device watcher with leasing: no presence read happens after dispose", async () => { + const harness = await createHarness(); + await harness.registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: "UDID-A", + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + harness.driver.setPhysicalPresence("UDID-A", true); + harness.engine.leasing.physicalDeviceWatcher.start(); + harness.engine.dispose(); + + harness.clock.advance(120_000); + await harness.engine.settle(); + + expect(harness.driver.calls.filter((call) => call.operation === "listPresentPhysical")).toEqual( + [], + ); + }); + + it("does not connect core itself when a second leasing is built", async () => { + const harness = await createHarness(); + const connect = vi.spyOn(harness.engine.core, "connect"); + + createLeasing({ + clock: harness.clock, + components, + config: config(), + core: harness.engine.core, + eventBus: harness.bus, + idGenerator: { generate: () => "1" }, + }); + + expect(connect).not.toHaveBeenCalled(); + }); +}); diff --git a/src/leasing/create-leasing.test.ts b/src/leasing/create-leasing.test.ts index eea401ea8..6f97cd1db 100644 --- a/src/leasing/create-leasing.test.ts +++ b/src/leasing/create-leasing.test.ts @@ -1,268 +1,10 @@ -import { describe, expect, it, vi } from "vitest"; -import { - buildCapacityFigures, - CleanupReaper, - type ModelPreferences, - BootTimeoutError, - type ComponentInstaller, - type Config, - type LeaseGrant, - type DeviceMode, - type DeviceSpec, - type DriverDevice, - type DriverEstimate, - DriverCrashError, - type VirtualSpec, - type LeaseProgress, - type Registry, - UnknownLeaseError, -} from "../core/index.js"; -import { type CapacityLimits, type ResourceStrategyOptions, testStateOf } from "../core/testing.js"; -import { - capacityChangedPayload, - FakeDriver, - loadTestRegistry, - openTestState, - testComponentWiring, -} from "../core/testing.js"; -import type { StateDatabase } from "../state/index.js"; - -import { EventBus, type EventMap } from "../bus/index.js"; -import { - FakeClock, - FakeSystemStats, - JsonLinesLogger, - type Logger, - MemoryLogSink, - MemoryFilesystem, -} from "../ports/index.js"; -import { createLeasing } from "./index.js"; -import { - DaemonUpdatingError, - NoCapacityError, - ProvisionFailedError, -} from "./lease-acquisition-coordinator.js"; -import { LeaseHealthMonitor } from "./lease-health-monitor.js"; -import type { LeaseEndParticipant } from "./lease-ports.js"; -import { createTestEngine, type TestEngineConfig } from "./testing.js"; -import { QueueTimeoutError, RequestCancelledError } from "./wait-queue.js"; - -const gibibyte = 1024 ** 3; -const request = { - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", -} as const; - -const androidRequest = { - model: "Pixel 9", - osVersion: "36", - platform: "android", -} as const; - -function config( - overrides: Partial = {}, - warmPoolEnabled = true, - idleShutdownAfterMs = 10_000, - warmPool: Partial = {}, -): TestEngineConfig { - return { - devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, - health: { - enabled: true, - maxConcurrentRecoveries: 1, - maxRecoveryAttempts: 3, - probeIntervalMs: 30_000, - recoveryBackoffMs: 5_000, - stableObservations: 2, - }, - stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, - idle: { deleteAfterMs: 60_000, shutdownAfterMs: idleShutdownAfterMs }, - lease: { - defaultTtlMs: 100, - maxTtlMs: 14_400_000, - identity: { ios: "reusable", android: "reusable" }, - requestRetentionMs: 600_000, - endedIdRetentionMs: 600_000, - maxRequestRecords: 10_000, - maxPerRequester: 1, - endWaitMs: 600_000, - ...overrides, - }, - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 1 + 1, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - }, - warmPool: { - enabled: warmPoolEnabled, - maxConcurrentBoots: 1, - reserveRunning: { android: 0, ios: 0 }, - targets: [], - quarantine: { - maxRetries: 3, - maxRetryBackoffMs: 300_000, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - }, - ...warmPool, - }, - }; -} - -function resourceOptions(source: Pick): ResourceStrategyOptions { - if (source.capacity.strategy !== "resource") throw new Error("expected the resource strategy"); - return source.capacity.config; -} - -function withCapacity( - base: ResourceStrategyOptions, - overrides: Partial, -): ResourceStrategyOptions { - return { - limits: overrides.limits ?? base.limits, - ramBudget: overrides.ramBudget ?? base.ramBudget, - }; -} - -function capacityConfig( - base: Pick, - options: Partial & { readonly capacity?: Config["capacity"] }, -): Config["capacity"] { - return ( - options.capacity ?? { - strategy: "resource", - config: withCapacity(resourceOptions(base), options), - } - ); -} - -// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. -async function createHarness( - options: { - readonly components?: Pick; - readonly driver?: FakeDriver; - readonly drivers?: readonly FakeDriver[]; - readonly state?: StateDatabase; - readonly identity?: Config["lease"]["identity"]; - readonly lease?: Partial; - /** why: a participant holds a released device in `reclaiming` until it returns. */ - readonly leaseEndParticipants?: readonly LeaseEndParticipant[]; - readonly capacity?: Config["capacity"]; - readonly defaultModes?: Parameters[0]["defaultModes"]; - readonly limits?: CapacityLimits; - readonly logger?: Logger; - readonly modelPreferences?: ModelPreferences; - readonly ramBudget?: ResourceStrategyOptions["ramBudget"]; - readonly totalRamBytes?: number; - readonly warmPoolEnabled?: boolean; - readonly warmPool?: Partial; - /** why: how long a ready device may idle before the pool shuts it down. */ - readonly idleShutdownAfterMs?: number; - /** why: starts the warm pool as the daemon's startup does, so it learns from the leases and reacts to events. */ - readonly startPool?: boolean; - } = {}, -) { - const clock = new FakeClock(1_000); - const state = options.state ?? (await openTestState()); - const bus = new EventBus(clock); - const driver = - options.driver ?? new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - let nextId = options.state === undefined ? 1 : 1_000; - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => `${nextId++}` }, - ...(options.identity === undefined ? {} : { leaseIdentity: options.identity }), - state, - }); - const baseConfig = config( - { - ...options.lease, - ...(options.identity === undefined ? {} : { identity: options.identity }), - }, - options.warmPoolEnabled ?? true, - options.idleShutdownAfterMs, - options.warmPool, - ); - const engineConfig: TestEngineConfig = { - ...baseConfig, - capacity: capacityConfig(baseConfig, options), - }; - const totalRamBytes = options.totalRamBytes ?? 32 * gibibyte; - const drivers = options.drivers ?? [driver]; - const wiring = testComponentWiring({ - clock, - components: options.components, - drivers, - eventBus: bus, - registry, - }); - const engine = createTestEngine({ - ...wiring, - clock, - config: engineConfig, - drivers, - ...(options.defaultModes === undefined ? {} : { defaultModes: options.defaultModes }), - eventBus: bus, - idGenerator: { generate: () => `request-${nextId++}` }, - ...(options.leaseEndParticipants === undefined - ? {} - : { leaseEndParticipants: options.leaseEndParticipants }), - ...(options.logger === undefined ? {} : { logger: options.logger }), - ...(options.modelPreferences === undefined - ? {} - : { modelPreferences: options.modelPreferences }), - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes }), - }); - - if (options.startPool === true) await engine.convergeRunningCapacity(); - return { bus, clock, decisions: wiring.decisions, driver, engine, registry, state }; -} - -async function seedReady( - harness: Awaited>, - spec: VirtualSpec = { ...request, physical: false }, - mode: DeviceMode = "full", - driver: FakeDriver = harness.driver, -) { - const driverDevice = await driver.provision(spec); - // why: Running on the machine too: a daemon start reads that of a leased device to keep its lease. - const booted = await driver.makeReady(driverDevice, { mode, purpose: "prepare" }); - const device = await harness.registry.registerDevice({ - driverData: driverDevice.driverData, - driverDeviceId: driverDevice.deviceId, - provisionDuration: 0, - spec, - }); - return harness.registry.transitionDevice( - device.id, - "ready", - { event: "device.ready", payload: { bootDuration: 0, deviceId: device.id } }, - booted.mode === undefined ? undefined : { mode: booted.mode }, - ); -} - -async function flush(): Promise { - for (let count = 0; count < 100; count += 1) { - await Promise.resolve(); - } -} - -async function settledOrPending(promise: Promise): Promise { - return Promise.race([ - promise.catch((error: unknown) => error), - flush().then(() => "still pending"), - ]); -} +import { describe, expect, it } from "vitest"; +import { DriverCrashError, type LeaseProgress } from "../core/index.js"; +import { FakeDriver, loadTestRegistry, openTestState } from "../core/testing.js"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { request, createHarness, seedReady, flush } from "./test-create-leasing.js"; +import { NoCapacityError, type LeaseEndParticipant } from "./index.js"; describe("createLeasing", () => { it("hands the caller back before the destroy, keeps the released device shut down, then grants the waiter a new device", async () => { @@ -676,4714 +418,4 @@ describe("createLeasing", () => { }), ).resolves.toMatchObject({ device: { spec: { platform: "android" } } }); }); - - it("shuts down an unleased ready device over maxRunning at startup with initiator warm-pool, never a leased one, and is idempotent", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 1 }, - maxRunning: 1, - }, - }); - const leasedDevice = await seedReady(harness); - const unleasedDevice = await seedReady(harness); - await harness.registry.createLease({ - deviceId: leasedDevice.id, - requesterId: "active", - ownerId: "active", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - - await harness.engine.convergeRunningCapacity(); - await harness.engine.settle(); - await harness.engine.convergeRunningCapacity(); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: leasedDevice.id, state: "leased" }, - { id: unleasedDevice.id, state: "shutdown" }, - ]); - expect(harness.driver.calls.filter((call) => call.operation === "shutdown")).toHaveLength(1); - expect(harness.engine.runningCapacity.global.overLimit).toBe(false); - expect(harness.bus.replay().filter((event) => event.event === "device.shutdown")).toMatchObject( - [{ payload: { deviceId: unleasedDevice.id, initiator: "warm-pool" } }], - ); - }); - - it("retains in-limit warm devices at startup and never boots shutdown inventory", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 2, - }, - }); - const warm = await seedReady(harness); - const shutdown = await seedReady(harness); - await harness.registry.transitionDevice(shutdown.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: shutdown.id, initiator: "test" }, - }); - const callsBefore = harness.driver.calls.length; - - await harness.engine.convergeRunningCapacity(); - - expect(harness.registry.snapshot.devices.find((item) => item.id === warm.id)?.state).toBe( - "ready", - ); - expect(harness.registry.snapshot.devices.find((item) => item.id === shutdown.id)?.state).toBe( - "shutdown", - ); - expect(harness.driver.calls.slice(callsBefore).map((call) => call.operation)).toEqual([ - "listManaged", - "listSources", - ]); - }); - - it("shuts an orphaned reclaiming device down and destroys it through its driver", async () => { - const harness = await createHarness(); - const device = await seedReady(harness); - const lease = await harness.registry.createLease({ - deviceId: device.id, - requesterId: "former", - ownerId: "former", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await harness.registry.beginRelease(lease.id); - - await harness.engine.convergeRunningCapacity(); - - expect(harness.registry.snapshot.devices[0]?.state).toBe("deleted"); - const operations = harness.driver.calls.map((call) => call.operation); - expect(operations).toContain("shutdown"); - expect(operations).toContain("destroy"); - }); - - it("reports unavoidable leased running overage", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 1 }, - maxRunning: 1, - }, - }); - for (const requesterId of ["one", "two"]) { - const device = await seedReady(harness); - await harness.registry.createLease({ - deviceId: device.id, - ownerId: requesterId, - requesterId, - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - } - - await harness.engine.convergeRunningCapacity(); - - expect(harness.driver.calls.filter((call) => call.operation === "shutdown")).toHaveLength(0); - expect(harness.engine.runningCapacity.global.overLimit).toBe(true); - await expect( - harness.engine.request(request, { - noWait: true, - ownerId: "three", - requesterId: "three", - }), - ).rejects.toBeInstanceOf(NoCapacityError); - }); - - it("holds one running reservation across provision and readiness", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - platform: "ios", - }); - driver.hangMakeReady(); - const harness = await createHarness({ driver }); - - const first = harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const second = harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - await flush(); - - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); - expect(harness.engine.runningCapacity.global.reserved).toBe(1); - expect(harness.engine.queueDepth).toBe(1); - - driver.releaseMakeReady(); - const grant = await first; - await harness.engine.release(grant.lease.id, "explicit"); - await expect(second).resolves.toMatchObject({ - lease: { ownerId: "agent-2", requesterId: "agent-2" }, - }); - }); - - it("starts exactly one of two existing shutdown devices at running capacity one", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const harness = await createHarness({ driver }); - for (let index = 0; index < 2; index += 1) { - const device = await seedReady(harness); - await harness.registry.transitionDevice(device.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: device.id, initiator: "test" }, - }); - } - const seededBoots = driver.calls.filter((call) => call.operation === "makeReady").length; - driver.hangMakeReady(); - - void harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - void harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - await flush(); - - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength( - seededBoots + 1, - ); - expect(harness.engine.queueDepth).toBe(1); - driver.releaseMakeReady(); - }); - - it("serializes simultaneous requests so one device of capacity starts exactly one provision", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { provision: 20 }, - platform: "ios", - }); - const harness = await createHarness({ driver }); - - const first = harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const second = harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - await flush(); - - expect(harness.engine.queueDepth).toBe(1); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - clock.advance(20); - await first; - await flush(); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - - const released = (await first).device.id; - await harness.engine.release((await first).lease.id, "explicit"); - await flush(); - clock.advance(20); - const granted = await second; - expect(granted).toMatchObject({ lease: { ownerId: "agent-2", requesterId: "agent-2" } }); - expect(granted.device.id).not.toBe(released); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("grants an existing matching ready device and does not match a different spec", async () => { - const harness = await createHarness(); - const matching = await seedReady(harness); - await seedReady(harness, { - physical: false, - model: "iPhone SE", - osVersion: "26.5", - platform: "ios", - }); - const progress: string[] = []; - const provisions = () => - harness.driver.calls.filter((call) => call.operation === "provision").length; - const provisionedBefore = provisions(); - - const grant = await harness.engine.request(request, { - onProgress: (update) => progress.push(update.stage), - requesterId: "agent-1", - ownerId: "agent-1", - }); - - expect(grant.device.id).toBe(matching.id); - expect(progress).toEqual([]); - expect(provisions()).toBe(provisionedBefore); - }); - - it("provisions and makes a device ready when capacity is available, returning progress estimates", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - estimateMs: { boot: 20, provision: 10, reclaim: 34 }, - platform: "ios", - }); - const harness = await createHarness({ driver }); - - const progress: string[] = []; - const grant = await harness.engine.request(request, { - onProgress: (update) => progress.push(update.stage), - requesterId: "agent-1", - ownerId: "agent-1", - }); - - // why: `estimatedReclaimMs` is the one figure describing work still ahead of the holder -- - // what releasing this device will cost -- rather than work already done to grant it. - expect(grant).toMatchObject({ - device: { spec: request, state: "leased" }, - timing: { - estimatedBootMs: 20, - estimatedProvisionMs: 10, - estimatedReadyMs: 30, - estimatedReclaimMs: 34, - }, - }); - expect(driver.calls.map((call) => call.operation)).toEqual([ - "resolveSpec", - "provision", - "makeReady", - ]); - }); - - it("prices the eventual reclaim even when a warm device is granted with no work at all", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - estimateMs: { boot: 20, provision: 10, reclaim: 34 }, - platform: "ios", - }); - const harness = await createHarness({ driver }); - await seedReady(harness); - - const grant = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - - expect(grant.timing).toEqual({ - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReadyMs: 0, - estimatedReclaimMs: 34, - }); - }); - - it("reports only the selected work as it begins, quoting a shut-down device's own prepare-boot figure", async () => { - const clock = new FakeClock(1_000); - // why: A real driver quotes a persisted-baseline boot below its cold figure; the fake does so only - // for an estimate that names the device and the prepare purpose. - class DeviceQuotingDriver extends FakeDriver { - override estimate(estimate: DriverEstimate, spec: DeviceSpec): number { - if (estimate.operation === "boot" && estimate.purpose === "prepare") { - return estimate.device === undefined ? 99 : 4; - } - return super.estimate(estimate, spec); - } - } - const driver = new DeviceQuotingDriver({ - availableOsVersions: ["26.5"], - clock, - estimateMs: { boot: 20, provision: 10, reclaim: 15 }, - platform: "ios", - }); - const harness = await createHarness({ driver, idleShutdownAfterMs: 0 }); - const provisioned: string[] = []; - - const provisionedGrant = await harness.engine.request(request, { - onProgress: (progress) => provisioned.push(progress.stage), - requesterId: "provisioned", - ownerId: "provisioned", - }); - expect(provisioned).toEqual(["provisioning", "booting"]); - await harness.engine.release(provisionedGrant.lease.id, "explicit"); - await harness.engine.settle(); - - // why: a released virtual device is destroyed, so the shut-down device is seeded. - const shutdown = await seedReady(harness); - await harness.registry.transitionDevice(shutdown.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: shutdown.id, initiator: "test" }, - }); - const booted: string[] = []; - const bootEtas: (number | undefined)[] = []; - const bootedGrant = await harness.engine.request(request, { - onProgress: (progress) => { - booted.push(progress.stage); - bootEtas.push(progress.stage === "booting" ? progress.etaMs : undefined); - }, - requesterId: "shutdown", - ownerId: "shutdown", - }); - expect(booted).toEqual(["booting"]); - expect(bootEtas).toEqual([4]); - expect(bootedGrant.timing.estimatedBootMs).toBe(4); - }); - - it("grants a shut-down device with the cold boot figure when its own boot estimate rejects", async () => { - const clock = new FakeClock(1_000); - class RejectingDeviceEstimateDriver extends FakeDriver { - override estimate(estimate: DriverEstimate, spec: DeviceSpec): number { - if (estimate.operation === "boot" && estimate.device !== undefined) { - return Promise.reject(new Error("emulator -version failed")) as unknown as number; - } - return super.estimate(estimate, spec); - } - } - const driver = new RejectingDeviceEstimateDriver({ - availableOsVersions: ["26.5"], - clock, - estimateMs: { boot: 20, provision: 10, reclaim: 15 }, - platform: "ios", - }); - const harness = await createHarness({ driver, idleShutdownAfterMs: 0 }); - const shutdown = await seedReady(harness); - await harness.registry.transitionDevice(shutdown.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: shutdown.id, initiator: "test" }, - }); - - const granted = harness.engine - .request(request, { requesterId: "b", ownerId: "b" }) - .then((grant) => grant.timing.estimatedBootMs); - // why: Without the fallback the waiter hangs, so the race turns that hang into a named failure. - const outcome = await Promise.race([ - granted, - new Promise((resolve) => setTimeout(() => resolve("still pending"), 1_000)), - ]); - - expect(outcome).toBe(20); - }); - - it("reports queue insertion without speculative work and isolates callback failures", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const harness = await createHarness({ driver }); - const holder = await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - const progress: string[] = []; - const queued = harness.engine.request(request, { - onProgress: (update) => progress.push(update.stage), - requesterId: "queued", - ownerId: "queued", - }); - await flush(); - expect(progress).toEqual(["queued"]); - - await expect( - harness.engine.request(request, { - noWait: true, - onProgress: () => progress.push("unexpected"), - requesterId: "no-wait", - ownerId: "no-wait", - }), - ).rejects.toBeInstanceOf(NoCapacityError); - expect(progress).toEqual(["queued"]); - - await harness.engine.release(holder.lease.id, "explicit"); - await queued; - - const callbackFailure = await createHarness(); - await expect( - callbackFailure.engine.request(request, { - onProgress: () => { - throw new Error("client disconnected"); - }, - requesterId: "throwing-callback", - ownerId: "throwing-callback", - }), - ).resolves.toMatchObject({ device: { state: "leased" } }); - expect(callbackFailure.driver.calls.map((call) => call.operation)).toContain("makeReady"); - }); - - it("cancels a queued request through the QueueControl facade and frees the requester for a later grant", async () => { - const harness = await createHarness(); - const holder = await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - const queued = harness.engine.request(request, { - ownerId: "queued", - requesterId: "queued", - }); - await flush(); - - await expect(harness.engine.cancelPending("nobody")).resolves.toBe("not-found"); - await expect(harness.engine.cancelPending("queued")).resolves.toBe("cancelled"); - await expect(queued).rejects.toBeInstanceOf(RequestCancelledError); - - await harness.engine.release(holder.lease.id, "explicit"); - await expect( - harness.engine.request(request, { ownerId: "queued", requesterId: "queued" }), - ).resolves.toMatchObject({ lease: { ownerId: "queued", requesterId: "queued" } }); - }); - - it("stamps queue.changed with the wait-queue module when a request queues and when it leaves", async () => { - const harness = await createHarness(); - const modules: { depth: number; module: string }[] = []; - harness.bus.subscribe("queue.changed", (envelope) => - modules.push({ depth: envelope.payload.depth, module: envelope.module }), - ); - const holder = await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - const queued = harness.engine.request(request, { - ownerId: "queued", - requesterId: "queued", - }); - await flush(); - expect(modules).toEqual([{ depth: 1, module: "wait-queue" }]); - - await harness.engine.release(holder.lease.id, "explicit"); - await queued; - expect(modules).toEqual([ - { depth: 1, module: "wait-queue" }, - { depth: 0, module: "wait-queue" }, - ]); - }); - - it("queues at capacity in FIFO order across three waiters", async () => { - const harness = await createHarness(); - const first = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - // why: Each waiter records its grant and hands the device straight back, so the queue drains in - // whatever order it serves and the assertion below sees that order rather than hanging. - const granted: string[] = []; - const waiters = ["agent-2", "agent-3", "agent-4"].map((requesterId) => - harness.engine.request(request, { ownerId: requesterId, requesterId }).then(async (grant) => { - granted.push(grant.lease.requesterId); - await harness.engine.release(grant.lease.id, "explicit"); - }), - ); - await flush(); - - await harness.engine.release(first.lease.id, "explicit"); - await Promise.all(waiters); - - expect(granted).toEqual(["agent-2", "agent-3", "agent-4"]); - }); - - it("wakes exactly the queue head on release and grants it a new device, not the destroyed one", async () => { - const harness = await createHarness(); - const first = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const second = harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - const third = harness.engine.request(request, { - ownerId: "agent-3", - requesterId: "agent-3", - }); - await flush(); - - await harness.engine.release(first.lease.id, "explicit"); - const secondGrant = await second; - await flush(); - - expect(secondGrant.device.id).not.toBe(first.device.id); - expect(harness.registry.snapshot.leases).toHaveLength(1); - expect(harness.registry.snapshot.leases[0]?.requesterId).toBe("agent-2"); - expect(harness.driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - await harness.engine.release(secondGrant.lease.id, "explicit"); - await expect(third).resolves.toMatchObject({ - lease: { ownerId: "agent-3", requesterId: "agent-3" }, - }); - }); - - it("provisions a new device for the queue head instead of deadlocking at capacity", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const harness = await createHarness({ driver }); - const first = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const queued = harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - await flush(); - - await harness.engine.release(first.lease.id, "explicit"); - - const granted = await queued; - expect(granted.device.id).not.toBe(first.device.id); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2); - }); - - it("rejects a timed-out queue entry and skips it on a later release", async () => { - const harness = await createHarness(); - const first = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const progress: string[] = []; - const timedOut = harness.engine.request(request, { - onProgress: (update) => progress.push(update.stage), - requesterId: "agent-2", - ownerId: "agent-2", - timeoutMs: 10, - }); - const next = harness.engine.request(request, { - ownerId: "agent-3", - requesterId: "agent-3", - }); - await flush(); - - // why: Each outcome settles against a flush rather than an await, so a waiter left pending - // fails the assertion that names it instead of hanging the test. - harness.clock.advance(10); - expect(await settledOrPending(timedOut)).toBeInstanceOf(QueueTimeoutError); - expect(progress).toEqual(["queued"]); - await harness.engine.release(first.lease.id, "explicit"); - - expect(await settledOrPending(next)).toMatchObject({ - lease: { ownerId: "agent-3", requesterId: "agent-3" }, - }); - }); - - it("expires an un-renewed lease at its deadline and serves the queue with its device", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); - await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - const queued = harness.engine.request(request, { - ownerId: "agent-2", - requesterId: "agent-2", - }); - await flush(); - - harness.clock.advance(9); - await flush(); - expect(await settledOrPending(queued)).toBe("still pending"); - - harness.clock.advance(1); - await flush(); - await expect(queued).resolves.toMatchObject({ - lease: { ownerId: "agent-2", requesterId: "agent-2" }, - }); - }); - - it("renews via the engine, re-arming the expiry timer at the new deadline", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); - const granted = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - - harness.clock.advance(6); - const renewed = await harness.engine.renew(granted.lease.id, 20); - expect(renewed).toMatchObject({ lastRenewedAt: 1_006, ttlMs: 20, ttlDeadline: 1_026 }); - expect(harness.bus.replay().find((event) => event.event === "lease.renewed")?.payload).toEqual({ - leaseId: granted.lease.id, - newDeadline: 1_026, - }); - - // why: The grant-time deadline (1_010) has now passed, but the renewal re-armed the expiry - // timer at the new one, so the lease must still be alive. - harness.clock.advance(4); - await flush(); - expect(harness.registry.snapshot.leases).toHaveLength(1); - - harness.clock.advance(16); - await flush(); - expect(harness.registry.snapshot.leases).toHaveLength(0); - }); - - it("grants at the request's own ttlMs, and re-applies that width on a body-less renew", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 10 } }); - const granted = await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - ttlMs: 40, - }); - expect(granted.lease).toMatchObject({ ttlMs: 40, ttlDeadline: 1_040 }); - - harness.clock.advance(5); - // why: No TTL named: the lease keeps its own 40ms width rather than snapping back to the - // 10ms `lease.defaultTtlMs` (ADR 0004 §4). - await expect(harness.engine.renew(granted.lease.id)).resolves.toMatchObject({ - ttlMs: 40, - ttlDeadline: 1_045, - }); - }); - - it("retries after a provision failure without leaking capacity", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("provision", 1, new DriverCrashError("simulator exited")); - const harness = await createHarness({ driver }); - - // why: A request that never retried would sit pending; settling against a flush names that. - const grant = await settledOrPending( - harness.engine.request(request, { ownerId: "agent-1", requesterId: "agent-1" }), - ); - - expect(grant).toMatchObject({ device: { state: "leased" } }); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - expect( - harness.registry.snapshot.devices.filter((device) => device.state !== "deleted"), - ).toHaveLength(1); - }); - - it("destroys a registered device and returns BootTimeoutError when readiness fails", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("makeReady", 1, new Error("boot failed")); - const harness = await createHarness({ driver }); - - await expect( - harness.engine.request(request, { ownerId: "agent-1", requesterId: "agent-1" }), - ).rejects.toBeInstanceOf(BootTimeoutError); - - expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); - expect(driver.calls.map((call) => call.operation)).toContain("destroy"); - }); - - it("emits committed happy-path facts in lifecycle order", async () => { - const harness = await createHarness(); - - await harness.engine.request(request, { - ownerId: "agent-1", - requesterId: "agent-1", - }); - - expect(harness.bus.replay().map((event) => event.event)).toEqual([ - "lease.requested", - "device.provisioned", - "device.ready", - "lease.granted", - ]); - }); -}); - -describe("createLeasing RAM budget by mode", () => { - const slimRequest = { ...request, mode: "slim" } as const; - const roomy: CapacityLimits = { - android: { maxDevices: 4, maxRunning: 4 }, - ios: { maxDevices: 8, maxRunning: 8 }, - maxRunning: 12, - }; - /** why: With 9 GiB of RAM the budget is 5 GiB: one full iOS device (3 GiB) and two slim ones fit. */ - const ramBudget = { - androidBytesPerDevice: 4 * gibibyte, - iosBytesPerDevice: 3 * gibibyte, - iosSlimBytesPerDevice: gibibyte, - }; - - function operations(driver: FakeDriver, operation: string): number { - return driver.calls.filter((call) => call.operation === operation).length; - } - - it("refuses a full spec, which a slim request on a runtime that cannot be slimmed resolves to, before it is provisioned, and counts it full after it boots", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.4", "26.5"], - clock, - platform: "ios", - slimmableOsVersions: ["26.4"], - }); - const harness = await createHarness({ - driver, - limits: roomy, - ramBudget, - totalRamBytes: 9 * gibibyte, - }); - - const unslimmable = await harness.engine.request(slimRequest, { - ownerId: "unslimmable", - requesterId: "unslimmable", - }); - expect(unslimmable.device.mode).toBe("full"); - expect(harness.engine.ramBudget?.usedBytes).toBe(3 * gibibyte); - - // why: 3 GiB used of 5: the next device needs 3 more and is refused before any provision. - const provisions = operations(driver, "provision"); - await expect( - harness.engine.request(slimRequest, { noWait: true, ownerId: "next", requesterId: "next" }), - ).rejects.toBeInstanceOf(NoCapacityError); - expect(operations(driver, "provision")).toBe(provisions); - - // why: Control: had the first device come up slim, 1 GiB would be used and the next would fit. - const control = await createHarness({ - driver: new FakeDriver({ - availableOsVersions: ["26.4"], - clock: new FakeClock(1_000), - platform: "ios", - slimmableOsVersions: ["26.4"], - }), - limits: roomy, - ramBudget, - totalRamBytes: 9 * gibibyte, - }); - const slimmable = { ...slimRequest, osVersion: "26.4" }; - await control.engine.request(slimmable, { ownerId: "first", requesterId: "first" }); - await expect( - control.engine.request(slimmable, { noWait: true, ownerId: "next", requesterId: "next" }), - ).resolves.toMatchObject({ device: { mode: "slim" } }); - }); - - it("counts a slim-spec device quarantined from provisioning at the full size", async () => { - const harness = await createHarness({ limits: roomy, ramBudget, totalRamBytes: 9 * gibibyte }); - const registered = await harness.registry.registerDevice({ - driverData: { fakeDeviceId: "driver-stalled" }, - driverDeviceId: "driver-stalled", - provisionDuration: 0, - spec: { ...slimRequest, physical: false }, - }); - const quarantined = await harness.registry.enterQuarantine(registered.id, 60_000, { - kind: "purge-failed", - }); - - expect(quarantined.state).toBe("quarantined"); - expect(harness.engine.ramBudget?.usedBytes).toBe(ramBudget.iosBytesPerDevice); - }); - - it("keeps the lease of a device planned as slim whose driver reports full, shuts nothing down, and counts it full", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - mode: "full", - platform: "ios", - slimmableOsVersions: ["26.5"], - }); - const harness = await createHarness({ - driver, - limits: roomy, - ramBudget, - totalRamBytes: 9 * gibibyte, - }); - - const leased = await harness.engine.request(slimRequest, { ownerId: "a", requesterId: "a" }); - - expect(leased.device.mode).toBe("full"); - expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual([leased.lease.id]); - expect(operations(driver, "shutdown")).toBe(0); - expect(harness.engine.ramBudget).toEqual({ - limitBytes: 5 * gibibyte, - overLimit: false, - usedBytes: 3 * gibibyte, - }); - }); - - it("over its limit after a restart with larger sizes, creates no device in either mode and boots no shut-down slim device, but grants an idle one; releasing the idle one destroys it and brings it back under", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - knownModels: ["iPhone 16", "iPhone 16 Pro"], - availableOsVersions: ["26.4", "26.5"], - clock, - platform: "ios", - slimmableOsVersions: ["26.4", "26.5"], - }); - const small = { ...ramBudget, iosSlimBytesPerDevice: 0.5 * gibibyte }; - const before = await createHarness({ - driver, - idleShutdownAfterMs: 0, - limits: roomy, - ramBudget: small, - totalRamBytes: 9 * gibibyte, - }); - const idleSpec = { ...slimRequest, osVersion: "26.4" }; - const held = await before.engine.request(slimRequest, { ownerId: "held", requesterId: "held" }); - // why: a released virtual device is destroyed, so the idle devices are seeded rather than released. - const toShutDown = await seedReady(before, { ...slimRequest, physical: false }, "slim"); - const idle = await seedReady(before, { ...idleSpec, physical: false }, "slim"); - await before.engine.cleanup.execute({ - action: "shutdown", - reason: "test", - rule: "test", - target: toShutDown.id, - }); - expect(before.engine.ramBudget?.usedBytes).toBe(1 * gibibyte); - - const after = await createHarness({ - driver, - state: before.state, - idleShutdownAfterMs: 0, - limits: roomy, - ramBudget: { ...ramBudget, iosSlimBytesPerDevice: 3 * gibibyte }, - totalRamBytes: 9 * gibibyte, - }); - await after.engine.convergeRunningCapacity(); - expect(after.engine.ramBudget).toEqual({ - limitBytes: 5 * gibibyte, - overLimit: true, - usedBytes: 6 * gibibyte, - }); - - const provisions = operations(driver, "provision"); - const boots = operations(driver, "makeReady"); - for (const [owner, wanted] of [ - ["new-full", request], - ["new-slim", { ...slimRequest, model: "iPhone 16 Pro" }], - ["boot-slim", slimRequest], - ] as const) { - await expect( - after.engine.request(wanted, { noWait: true, ownerId: owner, requesterId: owner }), - ).rejects.toBeInstanceOf(NoCapacityError); - } - expect(operations(driver, "provision")).toBe(provisions); - expect(operations(driver, "makeReady")).toBe(boots); - expect(after.registry.snapshot.devices.find((d) => d.id === toShutDown.id)?.state).toBe( - "shutdown", - ); - - const reused = await after.engine.request(idleSpec, { - noWait: true, - ownerId: "reuse", - requesterId: "reuse", - }); - expect(reused.device.id).toBe(idle.id); - expect(after.registry.snapshot.leases.map((lease) => lease.deviceId)).toContain(held.device.id); - expect(after.engine.ramBudget?.overLimit).toBe(true); - - await after.engine.release(reused.lease.id, "explicit"); - await after.engine.settle(); - await after.engine.cleanup.execute({ - action: "destroy", - reason: "test", - rule: "test", - target: toShutDown.id, - }); - await after.engine.settle(); - expect(operations(driver, "makeReady")).toBe(boots); - expect(after.engine.ramBudget).toEqual({ - limitBytes: 5 * gibibyte, - overLimit: false, - usedBytes: 3 * gibibyte, - }); - }); -}); - -// why: #43: startup convergence used to await each orphaned held lease's device reclaim -// inline (an erase measured ~34s for one simulator), so N orphaned leases cost N -// serial erases before any other request could be served. These cover the shape -// that replaced it: the lease is released registry-only on the convergence path, -// and its reclaim proceeds in the background. -describe("createLeasing startup reclaim backgrounding (#43)", () => { - it("converges without waiting for an in-flight destroy, and a fresh request is served immediately after", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - // why: Matches the issue's measured single-simulator-erase cost -- the old inline - // await would have forced convergence to sit through this. - latencyMs: { destroy: 34_000 }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 2, - }, - }); - // why: under ADR 0004 convergence itself releases nothing -- there is no orphan sweep left -- - // so the in-flight destroy it must not block on is produced the way one actually arises: - // a release commits the registry half and hands the 34s destroy off to the background. - const released = await harness.engine.request(request, { - ownerId: "previous-holder", - requesterId: "previous-holder", - }); - await harness.engine.release(released.lease.id, "explicit"); - - const convergeStartedAt = harness.clock.now(); - await harness.engine.convergeRunningCapacity(); - // why: the clock never had to move for convergence to finish: it did not sit through - // the 34s destroy above. - expect(harness.clock.now()).toBe(convergeStartedAt); - expect( - harness.registry.snapshot.devices.find((device) => device.id === released.device.id)?.state, - ).toBe("shutdown"); - expect(harness.registry.snapshot.leases).toEqual([]); - - // why: a fresh request is servable right away -- capacity allows a second device, and the - // one still mid-destroy never blocks it. Still 0ms elapsed. - const granted = await harness.engine.request(request, { - requesterId: "new-agent", - ownerId: "new-agent", - }); - expect(granted.device.id).not.toBe(released.device.id); - expect(harness.clock.now()).toBe(convergeStartedAt); - }); - - it("never grants a device whose background destroy is still in flight", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { destroy: 34_000 }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 1, - }, - }); - const released = await harness.engine.request(request, { - ownerId: "previous-holder", - requesterId: "previous-holder", - }); - await harness.engine.release(released.lease.id, "explicit"); - - await harness.engine.convergeRunningCapacity(); - expect(harness.registry.snapshot.devices).toMatchObject([{ state: "shutdown" }]); - - // why: a device that ended a lease is never granted again, even to reboot it. With maxDevices:1 - // there is also no room to provision a second device, so the request is refused rather than - // handed the one still mid-destroy. - await expect( - harness.engine.request(request, { - noWait: true, - ownerId: "new-agent", - requesterId: "new-agent", - }), - ).rejects.toBeInstanceOf(NoCapacityError); - - clock.advance(34_000); - await flush(); - expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); - - // why: once the destroy finishes the slot is free again, and a new device is provisioned for it. - const granted = await harness.engine.request(request, { - ownerId: "new-agent", - requesterId: "new-agent", - }); - expect(granted.device.id).not.toBe(released.device.id); - }); - - it("recovers a background destroy interrupted by a daemon crash on the next start", async () => { - const state = await openTestState(); - let nextId = 1; - const idGenerator = { generate: () => `${nextId++}` }; - const systemStats = () => - new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }); - // why: the physical device survives a daemon restart even though the daemon's - // in-memory state does not -- represented here by one FakeDriver instance - // shared across both simulated processes below. - // why: the first destroy never returns, as one in a crashed daemon never does. - class CrashingDestroyDriver extends FakeDriver { - #destroys = 0; - override async destroy(device: DriverDevice): Promise { - this.#destroys += 1; - if (this.#destroys === 1) return new Promise(() => {}); - return super.destroy(device); - } - } - const driver = new CrashingDestroyDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - - const clock1 = new FakeClock(1_000); - const bus1 = new EventBus(clock1); - const registry1 = await loadTestRegistry({ - clock: clock1, - eventBus: bus1, - idGenerator, - state, - }); - const engine1 = createTestEngine({ - ...testComponentWiring({ - clock: clock1, - drivers: [driver], - eventBus: bus1, - registry: registry1, - }), - clock: clock1, - config: config(), - drivers: [driver], - eventBus: bus1, - idGenerator, - registry: registry1, - state: testStateOf(registry1), - systemStats: systemStats(), - }); - const granted = await engine1.request(request, { - ownerId: "previous-holder", - requesterId: "previous-holder", - }); - // why: the holder released, so the destroy behind it is running in the background... - await engine1.release(granted.lease.id, "explicit"); - - // why: ...convergence leaves it alone (it is claimed by this process)... - await engine1.convergeRunningCapacity(); - expect(registry1.snapshot.devices).toMatchObject([ - { id: granted.device.id, state: "shutdown" }, - ]); - // why: ...and the process is abandoned before the driver's 34s destroy resolves -- - // exactly like a crashed daemon. Nothing here ever settles that promise. - - // why: next start: a fresh process reads the same persisted registry (state.db - // survives the crash) and reconnects to the same physical driver, but its - // in-memory claim tracking starts empty -- there is no live claim for this - // device, because the process that held it is gone. - const clock2 = new FakeClock(clock1.now()); - const bus2 = new EventBus(clock2); - const registry2 = await loadTestRegistry({ - clock: clock2, - eventBus: bus2, - idGenerator, - state, - }); - const engine2 = createTestEngine({ - ...testComponentWiring({ - clock: clock2, - drivers: [driver], - eventBus: bus2, - registry: registry2, - }), - clock: clock2, - config: config(), - drivers: [driver], - eventBus: bus2, - idGenerator, - registry: registry2, - state: testStateOf(registry2), - systemStats: systemStats(), - }); - - await engine2.convergeRunningCapacity(); - await flush(); - - // why: recovered by destroying the spent device again, never by reclaiming it. - expect(registry2.snapshot.devices).toMatchObject([{ id: granted.device.id, state: "deleted" }]); - expect(driver.calls.filter((call) => call.operation === "reclaim")).toHaveLength(0); - expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); - }); - - it("leaves no armed timer behind for a lease it is disposed with", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 900_000 } }); - await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - expect(harness.clock.pendingTimerCount).toBeGreaterThan(0); - - harness.engine.dispose(); - - // why: A real `setTimeout` holds the event loop open, so a timer surviving disposal is a - // daemon that has decided to stop and cannot: `daemon stop` used to hang for the whole - // of an outstanding lease's TTL. Under ADR 0004 that is every lease, since a stop - // releases none of them. - expect(harness.clock.pendingTimerCount).toBe(0); - // why: The lease itself is untouched -- disposal cancels a timer, it does not expire a - // lease, which is what lets a lease survive the restart with its deadline intact. - expect(harness.registry.snapshot.leases).toHaveLength(1); - }); -}); - -describe("createLeasing fresh lease identity (#75)", () => { - const freshIos = { android: "reusable", ios: "fresh" } as const; - - function driverDeviceIdOf( - harness: { readonly registry: Registry }, - deviceId: string, - ): string | undefined { - return harness.registry.snapshot.devices.find((device) => device.id === deviceId) - ?.driverDeviceId; - } - - function destroyedDriverDeviceIds(driver: FakeDriver): string[] { - return driver.calls - .filter((call) => call.operation === "destroy") - .map((call) => (call.arguments[0] as { readonly deviceId: string }).deviceId); - } - - it("gives two sequential fresh leases for one shape different driver device ids", async () => { - const harness = await createHarness({ identity: freshIos }); - - const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - const second = await harness.engine.request(request, { - ownerId: "second", - requesterId: "second", - }); - - expect(second.device.id).not.toBe(first.device.id); - expect(driverDeviceIdOf(harness, second.device.id)).not.toBe( - driverDeviceIdOf(harness, first.device.id), - ); - expect(harness.driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("takes a released fresh device to deleted and destroys its driver device, without an erase", async () => { - const harness = await createHarness({ identity: freshIos }); - const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - const driverDeviceId = driverDeviceIdOf(harness, granted.device.id); - - await harness.engine.release(granted.lease.id, "explicit"); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: granted.device.id, state: "deleted" }, - ]); - expect(destroyedDriverDeviceIds(harness.driver)).toEqual([driverDeviceId]); - expect(harness.driver.calls.map((call) => call.operation)).not.toContain("reclaim"); - expect(harness.bus.replay().filter((event) => event.event === "device.deleted")).toMatchObject([ - { payload: { deviceId: granted.device.id, initiator: "lease-end" } }, - ]); - expect(harness.bus.replay().map((event) => event.event)).not.toContain("device.reclaimed"); - }); - - it("deletes the device of an expired fresh lease", async () => { - const harness = await createHarness({ identity: freshIos, lease: { defaultTtlMs: 10 } }); - const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - - harness.clock.advance(10); - await flush(); - await harness.engine.settle(); - - expect(harness.bus.replay().map((event) => event.event)).toContain("lease.expired"); - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: granted.device.id, state: "deleted" }, - ]); - expect(destroyedDriverDeviceIds(harness.driver)).toHaveLength(1); - }); - - it("the health monitor clears a device's recovery mark only once the decision section is free", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 14_400_000 } }); - const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - await harness.registry.markRecoveryAttempt(granted.device.id, harness.clock.now()); - let open!: () => void; - const held = harness.decisions.run( - () => - new Promise((resolve) => { - open = resolve; - }), - ); - - harness.engine.healthMonitor.start(); - harness.clock.advance(30_000); - await flush(); - expect(harness.registry.snapshot.devices[0]?.recoveringSince).toBeDefined(); - open(); - await held; - await flush(); - harness.engine.healthMonitor.dispose(); - - expect(harness.registry.snapshot.devices[0]?.recoveringSince).toBeUndefined(); - }); - - it("deletes the device of a recovery-driven lease termination", async () => { - const harness = await createHarness({ - identity: freshIos, - lease: { defaultTtlMs: 14_400_000 }, - }); - const granted = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - harness.driver.setManagedReality({ devices: [], processes: [] }); - - harness.engine.healthMonitor.start(); - for (let tick = 0; tick < 3; tick += 1) { - harness.clock.advance(30_000); - await flush(); - } - await harness.engine.settle(); - harness.engine.healthMonitor.dispose(); - - expect(harness.bus.replay().filter((event) => event.event === "lease.released")).toMatchObject([ - { payload: { leaseId: granted.lease.id, reason: "device-lost" } }, - ]); - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: granted.device.id, state: "deleted" }, - ]); - }); - - it("quarantines a fresh device whose delete fails, keeps it ungrantable, and reports strategy delete", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("destroy", 1, new DriverCrashError("delete exploded")); - const harness = await createHarness({ - driver, - identity: freshIos, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 3, - }, - }); - const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); - - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: first.device.id, state: "quarantined" }, - ]); - expect( - harness.bus.replay().filter((event) => event.event === "device.purge-failed"), - ).toMatchObject([ - { - payload: { - attemptedStrategy: "delete", - deviceId: first.device.id, - error: "DriverCrashError: delete exploded", - leaseId: first.lease.id, - }, - }, - ]); - const second = await harness.engine.request(request, { - noWait: true, - ownerId: "second", - requesterId: "second", - }); - expect(second.device.id).not.toBe(first.device.id); - }); - - it("still deletes a device created under fresh after the configuration changes to reusable", async () => { - const state = await openTestState(); - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - let nextId = 1; - const idGenerator = { generate: () => `${nextId++}` }; - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const systemStats = new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }); - const beforeRegistry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator, - leaseIdentity: freshIos, - state, - }); - const before = createTestEngine({ - clock, - ...testComponentWiring({ clock, drivers: [driver], eventBus: bus, registry: beforeRegistry }), - config: config({ identity: freshIos }), - drivers: [driver], - eventBus: bus, - idGenerator, - registry: beforeRegistry, - state: testStateOf(beforeRegistry), - systemStats, - }); - const granted = await before.request(request, { ownerId: "a", requesterId: "a" }); - before.dispose(); - - const reusable = { android: "reusable", ios: "reusable" } as const; - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator, - leaseIdentity: reusable, - state, - }); - const after = createTestEngine({ - ...testComponentWiring({ - clock: clock, - drivers: [driver], - eventBus: bus, - registry: registry, - }), - clock, - config: config({ identity: reusable, defaultTtlMs: 14_400_000 }), - drivers: [driver], - eventBus: bus, - idGenerator, - registry, - state: testStateOf(registry), - systemStats, - }); - await after.convergeRunningCapacity(); - await after.release(granted.lease.id, "explicit"); - await after.settle(); - after.dispose(); - - expect(registry.snapshot.devices).toMatchObject([ - { id: granted.device.id, leaseIdentity: "fresh", state: "deleted" }, - ]); - expect(driver.calls.map((call) => call.operation)).not.toContain("reclaim"); - }); - - it("destroys a device created under reusable too, and grants the next requester a new device", async () => { - const harness = await createHarness(); - const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); - - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - const second = await harness.engine.request(request, { - ownerId: "second", - requesterId: "second", - }); - - expect(harness.registry.snapshot.devices).toMatchObject([ - { id: first.device.id, leaseIdentity: "reusable", state: "deleted" }, - { id: second.device.id, leaseIdentity: "reusable", state: "leased" }, - ]); - expect(second.device.id).not.toBe(first.device.id); - const operations = harness.driver.calls.map((call) => call.operation); - expect(operations).not.toContain("reclaim"); - expect(operations.filter((operation) => operation === "destroy")).toHaveLength(1); - }); - - async function restartAfterCrash(crashedIn: "reclaiming" | "shutdown") { - const state = await openTestState(); - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - let nextId = 1; - const idGenerator = { generate: () => `${nextId++}` }; - // why: One driver across both processes: the simulator outlives the daemon. - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const crashed = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator, - leaseIdentity: freshIos, - state, - }); - const driverDevice = await driver.makeReady( - await driver.provision({ ...request, physical: false }), - { - mode: "full", - purpose: "prepare", - }, - ); - const device = await crashed.registerDevice({ - driverData: driverDevice.driverData, - driverDeviceId: driverDevice.deviceId, - provisionDuration: 0, - spec: { ...request, physical: false }, - }); - await crashed.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await crashed.createLease({ - deviceId: device.id, - ownerId: "gone", - requesterId: "gone", - ttlDeadline: 60_000, - ttlMs: 60_000, - }); - await crashed.beginRelease(lease.id); - if (crashedIn === "shutdown") { - await driver.shutdown(driverDevice); - await crashed.completeReclaimWithoutPurge(device.id); - } - - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator, - leaseIdentity: freshIos, - state, - }); - const engine = createTestEngine({ - ...testComponentWiring({ - clock: clock, - drivers: [driver], - eventBus: bus, - registry: registry, - }), - clock, - config: config({ identity: freshIos }), - drivers: [driver], - eventBus: bus, - idGenerator, - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - }); - const operationsBeforeStart = driver.calls.length; - return { - device, - driver, - engine, - operationsSinceStart: () => - driver.calls.slice(operationsBeforeStart).map((call) => call.operation), - registry, - }; - } - - it.each([ - { - crashedIn: "reclaiming" as const, - operations: ["listManaged", "shutdown", "destroy", "listSources"], - title: "while it was reclaiming", - }, - { - crashedIn: "shutdown" as const, - operations: ["listManaged", "destroy", "listSources"], - title: "after its shutdown commit and before its delete", - }, - ])( - "deletes a fresh device on the next start when the daemon crashed $title", - async ({ crashedIn, operations }) => { - const restarted = await restartAfterCrash(crashedIn); - expect(restarted.registry.snapshot.devices).toMatchObject([{ state: crashedIn }]); - - await restarted.engine.convergeRunningCapacity(); - - expect(restarted.registry.snapshot.devices).toMatchObject([ - { id: restarted.device.id, state: "deleted" }, - ]); - expect(restarted.operationsSinceStart()).toEqual(operations); - }, - ); -}); - -describe("createLeasing logger wiring", () => { - function debugLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { - const sink = new MemoryLogSink(); - return { logger: new JsonLinesLogger({ clock: new FakeClock(0), level: "debug", sink }), sink }; - } - - it("createCore hands its logger to the quarantine coordinator: a failed retry is logged", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - driver.failOn("destroy", 2, new DriverCrashError("retry exploded")); - const { logger, sink } = debugLogger(); - const harness = await createHarness({ driver, logger }); - const first = await harness.engine.request(request, { - ownerId: "first", - requesterId: "first", - }); - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); - - harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); - await flush(); - - expect( - sink.records.filter((record) => record.message === "quarantine delete retry failed"), - ).toMatchObject([ - { - level: "warn", - module: "daemon.quarantine-coordinator", - fields: { - attempts: 1, - deviceId: first.device.id, - error: "DriverCrashError: retry exploded", - step: "delete", - }, - }, - ]); - }); - - it("createLeasing hands its logger to the acquisition coordinator: a failed eviction is logged", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("shutdown", 1, new DriverCrashError("cannot stop victim")); - const { logger, sink } = debugLogger(); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 1 }, - maxRunning: 1, - }, - logger, - }); - const victim = await seedReady(harness); - - await expect( - harness.engine.request( - { ...request, model: "iPhone SE" }, - { noWait: true, ownerId: "new-spec", requesterId: "new-spec" }, - ), - ).rejects.toBeInstanceOf(NoCapacityError); - - expect( - sink.records.filter((record) => record.message === "shutting down an eviction target failed"), - ).toMatchObject([ - { - level: "warn", - module: "daemon.lease-acquisition-coordinator", - fields: { deviceId: victim.id, requesterId: "new-spec", step: "shutdown" }, - }, - ]); - }); - - it("createLeasing hands its logger to the lease release coordinator: a background reclaim that cannot commit is logged", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { reclaim: 20 }, - platform: "ios", - }); - const { logger, sink } = debugLogger(); - const harness = await createHarness({ driver, logger }); - const first = await harness.engine.request(request, { - ownerId: "first", - requesterId: "first", - }); - await harness.engine.release(first.lease.id, "explicit"); - // why: The release has committed and the erase is still running. A disk that refuses the write - // makes the reclaim's own commit throw, with no caller left to reject to. - vi.spyOn(harness.state, "transaction").mockImplementation(() => { - throw new Error("EIO"); - }); - - clock.advance(20); - await harness.engine.settle(); - - expect( - sink.records.filter((record) => record.message === "background reclaim failed"), - ).toMatchObject([ - { - level: "error", - module: "daemon.lease-release-coordinator", - fields: { deviceId: first.device.id, leaseId: first.lease.id }, - }, - ]); - }); -}); - -describe("createLeasing restart recovery of stored lease requests", () => { - async function restartWithAgentQueued(idempotencyKey?: string) { - const before = await createHarness(); - await before.engine.request(request, { ownerId: "holder", requesterId: "holder" }); - void before.engine - .request(request, { - ownerId: "agent", - requesterId: "agent", - ...(idempotencyKey === undefined ? {} : { idempotencyKey }), - }) - .catch(() => undefined); - await expect - .poll(() => before.registry.leaseRequests().find((record) => record.requesterId === "agent")) - .toMatchObject({ state: "open" }); - - const after = await createHarness({ state: before.state }); - await after.engine.convergeRunningCapacity(); - return after; - } - - it.each([{ idempotencyKey: "key-1" as string | undefined }, { idempotencyKey: undefined }])( - "settles a request still open at a restart as failed and says the daemon restarted, whether it carried an idempotency key or not ($idempotencyKey)", - async ({ idempotencyKey }) => { - const after = await restartWithAgentQueued(idempotencyKey); - - expect( - after.registry.leaseRequests().find((record) => record.requesterId === "agent"), - ).toMatchObject({ - failure: { code: "INTERNAL", message: expect.stringContaining("daemon restarted") }, - state: "failed", - }); - expect( - after.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload), - ).toEqual([ - { - reason: "daemon-restarted", - requestId: expect.stringMatching(/^req_/), - requester: "agent", - requestSpec: request, - }, - ]); - if (idempotencyKey !== undefined) { - await expect( - after.engine.request(request, { - idempotencyKey, - ownerId: "agent", - requesterId: "agent", - }), - ).rejects.toMatchObject({ message: expect.stringContaining("daemon restarted") }); - } - }, - ); -}); - -describe("createLeasing a grant and its request's result", () => { - const asker = { idempotencyKey: "key-1", ownerId: "agent", requesterId: "agent" }; - - it("answers a repeat under the same key with the lease after the daemon stopped between the grant and the request book's settle write", async () => { - const before = await createHarness(); - await seedReady(before); - vi.spyOn(before.registry, "settleLeaseRequest").mockRejectedValue(new Error("disk gone")); - const grant = await before.engine.request(request, asker); - await flush(); - - const after = await createHarness({ driver: before.driver, state: before.state }); - await after.engine.convergeRunningCapacity(); - const repeat = await after.engine.request(request, asker); - - expect(repeat).toEqual(grant); - expect(repeat.lease.id).toBe(grant.lease.id); - expect(after.registry.leaseRequests()).toMatchObject([{ state: "granted" }]); - expect(after.registry.snapshot.leases).toHaveLength(1); - }); - - it("answers a repeat under the same key in the same process with the lease", async () => { - const harness = await createHarness(); - await seedReady(harness); - const grant = await harness.engine.request(request, asker); - await flush(); - - const repeat = await harness.engine.request(request, asker); - expect(repeat).toEqual(grant); - expect(repeat.lease.id).toBe(grant.lease.id); - expect(harness.registry.leaseRequests()).toMatchObject([{ grant, state: "granted" }]); - }); - - it("answers a repeat of a granted request whose lease was released since with the lease as recorded and the device as it is now, and a renew of that lease with UnknownLeaseError", async () => { - const harness = await createHarness(); - await seedReady(harness); - const grant = await harness.engine.request(request, asker); - await flush(); - await harness.engine.release(grant.lease.id, "explicit"); - await harness.engine.settle(); - - const repeat = await harness.engine.request(request, asker); - - expect(repeat.lease.id).toBe(grant.lease.id); - expect(grant.device.state).toBe("leased"); - expect(repeat).toEqual({ ...grant, device: harness.registry.snapshot.devices[0] }); - expect(repeat.device.state).toBe("deleted"); - await expect(harness.engine.renew(grant.lease.id)).rejects.toBeInstanceOf(UnknownLeaseError); - }); -}); - -describe("createLeasing class requests", () => { - it("creates the first model on the class's preference list the catalog lists, for a request naming a class", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - knownModels: ["iPhone 15", "iPhone 16"], - modelClasses: { "iPhone 15": "phone", "iPhone 16": "phone" }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 16", "iPhone 15"] } }, - }); - - const granted = await harness.engine.request( - { class: "phone", platform: "ios" }, - { ownerId: "agent", requesterId: "agent" }, - ); - - expect(granted.device.spec.model).toBe("iPhone 16"); - }); - - it("creates the first listed model that pairs with the requested OS version, not one that pairs with another", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["18.4", "26.5"], - clock: new FakeClock(1_000), - knownModels: ["iPhone 16", "iPhone 17"], - modelClasses: { "iPhone 16": "phone", "iPhone 17": "phone" }, - modelRuntimes: { "iPhone 16": ["18.4", "26.5"], "iPhone 17": ["26.5"] }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 16"] } }, - }); - - const granted = await harness.engine.request( - { class: "phone", osVersion: "18.4", platform: "ios" }, - { ownerId: "agent", requesterId: "agent" }, - ); - - expect(granted.device.spec).toMatchObject({ model: "iPhone 16", osVersion: "18.4" }); - }); - - it("refuses a class request naming an OS that is not installed as RUNTIME_MISSING without calling the installer", async () => { - const asked: unknown[] = []; - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - knownModels: ["iPhone 17"], - modelClasses: { "iPhone 17": "phone" }, - platform: "ios", - }); - const harness = await createHarness({ - components: { - claimProvision: () => () => undefined, - install: async (call) => { - asked.push(call); - return { outcome: "installed", version: "18.4" }; - }, - }, - driver, - modelPreferences: { ios: { phone: ["iPhone 17"] } }, - }); - - await expect( - harness.engine.request( - { class: "phone", osVersion: "18.4", platform: "ios" }, - { allowDownload: true, ownerId: "agent", requesterId: "agent" }, - ), - ).rejects.toMatchObject({ downloadable: false, name: "RuntimeMissingError" }); - expect(asked).toEqual([]); - }); -}); -describe("createLeasing: capacity.changed and queue.changed", () => { - type CapacityEvent = EventMap["capacity.changed"]; - - function capacityEvents(harness: Awaited>): CapacityEvent[] { - return harness.bus - .replay() - .filter((event) => event.event === "capacity.changed") - .map((event) => event.payload as CapacityEvent); - } - - function queueEvents(harness: Awaited>): number[] { - return harness.bus - .replay() - .filter((event) => event.event === "queue.changed") - .map((event) => (event.payload as EventMap["queue.changed"]).depth); - } - - function currentFigures(harness: Awaited>): CapacityEvent { - return capacityChangedPayload( - buildCapacityFigures(harness.registry.snapshot.virtualDevices, harness.engine), - ); - } - - it("emits capacity.changed and queue.changed once when the daemon has started, and none before", async () => { - const harness = await createHarness(); - await seedReady(harness); - expect([capacityEvents(harness), queueEvents(harness)]).toEqual([[], []]); - - await harness.engine.convergeRunningCapacity(); - - expect(capacityEvents(harness)).toEqual([currentFigures(harness)]); - expect(capacityEvents(harness)[0]).toMatchObject({ global: { warm: 1 }, ios: { warm: 1 } }); - expect(queueEvents(harness)).toEqual([0]); - }); - - it("emits capacity.changed after a provision commits, with the figures status.get reports at that moment", async () => { - const harness = await createHarness(); - await harness.engine.convergeRunningCapacity(); - const before = capacityEvents(harness).length; - const seen: Array<{ event: CapacityEvent; figures: CapacityEvent }> = []; - harness.bus.subscribe("capacity.changed", (envelope) => - seen.push({ event: envelope.payload, figures: currentFigures(harness) }), - ); - - await harness.registry.registerDevice({ - driverData: { fakeDeviceId: "sim-1" }, - driverDeviceId: "sim-1", - provisionDuration: 0, - spec: { ...request, physical: false }, - }); - - expect(capacityEvents(harness)).toHaveLength(before + 1); - expect(seen).toEqual([{ event: currentFigures(harness), figures: currentFigures(harness) }]); - expect(seen[0]?.event.ramBudget?.usedBytes).toBeGreaterThan(0); - }); - - it("emits capacity.changed when a provisioning reservation is taken, before any registry commit, and when it is released", async () => { - const harness = await createHarness(); - await harness.engine.convergeRunningCapacity(); - const seen: Array<{ reserved: number; devices: number }> = []; - harness.bus.subscribe("capacity.changed", (envelope) => - seen.push({ - devices: harness.registry.snapshot.devices.length, - reserved: envelope.payload.ios.reserved, - }), - ); - harness.driver.hangMakeReady(); - - const pending = harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - await flush(); - expect(seen[0]).toEqual({ devices: 0, reserved: 1 }); - harness.driver.releaseMakeReady(); - await pending; - await flush(); - - expect(seen.at(-1)?.reserved).toBe(0); - expect(capacityEvents(harness).at(-1)).toEqual(currentFigures(harness)); - }); - - it("emits capacity.changed after a device is deleted and after a lease release destroys its device", async () => { - const harness = await createHarness(); - const grant = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - await harness.engine.convergeRunningCapacity(); - expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 1 } }); - - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 0 } }); - - const warm = await seedReady(harness); - expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 1, running: 1 } }); - - await harness.registry.markDeviceMissing(warm.id, "test"); - expect(capacityEvents(harness).at(-1)).toMatchObject({ ios: { warm: 0, running: 0 } }); - expect(capacityEvents(harness).at(-1)).toEqual(currentFigures(harness)); - }); - - it("emits no capacity.changed when a commit leaves the capacity figures unchanged", async () => { - const harness = await createHarness(); - await seedReady(harness); - await harness.engine.convergeRunningCapacity(); - const before = capacityEvents(harness).length; - - await harness.registry.createLeaseRequest({ - ownerId: "a", - request, - requesterId: "a", - }); - await harness.registry.markForeignStateDetected(harness.registry.snapshot.devices[0]!.id, 1); - - expect(capacityEvents(harness)).toHaveLength(before); - }); - - it("never emits two consecutive capacity.changed events with equal figures across a whole lease", async () => { - const harness = await createHarness(); - await harness.engine.convergeRunningCapacity(); - const grant = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - - const events = capacityEvents(harness); - expect(events.length).toBeGreaterThan(3); - events.slice(1).forEach((event, index) => expect(event).not.toEqual(events[index])); - }); - - it("carries ramBudget under the resource strategy and omits it under the fixed strategy", async () => { - const resource = await createHarness(); - await resource.engine.convergeRunningCapacity(); - const fixed = await createHarness({ - capacity: { strategy: "fixed", config: { maxRunning: 2 } }, - }); - await fixed.engine.convergeRunningCapacity(); - - expect(capacityEvents(resource)[0]?.ramBudget).toEqual({ - limitBytes: expect.any(Number), - usedBytes: 0, - }); - expect(capacityEvents(fixed)).toHaveLength(1); - expect(capacityEvents(fixed)[0]).not.toHaveProperty("ramBudget"); - }); -}); - -describe("createLeasing: lease.rejected names its request", () => { - type Reason = Exclude; - type Rejection = EventMap["lease.rejected"]; - type Harness = Awaited>; - - interface Outcome { - readonly harness: Harness; - readonly requester: string; - readonly requestId?: string | undefined; - } - - const holder = { ownerId: "holder", requesterId: "holder" }; - - async function admittedId(harness: Harness, requesterId: string): Promise { - return harness.registry.leaseRequests().find((record) => record.requesterId === requesterId) - ?.id; - } - - /** why: One case per reason in `EventMap`, so a reason added there without a case fails typecheck. */ - const cases: Record Promise) | "gateway"> = { - timeout: async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - const timedOut = harness.engine.request(request, { - ownerId: "late", - requesterId: "late", - timeoutMs: 10, - }); - await flush(); - harness.clock.advance(10); - await settledOrPending(timedOut); - return { harness, requestId: await admittedId(harness, "late"), requester: "late" }; - }, - "no-wait": async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - await settledOrPending( - harness.engine.request(request, { noWait: true, ownerId: "now", requesterId: "now" }), - ); - return { harness, requestId: await admittedId(harness, "now"), requester: "now" }; - }, - "unresolvable-spec": async () => { - const harness = await createHarness(); - await settledOrPending( - harness.engine.request( - { ...request, osVersion: "1.0" }, - { ownerId: "odd", requesterId: "odd" }, - ), - ); - return { harness, requestId: await admittedId(harness, "odd"), requester: "odd" }; - }, - "no-worker": "gateway", - "worker-failed": "gateway", - "already-leased": async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - await settledOrPending(harness.engine.request(request, holder)); - return { harness, requester: "holder" }; - }, - "lease-id-taken": async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 2, maxRunning: 2 }, - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 4, - }, - }); - await harness.engine.request(request, { ...holder, leaseId: "myid" }); - await settledOrPending( - harness.engine.request(request, { - leaseId: "myid", - ownerId: "clash", - requesterId: "clash", - }), - ); - return { harness, requester: "clash" }; - }, - "lease-ended": async () => { - const harness = await createHarness(); - await harness.engine.request(request, { ...holder, leaseId: "myid" }); - await harness.engine.release("myid", "explicit"); - await settledOrPending( - harness.engine.request(request, { - leaseId: "myid", - ownerId: "again", - requesterId: "again", - }), - ); - return { harness, requester: "again" }; - }, - "boot-timeout": async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("makeReady", 1, new Error("boot failed")); - const harness = await createHarness({ driver }); - await settledOrPending( - harness.engine.request(request, { ownerId: "slow", requesterId: "slow" }), - ); - return { harness, requestId: await admittedId(harness, "slow"), requester: "slow" }; - }, - killed: async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - const queued = harness.engine.request(request, { ownerId: "next", requesterId: "next" }); - void queued.catch(() => undefined); - await flush(); - await harness.engine.nuke(false); - return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; - }, - updating: async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - void harness.engine - .request(request, { ownerId: "next", requesterId: "next" }) - .catch(() => undefined); - await flush(); - await harness.engine.leasing.holdForUpdate(); - return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; - }, - cancelled: async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - void harness.engine - .request(request, { ownerId: "next", requesterId: "next" }) - .catch(() => undefined); - await flush(); - await harness.engine.cancelPending("next"); - return { harness, requestId: await admittedId(harness, "next"), requester: "next" }; - }, - "daemon-restarted": async () => { - const before = await createHarness(); - await before.engine.request(request, holder); - void before.engine - .request(request, { ownerId: "agent", requesterId: "agent" }) - .catch(() => undefined); - await expect.poll(() => admittedId(before, "agent")).toEqual(expect.stringMatching(/^req_/)); - const requestId = await admittedId(before, "agent"); - const harness = await createHarness({ state: before.state }); - await harness.engine.convergeRunningCapacity(); - return { harness, requestId, requester: "agent" }; - }, - }; - - const workerReasons = (Object.keys(cases) as Reason[]).filter( - (reason) => cases[reason] !== "gateway", - ); - - it.each(workerReasons)( - "includes the request id and the requester when it rejects with %s", - async (reason) => { - const run = cases[reason]; - if (run === "gateway") throw new Error(`${reason} is the gateway's`); - - const { harness, requestId, requester } = await run(); - - const rejections = harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload as Rejection) - .filter((payload) => payload.reason === reason); - expect(rejections).toHaveLength(1); - expect(rejections[0]).toMatchObject({ - requestId: requestId ?? expect.stringMatching(/^req_/), - requester, - }); - }, - ); -}); - -describe("createLeasing queue timeout and failure storage", () => { - const android = { - physical: false, - model: "Pixel 9", - osVersion: "35", - platform: "android", - } as const; - - it("emits lease.rejected for a timed-out waiter under the wait queue's module, then lets the request behind it through", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const droid = new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }); - const harness = await createHarness({ driver: ios, drivers: [ios, droid] }); - await harness.engine.request(request, { ownerId: "holder", requesterId: "holder" }); - const blocking = harness.engine.request(request, { - ownerId: "head", - requesterId: "head", - timeoutMs: 10, - }); - const behind = harness.engine.request(android, { ownerId: "behind", requesterId: "behind" }); - await flush(); - // why: The android request could be served now, but it waits behind the iOS one at the head. - expect(await settledOrPending(behind)).toBe("still pending"); - - harness.clock.advance(10); - - expect(await settledOrPending(blocking)).toBeInstanceOf(QueueTimeoutError); - expect(await settledOrPending(behind)).toMatchObject({ lease: { requesterId: "behind" } }); - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => ({ - module: event.module, - reason: (event.payload as { reason: string }).reason, - })), - ).toEqual([{ module: "wait-queue", reason: "timeout" }]); - }); - - it("stores a failed request as INTERNAL with the error's own message when no classifier was given", async () => { - const harness = await createHarness(); - - await expect( - harness.engine.request(android, { ownerId: "agent", requesterId: "agent" }), - ).rejects.toThrow("No driver registered for platform: android"); - - expect(harness.registry.leaseRequests()).toMatchObject([ - { - failure: { code: "INTERNAL", message: "No driver registered for platform: android" }, - state: "failed", - }, - ]); - }); -}); - -describe("createLeasing wiring", () => { - const components = { - claimProvision: () => () => undefined, - install: async () => ({ outcome: "installed" as const, version: "26.5" }), - }; - - it("wires the leased-device health monitor, and leaves it out when told to", async () => { - const harness = await createHarness(); - const without = createLeasing({ - clock: harness.clock, - components, - config: config(), - core: harness.engine.core, - eventBus: harness.bus, - healthMonitor: false, - idGenerator: { generate: () => "1" }, - }); - - expect(harness.engine.leasing.healthMonitor).toBeInstanceOf(LeaseHealthMonitor); - expect(without.healthMonitor).toBeUndefined(); - }); - - it("hands core a leaseExpirer that expires through leasing: doctor --fix ends a lease past its deadline", async () => { - const harness = await createHarness(); - const grant = await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - // why: No timer may expire it first: only doctor's finding does. - harness.engine.dispose(); - harness.clock.advance(10 * 60_000 * 60); - - await harness.engine.core.doctor.reconcile({ fix: true }); - - expect(harness.registry.snapshot.leases.map((lease) => lease.id)).not.toContain(grant.lease.id); - expect(harness.bus.replay().map((event) => event.event)).toContain("lease.expired"); - }); - - it("disposes the physical device watcher with leasing: no presence read happens after dispose", async () => { - const harness = await createHarness(); - await harness.registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: "UDID-A", - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - harness.driver.setPhysicalPresence("UDID-A", true); - harness.engine.leasing.physicalDeviceWatcher.start(); - harness.engine.dispose(); - - harness.clock.advance(120_000); - await harness.engine.settle(); - - expect(harness.driver.calls.filter((call) => call.operation === "listPresentPhysical")).toEqual( - [], - ); - }); - - it("does not connect core itself when a second leasing is built", async () => { - const harness = await createHarness(); - const connect = vi.spyOn(harness.engine.core, "connect"); - - createLeasing({ - clock: harness.clock, - components, - config: config(), - core: harness.engine.core, - eventBus: harness.bus, - idGenerator: { generate: () => "1" }, - }); - - expect(connect).not.toHaveBeenCalled(); - }); -}); - -describe("createLeasing warm pool", () => { - const androidSpec = { - physical: false, - model: "Pixel 9", - osVersion: "36", - platform: "android", - } as const; - - it("shuts down one of three ready devices with initiator warm-pool after daemon.started when maxRunning is 2", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 2 }, - maxRunning: 2, - }, - }); - await seedReady(harness); - await seedReady(harness); - await seedReady(harness); - - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await harness.engine.settle(); - - const states = harness.registry.snapshot.devices.map((item) => item.state); - expect(states.filter((state) => state === "ready")).toHaveLength(2); - expect(states.filter((state) => state === "shutdown")).toHaveLength(1); - const shutdowns = harness.bus.replay().filter((event) => event.event === "device.shutdown"); - expect(shutdowns).toHaveLength(1); - expect(shutdowns[0]).toMatchObject({ payload: { initiator: "warm-pool" } }); - }); - - it("logs a warm pool shutdown that fails under the warm pool's own module and leaves the device ready", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("shutdown", 1, new DriverCrashError("cannot stop it")); - const sink = new MemoryLogSink(); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 1 }, - maxRunning: 1, - }, - logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), - }); - await seedReady(harness); - await seedReady(harness); - - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await harness.engine.settle(); - - expect(sink.records.filter((record) => record.level === "warn")).toMatchObject([ - { fields: { step: "shutdown" }, module: expect.stringContaining("warm-pool") }, - ]); - expect(harness.registry.snapshot.devices.map((item) => item.state)).toEqual(["ready", "ready"]); - }); - - it("a nuke that deletes devices is not undone by the warm pool booting a released device back", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const harness = await createHarness({ driver }); - await harness.engine.request(request, { ownerId: "held", requesterId: "held" }); - - await harness.engine.nuke(true); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); - }); - - it("triggers no pool pass from a fact before daemon.started, and ends with one of two ready devices shut down at one running slot", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 1 }, - maxRunning: 1, - }, - }); - await seedReady(harness); - await seedReady(harness); - - harness.bus.emit( - "device.deleted", - { deviceId: "x", initiator: "test", physical: false }, - "test", - ); - await harness.engine.settle(); - expect(harness.registry.snapshot.devices.map((item) => item.state)).toEqual(["ready", "ready"]); - - await harness.engine.convergeRunningCapacity(); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices.map((item) => item.state).sort()).toEqual([ - "ready", - "shutdown", - ]); - }); - - it("cancels the pool's tick when the engine is disposed", async () => { - const harness = await createHarness(); - await harness.engine.convergeRunningCapacity(); - expect(harness.clock.pendingTimerCount).toBe(1); - - harness.engine.dispose(); - - expect(harness.clock.pendingTimerCount).toBe(0); - }); - - it("a nuke without --delete-devices leaves the released devices destroyed, not booted back by the pool", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const harness = await createHarness({ driver }); - await harness.engine.convergeRunningCapacity(); - const held = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - - await harness.engine.nuke(false); - await harness.engine.settle(); - // why: Still inside idle.shutdownAfterMs of the forced release, and a fact that triggers a pass. - harness.bus.emit( - "cleanup.executed", - { action: "shutdown", reason: "test", ruleName: "test", target: held.device.id }, - "test", - ); - await harness.engine.settle(); - - expect( - harness.registry.snapshot.devices.find((item) => item.id === held.device.id)?.state, - ).toBe("deleted"); - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1); - }); - - it("accepts a new request again once a nuke has finished", async () => { - const harness = await createHarness(); - await harness.engine.request(request, { ownerId: "held", requesterId: "held" }); - - await harness.engine.nuke(false); - - await expect( - harness.engine.request(request, { ownerId: "after", requesterId: "after" }), - ).resolves.toMatchObject({ lease: { requesterId: "after" } }); - }); - - /** - * why: a released virtual device is destroyed, never booted back, so the device the warm pool - * boots is a shut-down one that never ended a lease, under a warm target of one. - * Its boot is held on the clock, with room for a second device. - */ - async function shutdownDeviceBooting(failBoot = false) { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - knownModels: ["iPhone 16"], - availableOsVersions: ["26.5"], - clock, - latencyMs: { makeReady: 50 }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 2, - }, - warmPool: { - targets: [{ count: 1, model: "iPhone 16", osVersion: "26.5", platform: "ios" }], - }, - }); - const seeding = seedReady(harness); - await flush(); - clock.advance(50); - const seeded = await seeding; - await harness.registry.transitionDevice(seeded.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: seeded.id, initiator: "test" }, - }); - if (failBoot) driver.failOn("makeReady", 2, new Error("boom")); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await vi.waitFor(() => - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), - ); - return { clock, driver, seeded, harness }; - } - - it("a nuke waits for a warm pool boot already in flight, then deletes the device", async () => { - const { clock, driver, harness } = await shutdownDeviceBooting(); - - const nuking = harness.engine.nuke(true); - await flush(); - clock.advance(50); - await nuking; - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices).toMatchObject([{ state: "deleted" }]); - expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); - }); - - describe("a request for a device on its way", () => { - it("a request arriving during a warm-pool boot that serves it is granted that device when the boot settles, without a device provisioned for it", async () => { - const { clock, driver, seeded, harness } = await shutdownDeviceBooting(); - - const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); - await flush(); - expect(await settledOrPending(second)).toBe("still pending"); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - - clock.advance(50); - await flush(); - // why: The pool's kick after its boot woke the request: it is granted without another clock tick. - expect(await settledOrPending(second)).not.toBe("still pending"); - const granted = await second; - - // why: the granted device leaves the warm target short, so the pool provisions a - // replacement of its own; none was provisioned for the request, which got the booting one. - expect(granted.device.id).toBe(seeded.id); - expect(harness.bus.replay().filter((event) => event.event === "device.ready")).toHaveLength( - 2, - ); - }); - - async function twoRoomsHarness(warmPoolEnabled: boolean) { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { makeReady: 50 }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 3, - }, - warmPoolEnabled, - }); - return { clock, driver, harness }; - } - - it("a request arriving while another request is creating its own device provisions its own device rather than waiting for that one", async () => { - const { clock, driver, harness } = await twoRoomsHarness(true); - - const first = harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - await vi.waitFor(() => - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(1), - ); - const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); - await flush(); - // why: Before either boot ends: the second request did not wait for the first one's device. - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - clock.advance(50); - await flush(); - clock.advance(50); - const granted = await Promise.all([first, second]); - - expect(granted[0].device.id).not.toBe(granted[1].device.id); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("a request arriving while another request is booting a shut-down device provisions its own device rather than waiting for that one", async () => { - const { clock, driver, harness } = await twoRoomsHarness(false); - // why: a released virtual device is destroyed, so the shut-down device is seeded. - const seeding = seedReady(harness); - await flush(); - clock.advance(50); - const held = await seeding; - await harness.registry.transitionDevice(held.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: held.id, initiator: "test" }, - }); - - const booting = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); - await vi.waitFor(() => - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), - ); - const other = harness.engine.request(request, { ownerId: "c", requesterId: "c" }); - await flush(); - // why: Before either boot ends: the other request did not wait for the one booting. - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - clock.advance(50); - await flush(); - clock.advance(50); - const granted = await Promise.all([booting, other]); - - expect(granted[0].device.id).toBe(held.id); - expect(granted[1].device.id).not.toBe(held.id); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("under noWait a request arriving during a warm-pool boot provisions its own device", async () => { - const { clock, driver, seeded, harness } = await shutdownDeviceBooting(); - - const second = harness.engine.request(request, { - noWait: true, - ownerId: "b", - requesterId: "b", - }); - await flush(); - clock.advance(50); - const granted = await second; - - expect(granted.device.id).not.toBe(seeded.id); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("a request waiting on a warm-pool boot that fails re-plans on the kick and boots the device the failed boot left", async () => { - const { clock, driver, seeded, harness } = await shutdownDeviceBooting(true); - - const second = harness.engine.request(request, { ownerId: "b", requesterId: "b" }); - await flush(); - clock.advance(50); - await flush(); - // why: The kick from the failed boot woke the request: it has started booting the device itself. - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(3); - clock.advance(50); - const granted = await second; - await harness.engine.settle(); - - // why: The failed boot left the device `shutdown` and unclaimed, so the request boots it itself. - expect(granted.device.id).toBe(seeded.id); - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(3); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - }); - }); - - it("a release at the running cap with a class request waiting provisions a new device for it, the released one being destroyed", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.0"], - clock, - knownModels: ["iPhone 15", "iPhone 17"], - modelClasses: { "iPhone 15": "phone", "iPhone 17": "phone" }, - platform: "ios", - }); - const harness = await createHarness({ - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 1 }, - maxRunning: 1, - }, - modelPreferences: { ios: { phone: ["iPhone 17", "iPhone 15"] } }, - }); - const iphone15 = await seedReady(harness, { - physical: false, - model: "iPhone 15", - osVersion: "26.0", - platform: "ios", - }); - const holder = await harness.engine.request( - { model: "iPhone 15", osVersion: "26.0", platform: "ios" }, - { ownerId: "holder", requesterId: "holder" }, - ); - expect(holder.device.id).toBe(iphone15.id); - const waiter = harness.engine.request( - { class: "phone", platform: "ios" }, - { ownerId: "waiter", requesterId: "waiter" }, - ); - await flush(); - expect(harness.engine.queueDepth).toBe(1); - - await harness.engine.release(holder.lease.id, "explicit"); - const granted = await waiter; - await harness.engine.settle(); - - expect(granted.device.id).not.toBe(iphone15.id); - expect(granted.device.spec.model).toBe("iPhone 17"); - expect(harness.registry.snapshot.devices.find((item) => item.id === iphone15.id)?.state).toBe( - "deleted", - ); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("destroys a released iOS device instead of booting it back when the running limit has room", async () => { - const driver = new FakeDriver({ - knownModels: ["iPhone 16"], - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const harness = await createHarness({ driver, startPool: true }); - const first = await harness.engine.request(request, { ownerId: "a", requesterId: "a" }); - - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - - expect( - harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, - ).toBe("deleted"); - const releasedDriverId = harness.registry.snapshot.devices.find( - (item) => item.id === first.device.id, - )?.driverDeviceId; - expect( - driver.calls.filter( - (call) => - call.operation === "makeReady" && - (call.arguments[0] as { readonly deviceId: string }).deviceId === releasedDriverId, - ), - ).toHaveLength(1); - }); - - it("with warmPool disabled a released fake iOS device and a released fake Android device are both deleted, with no reclaim", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - limits: { - android: { maxDevices: 2, maxRunning: 2 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 4, - }, - warmPoolEnabled: false, - }); - const iosLease = await harness.engine.request(request, { ownerId: "i", requesterId: "i" }); - const androidLease = await harness.engine.request(androidSpec, { - ownerId: "a", - requesterId: "a", - }); - - await harness.engine.release(iosLease.lease.id, "explicit"); - await harness.engine.release(androidLease.lease.id, "explicit"); - await harness.engine.settle(); - - const state = (id: string) => - harness.registry.snapshot.devices.find((item) => item.id === id)?.state; - expect(state(iosLease.device.id)).toBe("deleted"); - expect(state(androidLease.device.id)).toBe("deleted"); - const events = harness.bus.replay(); - expect(events.filter((event) => event.event === "device.reclaimed")).toEqual([]); - expect( - events.filter((event) => event.event === "device.deleted").map((event) => event.payload), - ).toMatchObject([ - { deviceId: iosLease.device.id, initiator: "lease-end" }, - { deviceId: androidLease.device.id, initiator: "lease-end" }, - ]); - }); -}); - -describe("createLeasing queued device cleanup", () => { - it("keeps an old shut-down device ready across reaper passes while its requester waits behind the queue head", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - knownModels: ["iPhone 16"], - platform: "ios", - }); - const android = new FakeDriver({ - availableOsVersions: ["35"], - clock: new FakeClock(1_000), - platform: "android", - }); - const harness = await createHarness({ - driver, - drivers: [driver, android], - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 3, - }, - }); - const other = { model: "Pixel 9", osVersion: "35", platform: "android" } as const; - await harness.engine.request(request, { - ownerId: "holder", - requesterId: "holder", - ttlMs: 3_600_000, - }); - // why: a released virtual device is destroyed, so the old shut-down device is seeded. - const previous = await seedReady(harness, { ...other, physical: false }, "full", android); - await harness.registry.transitionDevice(previous.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: previous.id, initiator: "test" }, - }); - harness.clock.advance(10_001); - await harness.engine.convergeRunningCapacity(); - const pending = ["head", "behind"].map((requesterId, index) => - harness.engine - .request(index === 0 ? request : other, { - ownerId: requesterId, - requesterId, - }) - .catch((error: unknown) => error), - ); - await flush(); - await harness.engine.core.passWarmPool(); - await harness.engine.settle(); - const state = () => - harness.registry.snapshot.devices.find((device) => device.id === previous.id)?.state; - expect(harness.engine.waitingRequests().map((entry) => entry.requesterId)).toEqual([ - "head", - "behind", - ]); - expect(state()).toBe("ready"); - const reaper = new CleanupReaper({ - clock: harness.clock, - config: { ...config(), diskPressure: { freeBytesThreshold: 10 * gibibyte } }, - eventBus: harness.bus, - executor: harness.engine.cleanup, - filesystem: new MemoryFilesystem(), - registry: harness.registry, - targetedDevices: () => harness.engine.core.targetedDevices(), - }); - try { - for (let pass = 0; pass < 2; pass += 1) { - await reaper.run(); - await harness.engine.settle(); - } - expect( - harness.bus - .replay() - .filter( - (event) => - event.event === "device.shutdown" && - event.payload.deviceId === previous.id && - event.payload.initiator === "cleanup-reaper", - ), - "a device booted for a queued requester must not be reaped", - ).toEqual([]); - expect(state()).toBe("ready"); - await harness.engine.cancelPending("behind"); - expect( - [...(await harness.engine.core.targetedDevices())], - "cancelling the waiter removes cleanup protection immediately", - ).not.toContain(previous.id); - } finally { - reaper.dispose(); - await harness.engine.cancelPending("head"); - await harness.engine.cancelPending("behind"); - await Promise.all(pending); - harness.engine.dispose(); - } - }); -}); - -describe("createLeasing warm targets", () => { - const target = { - count: 2, - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - } as const; - const roomy = { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 4, maxRunning: 4 }, - maxRunning: 4, - }; - const eventsNamed = (harness: Awaited>, name: string) => - harness.bus.replay().filter((event) => event.event === name); - - async function drain( - harness: Awaited>, - driverClock?: FakeClock, - stepMs = 50, - ) { - for (let round = 0; round < 40; round += 1) { - await flush(); - harness.clock.advance(stepMs); - driverClock?.advance(stepMs); - } - await flush(); - await harness.engine.settle(); - } - - it("after daemon.started with a target of two and no devices, creates two devices one after the other, the second only after the first is ready", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { makeReady: 50 }, - platform: "ios", - }); - const harness = await createHarness({ driver, limits: roomy, warmPool: { targets: [target] } }); - await harness.engine.convergeRunningCapacity(); - - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await flush(); - // why: The first is mid-boot: created, not yet ready, and no second one started. - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); - expect(eventsNamed(harness, "device.ready")).toHaveLength(0); - await drain(harness, clock); - - const order = harness.bus - .replay() - .map((event) => event.event) - .filter((name) => name === "device.provisioned" || name === "device.ready"); - expect(order).toEqual([ - "device.provisioned", - "device.ready", - "device.provisioned", - "device.ready", - ]); - expect(harness.registry.snapshot.devices.map((device) => device.state)).toEqual([ - "ready", - "ready", - ]); - }); - - it("on a graceful drain finishes the creation in flight and creates no further device for the target", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { makeReady: 50 }, - platform: "ios", - }); - const harness = await createHarness({ driver, limits: roomy, warmPool: { targets: [target] } }); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await flush(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); - - const draining = harness.engine.core.drain(); - await drain(harness, clock); - await draining; - - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); - expect(harness.registry.snapshot.devices.map((device) => device.state)).toEqual(["ready"]); - }); - - it("creates no device for a target once the warm pool is closed for a stop", async () => { - const harness = await createHarness({ limits: roomy, warmPool: { targets: [target] } }); - await harness.engine.convergeRunningCapacity(); - harness.engine.core.closeWarmPool(); - - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - - expect(eventsNamed(harness, "device.provisioned")).toEqual([]); - }); - - it("grants the first lease of the targeted kind from a ready device with no booting stage, then keeps the count by creating another", async () => { - const harness = await createHarness({ limits: roomy, warmPool: { targets: [target] } }); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - const progress: LeaseProgress[] = []; - - const granted = await harness.engine.request(request, { - onProgress: (update) => progress.push(update), - ownerId: "agent", - requesterId: "agent", - ttlMs: 3_600_000, - }); - await drain(harness); - - expect(progress.map((update) => update.stage)).not.toContain("booting"); - expect(progress.map((update) => update.stage)).not.toContain("provisioning"); - expect(eventsNamed(harness, "lease.granted")[0]).toMatchObject({ payload: { source: "warm" } }); - const states = harness.registry.snapshot.devices.map((device) => device.state).sort(); - expect(states).toEqual(["leased", "ready", "ready"]); - expect(granted.device.spec).toMatchObject({ model: "iPhone 16", osVersion: "26.5" }); - }); - - it("makes one creation attempt per allowed attempt when every boot fails, and recovers when one succeeds", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("makeReady", 1, new DriverCrashError("no boot")); - driver.failOn("makeReady", 2, new DriverCrashError("no boot")); - const harness = await createHarness({ - driver, - limits: roomy, - warmPool: { targets: [{ ...target, count: 1 }] }, - }); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await harness.engine.settle(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); - - harness.clock.advance(60_000 - 1); - await harness.engine.settle(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(1); - harness.clock.advance(1); - await harness.engine.settle(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); - - harness.clock.advance(2 * 60_000 - 1); - await harness.engine.settle(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); - harness.clock.advance(1); - await harness.engine.settle(); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(3); - expect( - harness.registry.snapshot.devices.filter((device) => device.state === "ready"), - ).toHaveLength(1); - }); - - it("with a target naming an OS that is not installed, starts no install", async () => { - const install = vi.fn(async () => ({ outcome: "installed" as const, version: "27.0" })); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const harness = await createHarness({ - components: { claimProvision: () => () => undefined, install }, - driver, - limits: roomy, - warmPool: { targets: [{ ...target, osVersion: "27.0" }] }, - }); - await harness.engine.convergeRunningCapacity(); - - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - - expect(install).not.toHaveBeenCalled(); - expect(eventsNamed(harness, "component.install-started")).toEqual([]); - expect(driver.calls.filter((call) => call.operation === "installComponent")).toEqual([]); - expect(harness.registry.snapshot.devices).toEqual([]); - }); - - it("under fresh identity, creates a device before any lease, and after that lease ends and the spent device is deleted, creates the next", async () => { - const harness = await createHarness({ - identity: { android: "reusable", ios: "fresh" }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - warmPool: { targets: [{ ...target, count: 1 }] }, - }); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - const [first] = harness.registry.snapshot.devices; - expect(harness.registry.snapshot.devices).toHaveLength(1); - expect(first).toMatchObject({ leaseIdentity: "fresh", state: "ready" }); - - const granted = await harness.engine.request(request, { - ownerId: "a", - requesterId: "a", - ttlMs: 3_600_000, - }); - expect(granted.device.id).toBe(first?.id); - await harness.engine.release(granted.lease.id, "explicit"); - await drain(harness); - - const devices = harness.registry.snapshot.devices.filter( - (device) => device.state !== "deleted", - ); - expect( - harness.registry.snapshot.devices.find((device) => device.id === first?.id), - ).toMatchObject({ state: "deleted" }); - expect(devices).toHaveLength(1); - expect(devices[0]).toMatchObject({ leaseIdentity: "fresh", state: "ready" }); - expect(devices[0]?.id).not.toBe(first?.id); - expect(eventsNamed(harness, "device.provisioned")).toHaveLength(2); - }); - - it("shuts a never-leased ready device no target counts down with initiator warm-pool once it has been idle past idle.shutdownAfterMs", async () => { - const harness = await createHarness({ limits: roomy }); - const device = await seedReady(harness); - await harness.engine.convergeRunningCapacity(); - - // why: a pass runs on a trigger, and the periodic tick is 30 s away: a daemon.started fact is one - // at the threshold and one millisecond past it. - harness.clock.advance(10_000); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( - "ready", - ); - expect(eventsNamed(harness, "device.shutdown")).toEqual([]); - - harness.clock.advance(1); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - - expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( - "shutdown", - ); - expect(eventsNamed(harness, "device.shutdown")).toMatchObject([ - { payload: { deviceId: device.id, initiator: "warm-pool" } }, - ]); - }); - - it("keeps a never-leased ready device a target counts past idle.shutdownAfterMs", async () => { - const harness = await createHarness({ - limits: roomy, - warmPool: { targets: [{ ...target, count: 1 }] }, - }); - const device = await seedReady(harness); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await drain(harness); - - harness.clock.advance(5 * 60_000); - await harness.engine.settle(); - - expect(harness.registry.snapshot.devices.find((item) => item.id === device.id)?.state).toBe( - "ready", - ); - expect(eventsNamed(harness, "device.shutdown")).toEqual([]); - // why: The reaper reads the same set through the core, so it leaves the device alone too. - expect([...(await harness.engine.core.targetedDevices())]).toEqual([device.id]); - }); -}); - -describe("createLeasing's port for the warm pool", () => { - it("answers each platform's configured default mode, and full where none is configured", async () => { - const { engine } = await createHarness({ defaultModes: { ios: "slim" } }); - - const { defaultMode } = engine.leasing.corePorts.warmPoolDemand; - - expect([defaultMode("ios"), defaultMode("android")]).toEqual(["slim", "full"]); - }); -}); - -describe("createLeasing a lease ID chosen by the requester", () => { - const roomy = { - android: { maxDevices: 2, maxRunning: 2 }, - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 4, - }; - - function asking(requesterId: string, leaseId?: string, more: Record = {}) { - return { - ownerId: requesterId, - requesterId, - ...(leaseId === undefined ? {} : { leaseId }), - ...more, - }; - } - - async function withWaiting(requesterId: string, leaseId: string) { - const harness = await createHarness(); - await harness.engine.request(request, asking("holder")); - void harness.engine.request(request, asking(requesterId, leaseId)).catch(() => undefined); - await expect - .poll(() => - harness.registry.leaseRequests().find((record) => record.requesterId === requesterId), - ) - .toMatchObject({ state: "open" }); - return harness; - } - - it("a lease request with leaseId gets a lease with exactly that ID", async () => { - const harness = await createHarness(); - - const grant = await harness.engine.request(request, asking("agent-1", "ad-7f3a")); - - expect(grant.lease.id).toBe("ad-7f3a"); - expect(grant.lease).toMatchObject({ idChosenByRequester: true }); - expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["ad-7f3a"]); - }); - - it("a lease request without leaseId gets an lse_ ID as today", async () => { - const harness = await createHarness(); - - const grant = await harness.engine.request(request, asking("agent-1")); - - expect(grant.lease.id).toMatch(/^lse_/); - expect(grant.lease).toMatchObject({ idChosenByRequester: false }); - }); - - it("renew and release by a caller-chosen ID work", async () => { - const harness = await createHarness(); - await harness.engine.request(request, asking("agent-1", "ad-7f3a")); - - await expect(harness.engine.renew("ad-7f3a", 5_000)).resolves.toMatchObject({ - id: "ad-7f3a", - ttlMs: 5_000, - }); - await harness.engine.release("ad-7f3a", "explicit"); - - expect(harness.registry.snapshot.leases).toEqual([]); - expect( - harness.bus.replay().find((event) => event.event === "lease.released")?.payload, - ).toMatchObject({ - leaseId: "ad-7f3a", - }); - }); - - it("MyID and myid are two different IDs", async () => { - const harness = await createHarness({ limits: roomy }); - - const first = await harness.engine.request(request, asking("agent-1", "myid")); - const second = await harness.engine.request(request, asking("agent-2", "MyID")); - - expect([first.lease.id, second.lease.id]).toEqual(["myid", "MyID"]); - }); - - it("a second request for an ID held by an active lease fails with LEASE_ID_TAKEN", async () => { - const harness = await createHarness({ limits: roomy }); - await harness.engine.request(request, asking("agent-1", "myid")); - - await expect(harness.engine.request(request, asking("agent-2", "myid"))).rejects.toMatchObject({ - leaseId: "myid", - message: "lease ID myid is already in use", - name: "LeaseIdTakenError", - }); - expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual(["myid"]); - }); - - it("a second request for an ID held by a waiting request fails with LEASE_ID_TAKEN", async () => { - const harness = await withWaiting("agent-2", "myid"); - - const clash = harness.engine.request(request, asking("agent-3", "myid")); - void clash.catch(() => undefined); - - // why: Settled at once: a request that was let in would wait for the one device instead. - await expect(settledOrPending(clash)).resolves.toMatchObject({ - leaseId: "myid", - name: "LeaseIdTakenError", - }); - }); - - it("a requester that already holds a lease gets REQUESTER_ALREADY_LEASED, not LEASE_ID_TAKEN, even when it repeats its own lease ID", async () => { - const harness = await createHarness({ limits: roomy }); - const first = await harness.engine.request(request, asking("agent-1", "myid")); - - await expect(harness.engine.request(request, asking("agent-1", "myid"))).rejects.toMatchObject({ - existingLeaseId: first.lease.id, - name: "RequesterAlreadyLeasedError", - }); - }); - - it("a LEASE_ID_TAKEN refusal emits lease.rejected with reason lease-id-taken", async () => { - const harness = await createHarness({ limits: roomy }); - await harness.engine.request(request, asking("agent-1", "myid")); - - await harness.engine.request(request, asking("agent-2", "myid")).catch(() => undefined); - - const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); - expect(rejected.map((event) => event.payload)).toEqual([ - { - reason: "lease-id-taken", - requestId: expect.stringMatching(/^req_/), - requester: "agent-2", - requestSpec: request, - }, - ]); - expect(rejected.map((event) => event.module)).toEqual(["lease-acquisition-coordinator"]); - }); - - it("a caller re-sending a chosen lease id whose lease ended within the window gets LEASE_ENDED with the id and when it ended", async () => { - type Harness = Awaited>; - const dayMs = 24 * 60 * 60_000; - const freshDriver = () => - new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const cases: readonly { - readonly name: string; - readonly endIt: (harness: Harness, id: string) => Promise; - readonly outcome: "ended" | "granted" | "taken"; - readonly endedAt?: number; - /** why: the first lease asks for no id (`generated`), or for one the retry does not use (`other`). */ - readonly asked?: "generated" | "other"; - /** why: rows left in `ended_lease_ids` once the retry is answered. */ - readonly rows?: number; - }[] = [ - { - name: "released, and another lease commits after it", - endIt: async (harness) => { - await harness.engine.release("myid", "explicit"); - harness.clock.advance(5_000); - await harness.engine.request(request, asking("agent-3", "other")); - return harness; - }, - outcome: "ended", - endedAt: 1_000, - }, - { - name: "expired", - endIt: async (harness) => { - harness.clock.advance(100_000); - await flush(); - return harness; - }, - outcome: "ended", - endedAt: 101_000, - }, - { - name: "ended by nuke", - endIt: async (harness) => { - await harness.engine.nuke(false); - return harness; - }, - outcome: "ended", - endedAt: 1_000, - }, - { - name: "ended at a daemon start", - endIt: async (harness) => { - const restarted = await createHarness({ driver: freshDriver(), state: harness.state }); - await restarted.engine.convergeRunningCapacity(); - return restarted; - }, - outcome: "ended", - endedAt: 1_000, - }, - { - name: "after a daemon restart", - endIt: async (harness) => { - await harness.engine.release("myid", "explicit"); - return createHarness({ driver: freshDriver(), state: harness.state }); - }, - outcome: "ended", - endedAt: 1_000, - }, - { - name: "past the window", - endIt: async (harness) => { - await harness.engine.release("myid", "explicit"); - harness.clock.advance(dayMs + 1); - return harness; - }, - outcome: "granted", - rows: 0, - }, - { - name: "a lease with a generated id", - asked: "generated", - endIt: async (harness, id) => { - await harness.engine.release(id, "explicit"); - return harness; - }, - outcome: "granted", - rows: 0, - }, - { - name: "never used", - asked: "other", - endIt: async (harness) => harness, - outcome: "granted", - }, - { name: "still held", endIt: async (harness) => harness, outcome: "taken" }, - ]; - - for (const { name, asked, endIt, outcome, endedAt, rows } of cases) { - const first = await createHarness({ lease: { defaultTtlMs: 100_000 }, limits: roomy }); - const granted = await first.engine.request( - request, - asking("agent-1", asked === "generated" ? undefined : asked === "other" ? "other" : "myid"), - ); - const harness = await endIt(first, granted.lease.id); - const again = asked === "other" ? "myid" : granted.lease.id; - harness.clock.advance(1_000); - - const retry = harness.engine.request(request, asking("agent-2", again)); - - if (outcome === "ended") { - await expect(retry, name).rejects.toMatchObject({ - code: "LEASE_ENDED", - details: { endedAt, leaseId: "myid" }, - message: `Lease myid ended at ${new Date(endedAt ?? 0).toISOString()}; choose a new lease id`, - }); - } else if (outcome === "taken") { - await expect(retry, name).rejects.toMatchObject({ name: "LeaseIdTakenError" }); - } else { - await expect(retry, name).resolves.toMatchObject({ lease: { id: again } }); - } - if (rows !== undefined) { - expect( - harness.state.database.prepare("SELECT COUNT(*) AS n FROM ended_lease_ids").get(), - name, - ).toEqual({ n: rows }); - } - } - }); - - it("lease.requested and lease.rejected requestSpec, and the status waiting spec, carry no leaseId", async () => { - const harness = await withWaiting("agent-2", "myid"); - await settledOrPending(harness.engine.request(request, asking("agent-3", "myid"))); - - const requested = harness.bus.replay().filter((event) => event.event === "lease.requested"); - const rejected = harness.bus.replay().filter((event) => event.event === "lease.rejected"); - expect(requested.map((event) => event.payload)).toContainEqual( - expect.objectContaining({ requester: "agent-2", requestSpec: request }), - ); - expect( - rejected.map((event) => (event.payload as { requestSpec: unknown }).requestSpec), - ).toEqual([request]); - expect(harness.engine.waitingRequests().map((entry) => entry.spec)).toEqual([request]); - expect( - harness.registry.leaseRequests().find((record) => record.requesterId === "agent-2")?.request, - ).toEqual(request); - }); - - it("a retry with the same idempotency key and the same leaseId replays the first answer", async () => { - const harness = await createHarness(); - const asker = asking("agent-1", "myid", { idempotencyKey: "key-1" }); - const first = await harness.engine.request(request, asker); - - const retry = await harness.engine.request(request, asker); - - expect(retry).toEqual(first); - expect(retry.lease.id).toBe("myid"); - expect(harness.registry.snapshot.leases).toHaveLength(1); - }); - - it.each([ - ["a different leaseId", "other"], - ["no leaseId", undefined], - ])( - "a retry with the same idempotency key and %s fails with IDEMPOTENCY_CONFLICT", - async (_name, retryId) => { - const harness = await createHarness(); - await harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })); - - await expect( - harness.engine.request(request, asking("agent-1", retryId, { idempotencyKey: "key-1" })), - ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); - }, - ); - - it("a retry with the same idempotency key that adds a leaseId the first call did not have fails with IDEMPOTENCY_CONFLICT", async () => { - const harness = await createHarness(); - const first = await harness.engine.request( - request, - asking("agent-1", undefined, { idempotencyKey: "key-1" }), - ); - - await expect( - harness.engine.request(request, asking("agent-1", "myid", { idempotencyKey: "key-1" })), - ).rejects.toMatchObject({ name: "IdempotencyConflictError" }); - await expect( - harness.engine.request(request, asking("agent-1", undefined, { idempotencyKey: "key-1" })), - ).resolves.toEqual(first); - expect(harness.registry.snapshot.leases.map((lease) => lease.id)).toEqual([first.lease.id]); - }); - - it("after a daemon restart, an ID held only by a waiting request is accepted again", async () => { - const before = await withWaiting("agent-2", "myid"); - // why: Started again with room to spare, so the only thing that could refuse "myid" is the ID. - const after = await createHarness({ state: before.state, limits: roomy }); - await after.engine.convergeRunningCapacity(); - - await expect(after.engine.request(request, asking("agent-3", "myid"))).resolves.toMatchObject({ - lease: { id: "myid" }, - }); - }); -}); - -describe("createLeasing: a requester may hold up to lease.maxPerRequester leases (ADR 0023)", () => { - const roomy = { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 4, maxRunning: 4 }, - maxRunning: 5, - }; - const twoDevices = { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 3, - }; - const asking = (requesterId: string, more: Record = {}) => ({ - ownerId: requesterId, - requesterId, - ...more, - }); - const atLimit = (held: number, limit: number, leaseId: string) => - `Requester agent-1 already holds ${String(held)} leases and lease.maxPerRequester is ${String(limit)}; release one (\`simlock release ${leaseId}\`) before requesting another device`; - const waitingMessage = - "Requester agent-1 already has a waiting request; wait for it to be granted or cancel it before requesting another device"; - - it("with no lease.maxPerRequester, a requester holding a lease is refused with today's message, naming that lease, and is granted another once it releases it", async () => { - const harness = await createHarness({ limits: roomy }); - const first = await harness.engine.request(request, asking("agent-1")); - - const refusal = await harness.engine - .request(request, asking("agent-1")) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: first.lease.id, - message: `Requester agent-1 already holds lease ${first.lease.id}; release it (\`simlock release ${first.lease.id}\`) before requesting another device`, - name: "RequesterAlreadyLeasedError", - }); - - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - await expect(harness.engine.request(request, asking("agent-1"))).resolves.toMatchObject({ - lease: { ownerId: "agent-1", requesterId: "agent-1" }, - }); - }); - - it("with no lease.maxPerRequester, a requester with a waiting request is refused with today's message", async () => { - const harness = await createHarness(); - await harness.engine.request(request, asking("holder")); - void harness.engine.request(request, asking("agent-1")).catch(() => undefined); - await expect - .poll(() => - harness.registry.leaseRequests().find((record) => record.requesterId === "agent-1"), - ) - .toMatchObject({ state: "open" }); - - const refusal = await harness.engine - .request(request, asking("agent-1")) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: "Requester already has a lease or pending request: agent-1", - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with lease.maxPerRequester: 2, one requester holds two leases at once", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - - const first = await harness.engine.request(request, asking("agent-1")); - const second = await harness.engine.request(request, asking("agent-1")); - - expect(second.lease.id).not.toBe(first.lease.id); - expect( - harness.registry.snapshot.leases.filter((lease) => lease.requesterId === "agent-1"), - ).toHaveLength(2); - }); - - it("with lease.maxPerRequester: 2, a requester holding two leases is refused, the message naming the limit and the earlier-granted lease", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - await harness.engine.request(request, asking("agent-1", { leaseId: "zeta" })); - harness.clock.advance(5); - await harness.engine.request(request, asking("agent-1", { leaseId: "alpha" })); - - const refusal = await harness.engine - .request(request, asking("agent-1")) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: "zeta", - message: atLimit(2, 2, "zeta"), - name: "RequesterAlreadyLeasedError", - requesterId: "agent-1", - }); - }); - - it("with lease.maxPerRequester: 3, a requester holding one lease and one waiting request is refused with the waiting-request message", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 3 }, limits: twoDevices }); - await harness.engine.request(request, asking("agent-1")); - await harness.engine.request(request, asking("other")); - void harness.engine.request(request, asking("agent-1")).catch(() => undefined); - await expect - .poll(() => - harness.registry - .leaseRequests() - .filter((record) => record.requesterId === "agent-1" && record.state === "open"), - ) - .toHaveLength(1); - - const refusal = await harness.engine - .request(request, asking("agent-1")) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: waitingMessage, - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with lease.maxPerRequester per platform, a requester holding an iOS lease is granted an Android one", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: { ios: 1, android: 1 } }, - limits: roomy, - }); - const first = await harness.engine.request(request, asking("agent-1")); - - const second = await harness.engine.request(androidRequest, asking("agent-1")); - - expect(first.device.spec.platform).toBe("ios"); - expect(second.device.spec.platform).toBe("android"); - }); - - it.each([ - { - name: "limit 1 on iOS", - limit: { ios: 1, android: 1 }, - held: [request], - next: request, - message: (ids: string[]) => - `Requester agent-1 already holds iOS lease ${ids[0]}; release it (\`simlock release ${ids[0]}\`) before requesting another iOS device`, - }, - { - name: "an Android platform the object leaves out, so its limit is 1", - limit: { ios: 2 }, - held: [androidRequest], - next: androidRequest, - message: (ids: string[]) => - `Requester agent-1 already holds Android lease ${ids[0]}; release it (\`simlock release ${ids[0]}\`) before requesting another Android device`, - }, - { - name: "two iOS leases at an iOS limit of 2", - limit: { ios: 2, android: 1 }, - held: [request, request], - next: request, - message: (ids: string[]) => - `Requester agent-1 already holds 2 iOS leases and lease.maxPerRequester.ios is 2; release one (\`simlock release ${ids[0]}\`) before requesting another iOS device`, - }, - ])( - "with lease.maxPerRequester per platform, a requester at its limit on a platform is refused with that platform's message naming its earliest lease ($name)", - async ({ limit, held, next, message }) => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: limit }, - limits: roomy, - }); - const ids: string[] = []; - for (const spec of held) { - const grant = await harness.engine.request(spec, asking("agent-1")); - ids.push(grant.lease.id); - harness.clock.advance(5); - } - - const refusal = await harness.engine - .request(next, asking("agent-1")) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: ids[0], - message: message(ids), - name: "RequesterAlreadyLeasedError", - }); - }, - ); - - it.each([ - { - name: "an Android request while an iOS one waits is admitted", - limit: { ios: 1, android: 1 }, - iosDevices: 1, - next: androidRequest, - refusal: undefined, - }, - { - name: "an iOS request while an iOS one waits is refused", - limit: { ios: 1, android: 1 }, - iosDevices: 1, - next: request, - refusal: - "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", - }, - { - name: "an iOS request holding one iOS lease and waiting for another under an iOS limit of 2 is refused", - limit: { ios: 2 }, - iosDevices: 2, - next: request, - holdsIos: true, - refusal: - "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", - }, - ])( - "with lease.maxPerRequester per platform, a requester with a waiting request is refused another on that platform only ($name)", - async ({ limit, iosDevices, next, refusal, holdsIos }) => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: limit }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: iosDevices, maxRunning: iosDevices }, - maxRunning: iosDevices + 1, - }, - }); - await harness.engine.request(request, asking("holder")); - await harness.engine.request(androidRequest, asking("holder")); - if (holdsIos === true) await harness.engine.request(request, asking("agent-1")); - void harness.engine.request(request, asking("agent-1")).catch(() => undefined); - const openOf = (requesterId: string) => - harness.registry - .leaseRequests() - .filter((record) => record.requesterId === requesterId && record.state === "open"); - await expect.poll(() => openOf("agent-1")).toHaveLength(1); - - const outcome = harness.engine.request(next, asking("agent-1")).catch((e: unknown) => e); - - await harness.engine.settle(); - const state = await settledOrPending(outcome); - if (refusal === undefined) { - expect(state).toBe("still pending"); - expect(openOf("agent-1")).toHaveLength(2); - } else { - expect(state).toMatchObject({ message: refusal, name: "RequesterAlreadyLeasedError" }); - } - }, - ); - - it.each([ - { - name: "2, waiting for iOS, then Android: admitted", - limit: 2, - granted: [ - ["holder-1", request], - ["holder-2", androidRequest], - ], - waits: request, - next: androidRequest, - refusal: undefined, - }, - { - name: "2, waiting for iOS, then iOS: refused", - limit: 2, - granted: [ - ["holder-1", request], - ["holder-2", androidRequest], - ], - waits: request, - next: request, - refusal: waitingMessage, - }, - { - name: "1, waiting for iOS, then Android: refused", - limit: 1, - granted: [ - ["holder-1", request], - ["holder-2", androidRequest], - ], - waits: request, - next: androidRequest, - refusal: "Requester already has a lease or pending request: agent-1", - }, - { - name: "2, holding iOS and waiting for Android, then iOS: refused", - limit: 2, - granted: [ - ["agent-1", request], - ["holder-2", androidRequest], - ], - waits: androidRequest, - next: request, - refusal: waitingMessage, - }, - ] as const)( - "with a numeric lease.maxPerRequester, waiting and held requests share the limit across platforms ($name)", - async ({ limit, granted, waits, next, refusal }) => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: limit }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - }); - for (const [requesterId, spec] of granted) { - await harness.engine.request(spec, asking(requesterId)); - } - void harness.engine.request(waits, asking("agent-1")).catch(() => undefined); - await expect - .poll(() => - harness.registry - .leaseRequests() - .filter((record) => record.requesterId === "agent-1" && record.state === "open"), - ) - .toHaveLength(1); - - const outcome = harness.engine.request(next, asking("agent-1")).catch((e: unknown) => e); - - await harness.engine.settle(); - const state = await settledOrPending(outcome); - if (refusal === undefined) { - expect(state).toBe("still pending"); - } else { - expect(state).toMatchObject({ message: refusal, name: "RequesterAlreadyLeasedError" }); - } - }, - ); - - it("with lease.maxPerRequester per platform, a gateway probe is refused only by a waiting probe on its own platform", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: { ios: 1, android: 1 } }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - }); - await harness.engine.request(request, asking("holder-1")); - await harness.engine.request(androidRequest, asking("holder-2")); - const probe = (spec: typeof request | typeof androidRequest, id: string) => - harness.engine - .request(spec, asking("gw:1:agent-1", { fleetRequestId: id })) - .catch((e: unknown) => e); - void probe(request, "fleet_1"); - await expect - .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) - .toHaveLength(1); - - const android2 = probe(androidRequest, "fleet_2"); - const ios2 = probe(request, "fleet_3"); - - await expect(ios2).resolves.toMatchObject({ - message: expect.stringContaining("already has a waiting iOS request"), - name: "RequesterAlreadyLeasedError", - }); - await expect(Promise.race([android2, Promise.resolve("waiting")])).resolves.toBe("waiting"); - }); - - it.each([ - { - name: "Android", - waiting: ["ios", "android"], - cancel: "android", - result: "cancelled", - left: ["ios"], - }, - { - name: "none", - waiting: ["ios", "android"], - cancel: undefined, - result: "cancelled", - left: ["android"], - }, - { - name: "iOS with only Android waiting", - waiting: ["android"], - cancel: "ios", - result: "not-found", - left: ["android"], - }, - ] as const)( - "cancelPending names the platform of the waiting request to cancel, and the earliest without one ($name)", - async ({ waiting, cancel, result, left }) => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: { ios: 1, android: 1 } }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - }); - await harness.engine.request(request, asking("holder-1")); - await harness.engine.request(androidRequest, asking("holder-2")); - const specs = { android: androidRequest, ios: request }; - for (const [index, platform] of waiting.entries()) { - void harness.engine.request(specs[platform], asking("agent-1")).catch(() => undefined); - await expect - .poll(() => - harness.registry - .leaseRequests() - .filter((record) => record.requesterId === "agent-1" && record.state === "open"), - ) - .toHaveLength(index + 1); - } - - await harness.engine.settle(); - await expect(harness.engine.cancelPending("agent-1", cancel)).resolves.toBe(result); - - expect( - harness.registry - .leaseRequests() - .filter((record) => record.requesterId === "agent-1" && record.state === "open") - .map((record) => record.request.platform), - ).toEqual(left); - }, - ); - - it("pendingRequestOwner names the owner of the waiting request of the platform it names, and of the earliest without one", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const harness = await createHarness({ - driver: ios, - drivers: [ios, android], - lease: { maxPerRequester: { ios: 1, android: 1 } }, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - }); - await harness.engine.request(request, asking("holder-1")); - await harness.engine.request(androidRequest, asking("holder-2")); - void harness.engine - .request(request, { ownerId: "ios-owner", requesterId: "agent-1" }) - .catch(() => undefined); - await expect - .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) - .toHaveLength(1); - void harness.engine - .request(androidRequest, { ownerId: "android-owner", requesterId: "agent-1" }) - .catch(() => undefined); - await expect - .poll(() => harness.registry.leaseRequests().filter((record) => record.state === "open")) - .toHaveLength(2); - - expect(harness.engine.pendingRequestOwner("agent-1", "android")).toBe("android-owner"); - expect(harness.engine.pendingRequestOwner("agent-1", "ios")).toBe("ios-owner"); - expect(harness.engine.pendingRequestOwner("agent-1")).toBe("ios-owner"); - }); - - it("a refusal at the lease limit emits lease.rejected with reason already-leased", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - await harness.engine.request(request, asking("agent-1")); - await harness.engine.request(request, asking("agent-1")); - - await harness.engine.request(request, asking("agent-1")).catch(() => undefined); - - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload), - ).toEqual([ - { - reason: "already-leased", - requestId: expect.stringMatching(/^req_/), - requester: "agent-1", - requestSpec: request, - }, - ]); - }); - - it("releasing one of two leases lets the requester's next request through", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - const first = await harness.engine.request(request, asking("agent-1")); - await harness.engine.request(request, asking("agent-1")); - await expect(harness.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ - name: "RequesterAlreadyLeasedError", - }); - - await harness.engine.release(first.lease.id, "explicit"); - - await expect(harness.engine.request(request, asking("agent-1"))).resolves.toMatchObject({ - lease: { id: expect.stringMatching(/^lse_/) }, - }); - }); - - it("at the lease limit, a repeat with a used idempotencyKey gets its stored answer", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - const first = await harness.engine.request( - request, - asking("agent-1", { idempotencyKey: "k1" }), - ); - await harness.engine.request(request, asking("agent-1", { idempotencyKey: "k2" })); - - const repeat = await harness.engine.request( - request, - asking("agent-1", { idempotencyKey: "k1" }), - ); - - expect(repeat).toEqual(first); - }); - - it("at the lease limit, a request naming a taken lease id gets REQUESTER_ALREADY_LEASED", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - await harness.engine.request(request, asking("agent-2", { leaseId: "taken" })); - await harness.engine.request(request, asking("agent-1")); - await harness.engine.request(request, asking("agent-1")); - - await expect( - harness.engine.request(request, asking("agent-1", { leaseId: "taken" })), - ).rejects.toMatchObject({ name: "RequesterAlreadyLeasedError" }); - }); - - it("after a restart with lease.maxPerRequester lowered from 3 to 1, a requester's three running leases are all kept, and its next request is refused naming the earliest one", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const before = await createHarness({ - driver, - lease: { maxPerRequester: 3 }, - limits: roomy, - }); - const earliest = await before.engine.request(request, asking("agent-1")); - before.clock.advance(5); - await before.engine.request(request, asking("agent-1")); - await before.engine.request(request, asking("agent-1")); - - const after = await createHarness({ - driver, - state: before.state, - lease: { maxPerRequester: 1 }, - limits: roomy, - }); - await after.engine.convergeRunningCapacity(); - - expect( - after.registry.snapshot.leases.filter((lease) => lease.requesterId === "agent-1"), - ).toHaveLength(3); - await expect(after.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ - existingLeaseId: earliest.lease.id, - name: "RequesterAlreadyLeasedError", - }); - }); - - it("a worker still refuses a local request of a requester at its limit while probes of other requesters are granted", async () => { - const harness = await createHarness({ lease: { maxPerRequester: 2 }, limits: roomy }); - await harness.engine.request(request, asking("agent-1")); - await harness.engine.request(request, asking("agent-1")); - - await expect(harness.engine.request(request, asking("agent-1"))).rejects.toMatchObject({ - name: "RequesterAlreadyLeasedError", - }); - await expect( - harness.engine.request(request, { - fleetRequestId: "req_gw1", - noWait: true, - ownerId: "gw:two", - requesterId: "gw:two", - }), - ).resolves.toMatchObject({ lease: { id: expect.stringMatching(/^lse_/) } }); - }); -}); - -describe("createLeasing: a gateway dispatch is a probe (ADR 0021)", () => { - type Harness = Awaited>; - - const holder = { ownerId: "holder", requesterId: "holder" }; - const probeOf = (requesterId: string, extra: Record = {}) => ({ - fleetRequestId: "req_gw1", - noWait: true, - ownerId: requesterId, - requesterId, - ...extra, - }); - - function payloads< - Name extends - | "lease.declined" - | "lease.rejected" - | "lease.requested" - | "lease.queued" - | "lease.granted", - >(harness: Harness, name: Name): Array { - return harness.bus - .replay() - .filter((event) => event.event === name) - .map((event) => event.payload as EventMap[Name]); - } - - async function admittedId(harness: Harness, requesterId: string): Promise { - return harness.registry.leaseRequests().find((record) => record.requesterId === requesterId) - ?.id; - } - - interface Declined { - readonly harness: Harness; - readonly payload: unknown; - readonly check?: () => void; - } - - const declinedCases: ReadonlyArray Promise]> = [ - [ - "no-wait", - async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - await expect(harness.engine.request(request, probeOf("gw:one"))).rejects.toBeInstanceOf( - NoCapacityError, - ); - return { - harness, - payload: { - fleetRequestId: "req_gw1", - reason: "no-wait", - requestId: await admittedId(harness, "gw:one"), - requestSpec: request, - requester: "gw:one", - }, - }; - }, - ], - [ - "unresolvable-spec", - async () => { - const harness = await createHarness(); - const odd = { ...request, model: "iPhone 99 Imaginary", osVersion: ">=18" }; - await settledOrPending(harness.engine.request(odd, probeOf("gw:odd"))); - return { - harness, - payload: expect.objectContaining({ - fleetRequestId: "req_gw1", - reason: "unresolvable-spec", - requestSpec: odd, - }), - }; - }, - ], - [ - "already-leased", - async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.hangMakeReady(); - const harness = await createHarness({ driver }); - void harness.engine.request(request, probeOf("gw:one")).catch(() => undefined); - await expect - .poll(() => - harness.registry.leaseRequests().find((record) => record.requesterId === "gw:one"), - ) - .toMatchObject({ state: "open" }); - await expect(harness.engine.request(request, probeOf("gw:one"))).rejects.toMatchObject({ - name: "RequesterAlreadyLeasedError", - }); - return { - harness, - payload: { - fleetRequestId: "req_gw1", - reason: "already-leased", - requestId: expect.stringMatching(/^req_/), - requestSpec: request, - requester: "gw:one", - }, - }; - }, - ], - [ - "lease-id-taken", - async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 2, maxRunning: 2 }, - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 4, - }, - }); - await harness.engine.request(request, { ...holder, leaseId: "myid" }); - await settledOrPending( - harness.engine.request(request, probeOf("gw:two", { leaseId: "myid" })), - ); - return { - harness, - payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "lease-id-taken" }), - }; - }, - ], - [ - "lease-ended", - async () => { - const harness = await createHarness(); - await harness.engine.request(request, { ...holder, leaseId: "myid" }); - await harness.engine.release("myid", "explicit"); - await expect( - harness.engine.request(request, probeOf("gw:two", { leaseId: "myid" })), - ).rejects.toMatchObject({ code: "LEASE_ENDED" }); - return { - harness, - payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "lease-ended" }), - }; - }, - ], - [ - "boot-timeout", - async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("makeReady", 1, new Error("boot failed")); - const harness = await createHarness({ driver }); - const stages: string[] = []; - await settledOrPending( - harness.engine.request(request, { - ...probeOf("gw:slow"), - onProgress: (progress) => stages.push(progress.stage), - }), - ); - return { - check: () => expect(stages.length).toBeGreaterThan(0), - harness, - payload: expect.objectContaining({ fleetRequestId: "req_gw1", reason: "boot-timeout" }), - }; - }, - ], - [ - "daemon-restarted", - async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.hangMakeReady(); - const before = await createHarness({ driver }); - void before.engine.request(request, probeOf("gw:open")).catch(() => undefined); - await expect - .poll(() => - before.registry.leaseRequests().find((record) => record.requesterId === "gw:open"), - ) - .toMatchObject({ fleetRequestId: "req_gw1", state: "open" }); - const requestId = await admittedId(before, "gw:open"); - const after = await createHarness({ state: before.state }); - await after.engine.convergeRunningCapacity(); - return { - harness: after, - payload: { - fleetRequestId: "req_gw1", - reason: "daemon-restarted", - requestId, - requestSpec: request, - requester: "gw:open", - }, - }; - }, - ], - ]; - - it.each(declinedCases)( - "emits one lease.declined with its fleetRequestId and reason %s, and no lease.rejected, for a probe refused that way", - async (_reason, run) => { - const { check, harness, payload } = await run(); - - expect(payloads(harness, "lease.declined")).toEqual([payload]); - expect(payloads(harness, "lease.rejected")).toEqual([]); - check?.(); - }, - ); - - it("still emits lease.rejected no-wait, and no lease.declined, for a local noWait request with no room", async () => { - const harness = await createHarness(); - await harness.engine.request(request, holder); - - await expect( - harness.engine.request(request, { noWait: true, ownerId: "local", requesterId: "local" }), - ).rejects.toBeInstanceOf(NoCapacityError); - - expect(payloads(harness, "lease.rejected")).toEqual([ - expect.objectContaining({ reason: "no-wait", requester: "local" }), - ]); - expect(payloads(harness, "lease.declined")).toEqual([]); - }); - - it("carries a probe's fleetRequestId on its lease.requested and lease.granted, and none on a local request's", async () => { - const harness = await createHarness({ - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 2, - }, - }); - - await harness.engine.request(request, probeOf("gw:one")); - await harness.engine.request(request, { ownerId: "local", requesterId: "local" }); - - const [requestedProbe, requestedLocal] = payloads(harness, "lease.requested"); - const [grantedProbe, grantedLocal] = payloads(harness, "lease.granted"); - expect(requestedProbe).toMatchObject({ fleetRequestId: "req_gw1", requester: "gw:one" }); - expect(grantedProbe).toMatchObject({ fleetRequestId: "req_gw1", requester: "gw:one" }); - expect(requestedLocal).toMatchObject({ requester: "local" }); - expect(requestedLocal).not.toHaveProperty("fleetRequestId"); - expect(grantedLocal).toMatchObject({ requester: "local" }); - expect(grantedLocal).not.toHaveProperty("fleetRequestId"); - }); - - it.each([ - { name: "a number", maxPerRequester: undefined }, - { name: "a per-platform object", maxPerRequester: { ios: 1, android: 1 } }, - ])( - "a worker grants a probe for a requester that already holds a lease from an earlier probe, lease.maxPerRequester being $name", - async ({ maxPerRequester }) => { - const harness = await createHarness({ - ...(maxPerRequester === undefined ? {} : { lease: { maxPerRequester } }), - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 4, - }, - }); - const first = await harness.engine.request(request, probeOf("gw:one")); - - const second = await harness.engine.request(request, probeOf("gw:one")); - - expect(second.lease.id).not.toBe(first.lease.id); - expect( - harness.registry.snapshot.leases.filter((lease) => lease.requesterId === "gw:one"), - ).toHaveLength(2); - }, - ); - - it("declines, rather than rejects, a request carrying fleetRequestId whose wait in the queue times out", async () => { - // why: The dispatcher only lets a probe in with noWait, so none is ever queued; this reaches the - // queue's timeout site directly, which applies the same rule as every other. - const harness = await createHarness(); - await harness.engine.request(request, holder); - const waiting = harness.engine.request(request, { - ...probeOf("gw:late", { noWait: false }), - timeoutMs: 10, - }); - await flush(); - harness.clock.advance(10); - - expect(await settledOrPending(waiting)).toBeInstanceOf(QueueTimeoutError); - - expect(payloads(harness, "lease.declined")).toEqual([ - expect.objectContaining({ fleetRequestId: "req_gw1", reason: "timeout" }), - ]); - expect(payloads(harness, "lease.rejected")).toEqual([]); - }); - - describe("a provision that fails twice", () => { - function failingTwice() { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("provision", 1, new DriverCrashError("simulator exited")); - driver.failOn("provision", 2, new DriverCrashError("simulator exited again")); - return driver; - } - - it("declines a probe with reason no-wait, answers PROVISION_FAILED with the driver's message, and emits no lease.queued", async () => { - const harness = await createHarness({ driver: failingTwice() }); - - const outcome = await settledOrPending(harness.engine.request(request, probeOf("gw:one"))); - - expect(outcome).toBeInstanceOf(ProvisionFailedError); - expect(outcome).toMatchObject({ message: "simulator exited again" }); - expect(payloads(harness, "lease.declined")).toEqual([ - expect.objectContaining({ fleetRequestId: "req_gw1", reason: "no-wait" }), - ]); - expect(payloads(harness, "lease.queued")).toEqual([]); - expect(payloads(harness, "lease.rejected")).toEqual([]); - }); - - it("rejects a local request in the same case with PROVISION_FAILED, lists nothing and emits no lease.queued", async () => { - const harness = await createHarness({ driver: failingTwice() }); - - const outcome = await settledOrPending( - harness.engine.request(request, { ownerId: "local", requesterId: "local" }), - ); - - expect(outcome).toBeInstanceOf(ProvisionFailedError); - expect(outcome).toMatchObject({ message: "simulator exited again" }); - expect(harness.engine.waitingRequests()).toEqual([]); - expect(payloads(harness, "lease.queued")).toEqual([]); - expect(payloads(harness, "lease.declined")).toEqual([]); - }); - }); -}); - -describe("createLeasing after a quarantine retry (ADR 0022 §2)", () => { - it("grants a request that waits at capacity a new simulator once the retry has deleted the quarantined one", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - const harness = await createHarness({ driver }); - const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); - const waiting = harness.engine.request(request, { ownerId: "second", requesterId: "second" }); - await flush(); - - harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); - await flush(); - - const granted = (await settledOrPending(waiting)) as LeaseGrant; - expect(granted).toMatchObject({ lease: { requesterId: "second" } }); - expect(granted.device.id).not.toBe(first.device.id); - expect( - harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, - ).toBe("deleted"); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - }); - - it("reports a warm target of one for the quarantined simulator's spec as ready 1 once the retry has deleted it and the pool has created a new one", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - const harness = await createHarness({ - driver, - warmPool: { targets: [{ count: 1, model: "iPhone 16", osVersion: "26.5", platform: "ios" }] }, - }); - await harness.engine.convergeRunningCapacity(); - harness.bus.emit("daemon.started", { configSnapshot: {}, version: "test" }, "test"); - await harness.engine.settle(); - const first = await harness.engine.request(request, { ownerId: "first", requesterId: "first" }); - await harness.engine.release(first.lease.id, "explicit"); - await harness.engine.settle(); - expect(harness.registry.snapshot.devices[0]?.state).toBe("quarantined"); - expect(harness.engine.core.warmPoolReader.figures().targets[0]?.ready).toBe(0); - - harness.clock.advance(config().warmPool.quarantine.retryBackoffMs); - await flush(); - await harness.engine.settle(); - - expect( - harness.bus - .replay() - .filter((event) => event.event === "device.deleted") - .map((event) => (event.payload as { deviceId: string }).deviceId), - ).toEqual([first.device.id]); - expect( - harness.registry.snapshot.devices.find((item) => item.id === first.device.id)?.state, - ).toBe("deleted"); - expect(harness.engine.core.warmPoolReader.figures().targets).toMatchObject([ - { count: 1, ready: 1 }, - ]); - }); -}); - -describe("createLeasing: a waiting request says why it is not placed (#636)", () => { - const runningLimit = { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 1 }, - maxRunning: 2, - }; - const asking = (requesterId: string, more: Record = {}) => ({ - ownerId: requesterId, - requesterId, - ...more, - }); - const listing = (harness: Awaited>) => - harness.engine.waitingRequests().map((entry) => ({ - reason: (entry as { reason?: string }).reason, - requesterId: entry.requesterId, - stage: entry.stage, - })); - - it("lists a queued request at the iOS running limit as platform-running-limit and one behind it as behind-earlier-request, and stores no reason", async () => { - const harness = await createHarness({ limits: runningLimit }); - await harness.engine.request(request, asking("holder")); - void harness.engine.request(request, asking("head")); - void harness.engine.request(request, asking("behind")); - await flush(); - - expect(listing(harness)).toEqual([ - { reason: "platform-running-limit", requesterId: "head", stage: "queued" }, - { reason: "behind-earlier-request", requesterId: "behind", stage: "queued" }, - ]); - const stored = harness.registry.leaseRequests().find((record) => record.requesterId === "head"); - expect(stored).toBeDefined(); - expect(stored).not.toHaveProperty("reason"); - }); - - it("follows the latest attempt: when the head is granted, the request behind it lists its own new reason", async () => { - const harness = await createHarness({ limits: runningLimit }); - const holder = await harness.engine.request(request, asking("holder")); - const head = harness.engine.request(request, asking("head")); - void harness.engine.request(request, asking("behind")); - await flush(); - - await harness.engine.release(holder.lease.id, "explicit"); - await head; - await harness.engine.settle(); - - expect(listing(harness)).toEqual([ - { reason: "platform-running-limit", requesterId: "behind", stage: "queued" }, - ]); - }); - - it("cancelling the queued head makes the request behind it plan again and list its own reason", async () => { - const harness = await createHarness({ limits: runningLimit }); - await harness.engine.request(request, asking("holder")); - const head = harness.engine.request(request, asking("head")); - void harness.engine.request(request, asking("behind")); - await flush(); - - await expect(harness.engine.cancelPending("head")).resolves.toBe("cancelled"); - await expect(head).rejects.toBeInstanceOf(RequestCancelledError); - await flush(); - - expect(listing(harness)).toEqual([ - { reason: "platform-running-limit", requesterId: "behind", stage: "queued" }, - ]); - }); - - it("plans again at once after a first provisioning failure, and a success then grants", async () => { - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - driver.failOn("provision", 1, new DriverCrashError("simulator exited")); - const harness = await createHarness({ driver }); - - const granted = await harness.engine.request(request, asking("retried")); - - expect(granted.lease.id).toBeDefined(); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(2); - expect(listing(harness)).toEqual([]); - }); - - it("rejects a request whose provision failed twice with PROVISION_FAILED, lists nothing for it, and the request behind it plans again", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { provision: 50 }, - platform: "ios", - }); - driver.failOn("provision", 1, new DriverCrashError("simulator exited")); - driver.failOn("provision", 2, new DriverCrashError("simulator exited again")); - const harness = await createHarness({ driver }); - const rejected = harness.engine - .request(request, asking("failing")) - .catch((error: unknown) => error); - await flush(); - clock.advance(50); - await flush(); - void harness.engine.request(request, asking("meanwhile")); - await flush(); - expect(listing(harness)).toEqual([ - { reason: undefined, requesterId: "failing", stage: "starting" }, - { reason: "device-limit", requesterId: "meanwhile", stage: "queued" }, - ]); - expect(harness.engine.waitingRequests()[0]).not.toHaveProperty("reason"); - - clock.advance(50); - await flush(); - - expect(await rejected).toBeInstanceOf(ProvisionFailedError); - expect(await rejected).toMatchObject({ message: "simulator exited again" }); - expect(listing(harness)).toEqual([ - { reason: undefined, requesterId: "meanwhile", stage: "starting" }, - ]); - }); - - it("refuses a queued request when the update hold begins and still grants the one that is provisioning", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { provision: 50 }, - platform: "ios", - }); - const harness = await createHarness({ driver, limits: runningLimit }); - const provisioning = harness.engine - .request(request, asking("provisioning")) - .catch((error: unknown) => error); - await flush(); - const queued = harness.engine - .request(request, asking("queued")) - .catch((error: unknown) => error); - await flush(); - expect(listing(harness)).toEqual([ - { reason: undefined, requesterId: "provisioning", stage: "starting" }, - { reason: "platform-running-limit", requesterId: "queued", stage: "queued" }, - ]); - - await harness.engine.leasing.holdForUpdate(); - clock.advance(50); - await flush(); - - expect(await queued).toBeInstanceOf(DaemonUpdatingError); - expect(await provisioning, "the request that was provisioning").toMatchObject({ - lease: { id: expect.any(String) }, - }); - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload as { requester: string; reason: string }) - .map(({ requester, reason }) => [requester, reason]), - ).toEqual([["queued", "updating"]]); - }); - - it("refuses the retry of a provision that fails while the update hold is on, with DAEMON_UPDATING and no second provision", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { provision: 50 }, - platform: "ios", - }); - driver.failOn("provision", 1, new DriverCrashError("simulator exited")); - const harness = await createHarness({ driver, limits: runningLimit }); - const failing = harness.engine - .request(request, asking("failing")) - .catch((error: unknown) => error); - await flush(); - - await harness.engine.leasing.holdForUpdate(); - clock.advance(50); - await flush(); - clock.advance(50); - await flush(); - - expect(await failing).toBeInstanceOf(DaemonUpdatingError); - expect(driver.calls.filter((call) => call.operation === "provision")).toHaveLength(1); - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload as { requester: string; reason: string }) - .map(({ requester, reason }) => [requester, reason]), - ).toEqual([["failing", "updating"]]); - }); - - it("refuses a request that is still resolving its spec when the update hold begins", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { resolveSpec: 50 }, - platform: "ios", - }); - const harness = await createHarness({ driver, limits: runningLimit }); - const resolving = harness.engine - .request(request, asking("resolving")) - .catch((error: unknown) => error); - await flush(); - - await harness.engine.leasing.holdForUpdate(); - clock.advance(50); - await flush(); - - expect(await resolving, "the request that was resolving").toBeInstanceOf(DaemonUpdatingError); - expect(driver.calls.map((call) => call.operation)).not.toContain("provision"); - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.rejected") - .map((event) => event.payload as { requester: string; reason: string }) - .map(({ requester, reason }) => [requester, reason]), - ).toEqual([["resolving", "updating"]]); - }); - - it("lists a request that waits for a device the warm pool is booting as waiting-for-device", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - knownModels: ["iPhone 16"], - availableOsVersions: ["26.5"], - clock, - latencyMs: { makeReady: 50 }, - platform: "ios", - }); - const harness = await createHarness({ - startPool: true, - driver, - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 2, maxRunning: 2 }, - maxRunning: 2, - }, - }); - const granting = harness.engine.request(request, asking("a")); - await flush(); - clock.advance(50); - const first = await granting; - await harness.engine.release(first.lease.id, "explicit"); - await vi.waitFor(() => - expect(driver.calls.filter((call) => call.operation === "makeReady")).toHaveLength(2), - ); - - void harness.engine.request(request, asking("b")); - await flush(); - - expect(listing(harness)).toEqual([ - { reason: "waiting-for-device", requesterId: "b", stage: "queued" }, - ]); - }); - - it("lists a request whose eviction failed as the capacity reason that asked for it", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("shutdown", 1, new DriverCrashError("cannot stop victim")); - const harness = await createHarness({ driver, limits: runningLimit }); - await seedReady(harness); - - void harness.engine.request({ ...request, model: "iPhone SE" }, asking("new-spec")); - await flush(); - - expect(listing(harness)).toEqual([ - { reason: "platform-running-limit", requesterId: "new-spec", stage: "queued" }, - ]); - }); - - it("lists a request whose only match is being destroyed as the capacity reason its plan gave", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { destroy: 20 }, - platform: "ios", - }); - const harness = await createHarness({ driver }); - const first = await harness.engine.request(request, asking("first")); - await harness.engine.release(first.lease.id, "explicit"); - - void harness.engine.request(request, asking("second")); - await flush(); - - expect(listing(harness)).toEqual([ - { reason: "device-limit", requesterId: "second", stage: "queued" }, - ]); - }); -}); - -describe("createLeasing: a baseline is a pool key", () => { - type Harness = Awaited>; - const plain = { ...request, physical: false } as const; - const copyOf = (name: string, version: number) => - ({ ...plain, baseline: { name, version } }) as const; - const named = { baseline: "checkout", platform: "ios" } as const; - - // why: the engine reads a baseline from `baselines` and copies the source the driver holds, so a row and a saved source stand in for an operator's save. - async function saveBaseline(harness: Harness, name: string, version: number): Promise { - const driverDevice = await harness.driver.provision(plain); - const ready = await harness.driver.makeReady(driverDevice, { - mode: "full", - purpose: "prepare", - }); - const saved = await harness.driver.saveSource({ ...ready, physical: false }); - harness.state.database - .prepare( - `INSERT INTO baselines - (name, version, platform, model, os_version, mode, saved_at, source_id) - VALUES (?, ?, 'ios', 'iPhone 16', '26.5', 'full', '2026-11-03T00:00:00.000Z', ?)`, - ) - .run(name, version, saved.sourceId); - } - - it.each([ - { - label: "an idle baseline device and a plain request", - idle: copyOf("checkout", 1), - ask: request, - granted: undefined, - reused: false, - }, - { - label: "an idle plain device and a request naming a baseline", - idle: plain, - ask: named, - granted: { name: "checkout", version: 2 }, - reused: false, - }, - { - label: "an idle device of another baseline", - idle: copyOf("login", 2), - ask: named, - granted: { name: "checkout", version: 2 }, - reused: false, - }, - { - label: "an idle device of an older version", - idle: copyOf("checkout", 1), - ask: named, - granted: { name: "checkout", version: 2 }, - reused: false, - }, - { - label: "an idle device of the baseline's current version (control)", - idle: copyOf("checkout", 2), - ask: named, - granted: { name: "checkout", version: 2 }, - reused: true, - }, - ])("never grants a device across pools: $label", async ({ ask, granted, idle, reused }) => { - const harness = await createHarness(); - await saveBaseline(harness, "checkout", 1); - await saveBaseline(harness, "checkout", 2); - await saveBaseline(harness, "login", 1); - const device = await seedReady(harness, idle); - - const grant = await harness.engine.request(ask, { ownerId: "o", requesterId: "r" }); - - expect(grant.device.id === device.id).toBe(reused); - expect(grant.device.spec.baseline).toEqual(granted); - }); }); diff --git a/src/leasing/test-create-leasing.ts b/src/leasing/test-create-leasing.ts new file mode 100644 index 000000000..f48c32262 --- /dev/null +++ b/src/leasing/test-create-leasing.ts @@ -0,0 +1,236 @@ +import { + type ModelPreferences, + type ComponentInstaller, + type Config, + type DeviceMode, + type VirtualSpec, +} from "../core/index.js"; +import { type CapacityLimits, type ResourceStrategyOptions, testStateOf } from "../core/testing.js"; +import { + FakeDriver, + loadTestRegistry, + openTestState, + testComponentWiring, +} from "../core/testing.js"; +import { type StateDatabase } from "../state/index.js"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, FakeSystemStats, type Logger } from "../ports/index.js"; +import { type LeaseEndParticipant } from "./lease-ports.js"; +import { createTestEngine, type TestEngineConfig } from "./testing.js"; + +export const gibibyte = 1024 ** 3; +export const request = { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", +} as const; + +export const androidRequest = { + model: "Pixel 9", + osVersion: "36", + platform: "android", +} as const; + +export function config( + overrides: Partial = {}, + warmPoolEnabled = true, + idleShutdownAfterMs = 10_000, + warmPool: Partial = {}, +): TestEngineConfig { + return { + devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, + health: { + enabled: true, + maxConcurrentRecoveries: 1, + maxRecoveryAttempts: 3, + probeIntervalMs: 30_000, + recoveryBackoffMs: 5_000, + stableObservations: 2, + }, + stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + idle: { deleteAfterMs: 60_000, shutdownAfterMs: idleShutdownAfterMs }, + lease: { + defaultTtlMs: 100, + maxTtlMs: 14_400_000, + identity: { ios: "reusable", android: "reusable" }, + requestRetentionMs: 600_000, + endedIdRetentionMs: 600_000, + maxRequestRecords: 10_000, + maxPerRequester: 1, + endWaitMs: 600_000, + ...overrides, + }, + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 1 + 1, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + warmPool: { + enabled: warmPoolEnabled, + maxConcurrentBoots: 1, + reserveRunning: { android: 0, ios: 0 }, + targets: [], + quarantine: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + ...warmPool, + }, + }; +} + +function resourceOptions(source: Pick): ResourceStrategyOptions { + if (source.capacity.strategy !== "resource") throw new Error("expected the resource strategy"); + return source.capacity.config; +} + +function withCapacity( + base: ResourceStrategyOptions, + overrides: Partial, +): ResourceStrategyOptions { + return { + limits: overrides.limits ?? base.limits, + ramBudget: overrides.ramBudget ?? base.ramBudget, + }; +} + +function capacityConfig( + base: Pick, + options: Partial & { readonly capacity?: Config["capacity"] }, +): Config["capacity"] { + return ( + options.capacity ?? { + strategy: "resource", + config: withCapacity(resourceOptions(base), options), + } + ); +} + +// fallow-ignore-next-line complexity -- one test harness builder; each option is a plain pass-through to a collaborator. +export async function createHarness( + options: { + readonly components?: Pick; + readonly driver?: FakeDriver; + readonly drivers?: readonly FakeDriver[]; + readonly state?: StateDatabase; + readonly identity?: Config["lease"]["identity"]; + readonly lease?: Partial; + /** why: a participant holds a released device in `reclaiming` until it returns. */ + readonly leaseEndParticipants?: readonly LeaseEndParticipant[]; + readonly capacity?: Config["capacity"]; + readonly defaultModes?: Parameters[0]["defaultModes"]; + readonly limits?: CapacityLimits; + readonly logger?: Logger; + readonly modelPreferences?: ModelPreferences; + readonly ramBudget?: ResourceStrategyOptions["ramBudget"]; + readonly totalRamBytes?: number; + readonly warmPoolEnabled?: boolean; + readonly warmPool?: Partial; + /** why: how long a ready device may idle before the pool shuts it down. */ + readonly idleShutdownAfterMs?: number; + /** why: starts the warm pool as the daemon's startup does, so it learns from the leases and reacts to events. */ + readonly startPool?: boolean; + } = {}, +) { + const clock = new FakeClock(1_000); + const state = options.state ?? (await openTestState()); + const bus = new EventBus(clock); + const driver = + options.driver ?? new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + let nextId = options.state === undefined ? 1 : 1_000; + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => `${nextId++}` }, + ...(options.identity === undefined ? {} : { leaseIdentity: options.identity }), + state, + }); + const baseConfig = config( + { + ...options.lease, + ...(options.identity === undefined ? {} : { identity: options.identity }), + }, + options.warmPoolEnabled ?? true, + options.idleShutdownAfterMs, + options.warmPool, + ); + const engineConfig: TestEngineConfig = { + ...baseConfig, + capacity: capacityConfig(baseConfig, options), + }; + const totalRamBytes = options.totalRamBytes ?? 32 * gibibyte; + const drivers = options.drivers ?? [driver]; + const wiring = testComponentWiring({ + clock, + components: options.components, + drivers, + eventBus: bus, + registry, + }); + const engine = createTestEngine({ + ...wiring, + clock, + config: engineConfig, + drivers, + ...(options.defaultModes === undefined ? {} : { defaultModes: options.defaultModes }), + eventBus: bus, + idGenerator: { generate: () => `request-${nextId++}` }, + ...(options.leaseEndParticipants === undefined + ? {} + : { leaseEndParticipants: options.leaseEndParticipants }), + ...(options.logger === undefined ? {} : { logger: options.logger }), + ...(options.modelPreferences === undefined + ? {} + : { modelPreferences: options.modelPreferences }), + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes }), + }); + + if (options.startPool === true) await engine.convergeRunningCapacity(); + return { bus, clock, decisions: wiring.decisions, driver, engine, registry, state }; +} + +export async function seedReady( + harness: Awaited>, + spec: VirtualSpec = { ...request, physical: false }, + mode: DeviceMode = "full", + driver: FakeDriver = harness.driver, +) { + const driverDevice = await driver.provision(spec); + // why: Running on the machine too: a daemon start reads that of a leased device to keep its lease. + const booted = await driver.makeReady(driverDevice, { mode, purpose: "prepare" }); + const device = await harness.registry.registerDevice({ + driverData: driverDevice.driverData, + driverDeviceId: driverDevice.deviceId, + provisionDuration: 0, + spec, + }); + return harness.registry.transitionDevice( + device.id, + "ready", + { event: "device.ready", payload: { bootDuration: 0, deviceId: device.id } }, + booted.mode === undefined ? undefined : { mode: booted.mode }, + ); +} + +export async function flush(): Promise { + for (let count = 0; count < 100; count += 1) { + await Promise.resolve(); + } +} + +export async function settledOrPending(promise: Promise): Promise { + return Promise.race([ + promise.catch((error: unknown) => error), + flush().then(() => "still pending"), + ]); +} diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index a1fda9b32..d950c8237 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -30,7 +30,6 @@ const SHRINK_ONLY: Record = { "src/cli/index.test.ts": 11674, "src/drivers/ios/index.test.ts": 6208, "src/drivers/ios/index.ts": 2700, - "src/leasing/create-leasing.test.ts": 5389, }; const CODE_LIMIT = 1400; From 6ed251f5b5509a1d217b36c712cc0835a312882d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:18:24 +0100 Subject: [PATCH 20/83] test(gateway): split fleet-requests.test.ts by topic --- src/gateway/fleet-requests-outcomes.test.ts | 457 +++++++ .../fleet-requests-requester-limit.test.ts | 273 ++++ .../fleet-requests-waiting-reasons.test.ts | 495 +++++++ src/gateway/fleet-requests.test.ts | 1175 +---------------- 4 files changed, 1226 insertions(+), 1174 deletions(-) create mode 100644 src/gateway/fleet-requests-outcomes.test.ts create mode 100644 src/gateway/fleet-requests-requester-limit.test.ts create mode 100644 src/gateway/fleet-requests-waiting-reasons.test.ts diff --git a/src/gateway/fleet-requests-outcomes.test.ts b/src/gateway/fleet-requests-outcomes.test.ts new file mode 100644 index 000000000..3d72705c2 --- /dev/null +++ b/src/gateway/fleet-requests-outcomes.test.ts @@ -0,0 +1,457 @@ +import { describe, expect, it } from "vitest"; +import type { EventBus, EventMap } from "../bus/index.js"; +import type { LeaseRecord } from "../admin/index.js"; +import { fromWireError, SimlockError } from "../contract/index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; +import type { WorkerReportedLease } from "./lease-index.js"; +import { grantFixture, leaseFixture, ScriptedWorkerClient, statusFixture } from "./test-support.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + roomierIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator records every fleet request's outcome (ADR 0021)", () => { + function oneWorker(overrides: Parameters[0] = {}) { + const fleet = buildFleet(overrides); + const client = new ScriptedWorkerClient(); + fleet.directory.add("wrk_a", client); + connectWorker(fleet.workers, "wrk_a"); + return { ...fleet, client }; + } + + function record( + eventBus: EventBus, + ...names: Name[] + ): Array<{ name: Name; payload: EventMap[Name] }> { + const seen: Array<{ name: Name; payload: EventMap[Name] }> = []; + for (const name of names) { + eventBus.subscribe(name, (envelope) => + seen.push({ name, payload: envelope.payload as EventMap[Name] }), + ); + } + return seen; + } + + async function start( + coordinator: FleetLeaseCoordinator, + overrides: Parameters[0] = {}, + ) { + let requestId: string | undefined; + const outcome = coordinator.request( + REQUEST, + requestOptions({ ...overrides, onAdmitted: (id) => (requestId = id) }), + ); + const ended = outcome.then( + (grant) => grant as unknown, + (error: unknown) => error, + ); + const state = promiseState(outcome); + await tick(); + return { ended, outcome, requestId: () => requestId, state }; + } + + it.each([ + ["a waiting caller", { noWait: false }], + ["a noWait caller", { noWait: true }], + ])("sends the gateway's request id as fleetRequestId, for %s", async (_label, caller) => { + const { client, coordinator } = oneWorker(); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { requestId } = await start(coordinator, caller); + + expect(requestId()).toMatch(/^req_/); + expect(client.lastRequestLeaseInput).toMatchObject({ + fleetRequestId: requestId(), + noWait: true, + }); + }); + + function twoWorkersAPreferred() { + const fleet = buildFleet(); + const preferred = new ScriptedWorkerClient(); + const other = new ScriptedWorkerClient(); + fleet.directory.add("wrk_a", preferred); + fleet.directory.add("wrk_b", other); + const capacity = statusFixture().capacity; + connectWorker(fleet.workers, "wrk_a", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, + }); + connectWorker(fleet.workers, "wrk_b"); + return { ...fleet, other, preferred }; + } + + it("sends the same fleetRequestId to the second worker when the first refuses NO_CAPACITY", async () => { + const { coordinator, other, preferred } = twoWorkersAPreferred(); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { requestId } = await start(coordinator, { noWait: true }); + + expect( + [preferred, other].map( + (client) => + (client.lastRequestLeaseInput as Record | undefined)?.fleetRequestId, + ), + ).toEqual([requestId(), requestId()]); + }); + + it("emits one lease.rejected worker-failed with the caller's error code for a request whose dispatch fails after a progress push", async () => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "lease.rejected", "request.granted"); + client.requestLeaseQueue.push({ + error: new SimlockError("INTERNAL", "domain", "the worker broke", {}), + kind: "error", + progress: [{ etaMs: 1_000, stage: "booting" }], + }); + + const { ended, requestId } = await start(coordinator); + + expect(await ended).toMatchObject({ code: "INTERNAL" }); + expect(events).toEqual([ + { + name: "lease.rejected", + payload: { + code: "INTERNAL", + reason: "worker-failed", + requestId: requestId(), + requestSpec: REQUEST, + requester: "agent-1", + worker: "wrk_a", + }, + }, + ]); + }); + + it.each([{ answer: null }, { answer: {} }, { answer: { lease: null } }])( + "ends a request whose worker answers a malformed grant ($answer) in lease.rejected worker-failed, with no unhandled rejection", + async ({ answer }) => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "lease.rejected"); + client.requestLeaseQueue.push({ grant: answer as never, kind: "grant" }); + const unhandled: unknown[] = []; + const listener = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", listener); + try { + const { ended, state } = await start(coordinator); + await new Promise((resolve) => setImmediate(resolve)); + expect(state.state).toBe("rejected"); + await expect(ended).resolves.toMatchObject({ code: "INTERNAL" }); + } finally { + process.off("unhandledRejection", listener); + } + + expect(unhandled).toEqual([]); + expect(events).toEqual([ + { name: "lease.rejected", payload: expect.objectContaining({ reason: "worker-failed" }) }, + ]); + }, + ); + + it.each([ + { + elapseMs: 0, + failure: "fails on a lost uplink", + script: (client: ScriptedWorkerClient) => + client.requestLeaseQueue.push({ + error: new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + "Daemon connection is closed", + {}, + ), + kind: "error", + }), + }, + { + elapseMs: 5_000, + failure: "times out", + script: (client: ScriptedWorkerClient) => client.requestLeaseQueue.push({ kind: "hang" }), + }, + ])( + "emits one lease.rejected worker-failed with code WORKER_UNREACHABLE, naming the worker, for a dispatch that $failure", + async ({ elapseMs, script }) => { + const { clock, client, coordinator, eventBus } = oneWorker({ leaseRequestTimeoutMs: 5_000 }); + const events = record(eventBus, "lease.rejected"); + script(client); + + const { ended } = await start(coordinator); + clock.advance(elapseMs); + await tick(); + + expect(await ended).toMatchObject({ code: "WORKER_UNREACHABLE" }); + expect(events).toEqual([ + { + name: "lease.rejected", + payload: expect.objectContaining({ + code: "WORKER_UNREACHABLE", + reason: "worker-failed", + worker: "wrk_a", + }), + }, + ]); + }, + ); + + it("carries the failing worker's id, not the one that only refused, as worker on a worker-failed rejection", async () => { + const { coordinator, eventBus, other } = twoWorkersAPreferred(); + const events = record(eventBus, "lease.rejected"); + other.requestLeaseQueue.push({ + error: new SimlockError("INTERNAL", "domain", "wrk_b broke", {}), + kind: "error", + progress: [{ etaMs: 1, stage: "booting" }], + }); + + await start(coordinator, { noWait: true }); + + expect(events).toEqual([ + { + name: "lease.rejected", + payload: expect.objectContaining({ reason: "worker-failed", worker: "wrk_b" }), + }, + ]); + }); + + it("ends a request never queued on a worker's cannot-serve refusal in one lease.rejected unresolvable-spec", async () => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "lease.rejected", "lease.queued"); + client.requestLeaseQueue.push({ + error: new SimlockError("RUNTIME_MISSING", "domain", "no such runtime", { + downloadable: false, + osVersion: "26.0", + platform: "ios", + }), + kind: "error", + }); + + const { ended, requestId } = await start(coordinator); + + expect(await ended).toMatchObject({ code: "RUNTIME_MISSING" }); + expect(events).toEqual([ + { + name: "lease.rejected", + payload: expect.objectContaining({ reason: "unresolvable-spec", requestId: requestId() }), + }, + ]); + }); + + it("emits no lease.rejected for a request one worker refuses NO_CAPACITY and another grants", async () => { + const { coordinator, eventBus, other } = twoWorkersAPreferred(); + const events = record(eventBus, "lease.rejected"); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { ended } = await start(coordinator, { noWait: true }); + + expect(await ended).toMatchObject({ lease: { worker: { id: "wrk_b" } } }); + expect(events).toEqual([]); + }); + + it("emits one request.granted with worker and both lease ids, and no workerId in its payload, for a settled grant", async () => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "request.granted", "lease.rejected"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { ended, requestId } = await start(coordinator); + + expect(await ended).toMatchObject({ lease: { id: "wrk_a.lse_1" } }); + expect(events).toEqual([ + { + name: "request.granted", + payload: { + leaseId: "wrk_a.lse_1", + requestId: requestId(), + worker: "wrk_a", + workerLeaseId: "lse_1", + }, + }, + ]); + expect(events[0]?.payload).not.toHaveProperty("workerId"); + }); + + it("emits lease.rejected lease-id-taken and no request.granted for a grant the lease index refuses", async () => { + const { client, coordinator, eventBus, leaseIndex } = oneWorker(); + const events = record(eventBus, "request.granted", "lease.rejected"); + client.requestLeaseQueue.push({ + beforeGrant: () => { + leaseIndex.rebuildFromWorker("wrk_b", [ + { + grantedAt: 1, + id: "myid", + idChosenByRequester: true, + ownerId: "agent-9", + requesterId: `${GATEWAY_PREFIX}agent-9`, + } as WorkerReportedLease, + ]); + }, + grant: grantFixture({ + lease: { ...grantFixture().lease, id: "myid", idChosenByRequester: true } as LeaseRecord, + }), + kind: "grant", + }); + + const { ended } = await start(coordinator, { leaseId: "myid" } as never); + + expect(await ended).toMatchObject({ code: "LEASE_ID_TAKEN" }); + expect(events).toEqual([ + { name: "lease.rejected", payload: expect.objectContaining({ reason: "lease-id-taken" }) }, + ]); + }); + + it("emits no request.granted for a grant that lands after dispose()", async () => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "request.granted", "lease.rejected"); + client.requestLeaseQueue.push({ + beforeGrant: () => coordinator.dispose(), + grant: grantFixture(), + kind: "grant", + }); + + const { ended } = await start(coordinator); + + expect(await ended).toMatchObject({ code: "DAEMON_STOPPING" }); + expect(events).toEqual([]); + }); + + it("emits no request.granted for a grant that lands after the dispatch timed out", async () => { + const { clock, client, coordinator, eventBus } = oneWorker({ leaseRequestTimeoutMs: 5_000 }); + const events = record(eventBus, "request.granted"); + client.requestLeaseQueue.push({ + beforeGrant: async () => { + await Promise.resolve(); + clock.advance(5_000); + }, + grant: grantFixture(), + kind: "grant", + }); + + const { ended } = await start(coordinator); + + expect(await ended).toMatchObject({ code: "WORKER_UNREACHABLE" }); + expect(events).toEqual([]); + }); + + it("emits no lease.rejected and no request.granted on dispose() with a queued and an in-flight request", async () => { + const { client, coordinator, directory, eventBus, workers } = oneWorker(); + const events = record(eventBus, "lease.rejected", "request.granted"); + client.requestLeaseQueue.push({ kind: "hang" }); + const inFlight = await start(coordinator); + const busy = new ScriptedWorkerClient(); + directory.add("wrk_b", busy); + connectWorker(workers, "wrk_b", { capacity: saturatedIos(), models: ["iPhone 18"] }); + const queued = await start(coordinator, { ownerId: "agent-2", requesterId: "agent-2" }); + expect(inFlight.state.state).toBe("pending"); + expect(coordinator.queueDepth).toBe(1); + + coordinator.dispose(); + await tick(); + + expect(await inFlight.ended).toMatchObject({ code: "DAEMON_STOPPING" }); + expect(await queued.ended).toMatchObject({ code: "DAEMON_STOPPING" }); + expect(events).toEqual([]); + }); + + it("emits no request.granted for a grant given back for a mismatched leaseId, and one for the retried grant", async () => { + const { client, coordinator, eventBus, workers } = oneWorker(); + const events = record(eventBus, "request.granted"); + const chosen = (leaseId: string) => + grantFixture({ + lease: { ...grantFixture().lease, id: leaseId, idChosenByRequester: true } as LeaseRecord, + }); + client.requestLeaseQueue.push( + { grant: chosen("not-what-was-sent"), kind: "grant" }, + { grant: chosen("myid"), kind: "grant" }, + ); + + const { ended, requestId } = await start(coordinator, { leaseId: "myid" } as never); + expect(events).toEqual([]); + + workers.refresh("wrk_a", { capacity: roomierIos() }); + await tick(); + + expect(await ended).toMatchObject({ lease: { id: "myid" } }); + expect(events).toEqual([ + { + name: "request.granted", + payload: { + leaseId: "myid", + requestId: requestId(), + worker: "wrk_a", + workerLeaseId: "myid", + }, + }, + ]); + }); + + it.each([ + [ + "REQUESTER_ALREADY_LEASED", + fromWireError("REQUESTER_ALREADY_LEASED", "already holds a lease", { requesterId: "gw" }), + ], + [ + "LEASE_ID_TAKEN", + fromWireError("LEASE_ID_TAKEN", "lease ID myid is already in use", { leaseId: "myid" }), + ], + ])( + "ends a request a worker refuses %s before any progress in lease.rejected worker-failed with that code, not the gateway's own lease-id-taken", + async (code, error) => { + const { client, coordinator, eventBus } = oneWorker(); + const events = record(eventBus, "lease.rejected"); + client.requestLeaseQueue.push({ error, kind: "error" }); + + const { ended } = await start(coordinator); + + expect(await ended).toMatchObject({ code }); + expect(events).toEqual([ + { + name: "lease.rejected", + payload: expect.objectContaining({ code, reason: "worker-failed", worker: "wrk_a" }), + }, + ]); + }, + ); +}); + +describe("FleetLeaseCoordinator: a lease.cancel sent right after its lease.request (#1492)", () => { + it.each([ + { fleetHas: "no room, so the request would queue", room: false }, + { fleetHas: "a free worker, so the request would be dispatched", room: true }, + ])( + "with $fleetHas, answers cancelled, rejects the request as cancelled, and leaves nothing waiting and nothing dispatched", + async ({ room }) => { + const fleet = buildFleet(); + const client = new ScriptedWorkerClient(); + fleet.directory.add("wrk_a", client); + connectWorker(fleet.workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + await fleet.coordinator.request(REQUEST, requestOptions()); + connectWorker(fleet.workers, "wrk_a", { capacity: room ? roomierIos() : saturatedIos() }); + client.requestLeaseQueue.push({ + grant: grantFixture({ lease: leaseFixture("lease-2", "dev_2") }), + kind: "grant", + }); + + const request = fleet.coordinator + .request(REQUEST, requestOptions({ ownerId: "agent-2", requesterId: "agent-2" })) + .then( + () => "granted", + (error: unknown) => error, + ); + const cancel = fleet.coordinator.cancelPending("agent-2"); + + await expect(cancel).resolves.toBe("cancelled"); + await expect(request).resolves.toMatchObject({ name: "RequestCancelledError" }); + await tick(); + expect(fleet.coordinator.waitingRequests()).toEqual([]); + expect( + client.calls.filter((call) => call === `lease.request:${GATEWAY_PREFIX}agent-2`), + ).toEqual([]); + }, + ); +}); diff --git a/src/gateway/fleet-requests-requester-limit.test.ts b/src/gateway/fleet-requests-requester-limit.test.ts new file mode 100644 index 000000000..8587cdacf --- /dev/null +++ b/src/gateway/fleet-requests-requester-limit.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it } from "vitest"; +import type { LeaseRecord } from "../admin/index.js"; +import { + deviceFixture, + grantFixture, + leaseFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator: a requester may hold up to lease.maxPerRequester fleet leases (ADR 0023)", () => { + const held = (id: string, grantedAt: number, requester = "agent-1") => ({ + grantedAt, + id, + ownerId: requester, + requesterId: `${GATEWAY_PREFIX}${requester}`, + }); + const atLimit = (held: number, limit: number, leaseId: string) => + `Requester agent-1 already holds ${String(held)} leases and lease.maxPerRequester is ${String(limit)}; release one (\`simlock release ${leaseId}\`) before requesting another device`; + const waitingMessage = + "Requester agent-1 already has a waiting request; wait for it to be granted or cancel it before requesting another device"; + + async function withWaitingRequest(maxPerRequester?: number) { + const built = buildFleet(maxPerRequester === undefined ? {} : { maxPerRequester }); + connectWorker(built.workers, "wrk_a", { capacity: saturatedIos() }); + built.directory.add("wrk_a", new ScriptedWorkerClient()); + void built.coordinator.request(REQUEST, requestOptions()).catch(() => undefined); + await tick(); + expect(built.coordinator.queueDepth).toBe(1); + return built; + } + + it("with no limit set, a requester holding a fleet lease is refused with today's message, naming that lease", async () => { + const { coordinator, leaseIndex } = buildFleet(); + leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: "wrk_a.lse_1", + message: + "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with limit 2, a requester holding two fleet leases is refused, naming the earlier-granted one", async () => { + const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: 2 }); + leaseIndex.rebuildFromWorker("wrk_a", [held("lse_9", 10), held("lse_1", 20)]); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: "wrk_a.lse_9", + message: atLimit(2, 2, "wrk_a.lse_9"), + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with limit 2, a requester holding one fleet lease is not refused", async () => { + const { coordinator, directory, leaseIndex, workers } = buildFleet({ maxPerRequester: 2 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); + + await expect(coordinator.request(REQUEST, requestOptions())).resolves.toMatchObject({ + lease: expect.anything(), + }); + }); + + describe("per platform", () => { + const ANDROID_REQUEST = { + physical: false, + model: "Pixel 9", + osVersion: "26.0", + platform: "android" as const, + }; + const iosHeld = (): LeaseRecord => ({ + ...(leaseFixture("lse_1", "dev_1") as LeaseRecord), + requesterId: `${GATEWAY_PREFIX}agent-1`, + }); + const androidGrant = () => { + const base = grantFixture(); + return grantFixture({ + device: { + ...base.device, + id: "dev_2", + spec: { ...base.device.spec, model: "Pixel 9", platform: "android" }, + }, + lease: leaseFixture("lse_2", "dev_2") as LeaseRecord, + }); + }; + const iosRefusal = (leaseId: string) => + `Requester agent-1 already holds iOS lease ${leaseId}; release it (\`simlock release ${leaseId}\`) before requesting another iOS device`; + const twoWorkers = (built: ReturnType, held: LeaseRecord[] = []) => { + const ios = new ScriptedWorkerClient(); + const android = new ScriptedWorkerClient(); + built.directory.add("wrk_a", ios); + built.directory.add("wrk_b", android); + connectWorker(built.workers, "wrk_a", { + devices: held.length === 0 ? [] : [deviceFixture("dev_1", "leased")], + leases: held, + }); + connectWorker(built.workers, "wrk_b", { models: ["Pixel 9"], platform: "android" }); + return { android, ios }; + }; + + it("with lease.maxPerRequester per platform, a requester holding an iOS fleet lease is granted an Android one on another worker and refused a second iOS one, also once the index is rebuilt from worker views alone", async () => { + const limit = { ios: 1, android: 1 }; + const live = buildFleet({ maxPerRequester: limit }); + const liveWorkers = twoWorkers(live); + liveWorkers.ios.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + liveWorkers.android.requestLeaseQueue.push({ grant: androidGrant(), kind: "grant" }); + await live.coordinator.request(REQUEST, requestOptions()); + + const android = await live.coordinator.request(ANDROID_REQUEST, requestOptions()); + const second = await live.coordinator + .request(REQUEST, requestOptions()) + .catch((e: unknown) => e); + + expect(android.lease.id).toBe("wrk_b.lse_2"); + expect(second).toMatchObject({ + existingLeaseId: "wrk_a.lse_1", + message: iosRefusal("wrk_a.lse_1"), + name: "RequesterAlreadyLeasedError", + }); + + const rebuilt = buildFleet({ maxPerRequester: limit }); + const rebuiltWorkers = twoWorkers(rebuilt, [iosHeld()]); + rebuiltWorkers.android.requestLeaseQueue.push({ grant: androidGrant(), kind: "grant" }); + + const againIos = await rebuilt.coordinator + .request(REQUEST, requestOptions()) + .catch((e: unknown) => e); + const againAndroid = await rebuilt.coordinator.request(ANDROID_REQUEST, requestOptions()); + + expect(againIos).toMatchObject({ message: iosRefusal("wrk_a.lse_1") }); + expect(againAndroid.lease.id).toBe("wrk_b.lse_2"); + }); + }); + + it("with a numeric limit of 1, a requester holding an iOS and an Android fleet lease is refused with today's message, naming the earlier one", async () => { + const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: 1 }); + leaseIndex.rebuildFromWorker( + "wrk_a", + [ + { ...held("lse_2", 20), deviceId: "dev_2" }, + { ...held("lse_1", 10), deviceId: "dev_1" }, + ], + [ + { id: "dev_1", spec: { platform: "ios" } }, + { id: "dev_2", spec: { platform: "android" } }, + ], + ); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: + "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with lease.maxPerRequester per platform, a fleet lease whose device a worker view does not list counts against every platform", async () => { + const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: { ios: 1, android: 1 } }); + leaseIndex.rebuildFromWorker("wrk_a", [{ ...held("lse_1", 10), deviceId: "dev_unlisted" }], []); + + const refusal = await coordinator + .request({ ...REQUEST, model: "Pixel 9", platform: "android" }, requestOptions()) + .catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: + "Requester agent-1 already holds Android lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another Android device", + name: "RequesterAlreadyLeasedError", + }); + }); + + describe("per platform, waiting", () => { + it("with lease.maxPerRequester per platform, a requester may wait for an iOS and an Android lease at once but not two of one platform, and lease.cancel names which one to cancel", async () => { + const built = buildFleet({ maxPerRequester: { ios: 1, android: 1 } }); + const capacity = statusFixture().capacity; + built.directory.add("wrk_a", new ScriptedWorkerClient()); + built.directory.add("wrk_b", new ScriptedWorkerClient()); + connectWorker(built.workers, "wrk_a", { capacity: saturatedIos() }); + connectWorker(built.workers, "wrk_b", { + capacity: { ...capacity, android: { ...capacity.android, maxRunning: 0 } }, + models: ["Pixel 9"], + platform: "android", + }); + const androidRequest = { + physical: false, + model: "Pixel 9", + osVersion: "26.0", + platform: "android" as const, + }; + void built.coordinator.request(REQUEST, requestOptions()).catch(() => undefined); + await tick(); + + const android = built.coordinator.request(androidRequest, requestOptions()); + void android.catch(() => undefined); + await tick(); + const secondIos = await Promise.race([ + built.coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e), + tick().then(() => "admitted"), + ]); + const platformsWaiting = () => + built.coordinator + .waitingRequests() + .map((waiting) => waiting.spec.platform) + .sort(); + + expect(platformsWaiting()).toEqual(["android", "ios"]); + expect(secondIos).toMatchObject({ + message: + "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", + name: "RequesterAlreadyLeasedError", + }); + await expect(built.coordinator.cancelPending("agent-1", "android")).resolves.toBe( + "cancelled", + ); + expect(platformsWaiting()).toEqual(["ios"]); + }); + }); + + it("with limit 2, a requester with a waiting fleet request is refused with the waiting-request message", async () => { + const { coordinator } = await withWaitingRequest(2); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: waitingMessage, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with limit 2, a requester with a waiting fleet request that also holds two fleet leases gets the waiting-request message", async () => { + const { coordinator, leaseIndex } = await withWaitingRequest(2); + leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1), held("lse_2", 2)]); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + message: waitingMessage, + name: "RequesterAlreadyLeasedError", + }); + }); + + it("with no limit set, a requester that holds a fleet lease and has a waiting request gets today's message naming that lease", async () => { + const { coordinator, leaseIndex } = await withWaitingRequest(); + leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); + + const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); + + expect(refusal).toMatchObject({ + existingLeaseId: "wrk_a.lse_1", + message: + "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", + }); + }); +}); diff --git a/src/gateway/fleet-requests-waiting-reasons.test.ts b/src/gateway/fleet-requests-waiting-reasons.test.ts new file mode 100644 index 000000000..b320365b9 --- /dev/null +++ b/src/gateway/fleet-requests-waiting-reasons.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import type { LeaseRecord } from "../admin/index.js"; +import { SimlockError } from "../contract/index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; +import { + grantFixture, + physicalDeviceFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + roomierIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator lists why each worker cannot take a waiting request (#637)", () => { + type Reasons = readonly { workerId: string; reason: string; capacityReason?: string }[]; + + // why: holds a `lease.request` until `release` runs, so a test can change the fleet while the gateway's probe is in flight. + class GatedClient extends ScriptedWorkerClient { + release: () => void = () => {}; + readonly #gate = new Promise((resolve) => { + this.release = resolve; + }); + + override async requestLease( + input: Parameters[0], + options?: Parameters[1], + ) { + await this.#gate; + return super.requestLease(input, options); + } + } + + function atRamBudget(): ReturnType["capacity"] { + const capacity = statusFixture().capacity; + return { ...capacity, ios: { ...capacity.ios, atRamBudget: true } }; + } + + function noCapacity(details?: unknown) { + return { + error: new SimlockError("NO_CAPACITY", "domain", "full", details as never), + kind: "error" as const, + }; + } + + function fleet( + ids: readonly string[], + setup: (id: string) => Parameters[2] = () => ({}), + gated: readonly string[] = [], + ) { + const fleetHarness = buildFleet(); + const clients: Record = {}; + for (const id of ids) { + const client = gated.includes(id) ? new GatedClient() : new ScriptedWorkerClient(); + clients[id] = client; + fleetHarness.directory.add(id, client); + connectWorker(fleetHarness.workers, id, setup(id)); + } + return { clients, ...fleetHarness }; + } + + async function ask( + coordinator: FleetLeaseCoordinator, + request: Parameters[0] = REQUEST, + options: Parameters[0] = {}, + ) { + const outcome = coordinator.request(request, requestOptions(options)); + const ended = outcome.catch((error: unknown) => error); + const state = promiseState(outcome); + await tick(); + return { ended, outcome, state }; + } + + const listed = (coordinator: FleetLeaseCoordinator, index = 0): Reasons | undefined => + (coordinator.waitingRequests()[index] as { workers?: Reasons } | undefined)?.workers; + + const saturatedExcept = (free: string) => (id: string) => + id === free ? {} : { capacity: saturatedIos() }; + + it("lists, in worker id order, drained for a drained worker and full with platform-running-limit for one with no free iOS slot, also when it has no free global slot either", async () => { + const capacity = statusFixture().capacity; + const bothFull = { + ...saturatedIos(), + global: { ...capacity.global, maxRunning: 0 }, + }; + const { coordinator, workers } = fleet(["wrk_c", "wrk_b", "wrk_a"], (id) => + id === "wrk_b" ? { capacity: saturatedIos() } : id === "wrk_c" ? { capacity: bothFull } : {}, + ); + await workers.setDrained("wrk_a", true); + + await ask(coordinator); + + expect(listed(coordinator)).toEqual([ + { reason: "drained", workerId: "wrk_a" }, + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_c" }, + ]); + }); + + it("lists full with global-running-limit for a worker with a free iOS slot and no free global slot", async () => { + const capacity = statusFixture().capacity; + const { coordinator } = fleet(["wrk_a", "wrk_b"], (id) => + id === "wrk_a" + ? { capacity: { ...capacity, global: { ...capacity.global, maxRunning: 0 } } } + : { capacity: saturatedIos() }, + ); + + await ask(coordinator); + + expect(listed(coordinator)?.[0]).toEqual({ + capacityReason: "global-running-limit", + reason: "full", + workerId: "wrk_a", + }); + }); + + it("lists one entry for every known view, a disconnected and an incompatible one included, and none for a worker removed", async () => { + const { coordinator, workers } = fleet(["wrk_a", "wrk_b"], (id) => + id === "wrk_a" ? { capacity: saturatedIos() } : {}, + ); + workers.incompatible( + "wrk_c", + undefined, + { gateway: { max: 20, min: 20 }, worker: { max: 4, min: 4 } }, + "0.2.0", + ); + workers.disconnected("wrk_b"); + + await ask(coordinator); + + expect(listed(coordinator)).toEqual([ + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_a" }, + { reason: "disconnected", workerId: "wrk_b" }, + { reason: "incompatible", workerId: "wrk_c" }, + ]); + + await workers.remove("wrk_b"); + await tick(); + + expect(listed(coordinator)?.map((entry) => entry.workerId)).toEqual(["wrk_a", "wrk_c"]); + }); + + it.each([ + [ + "platform-running-limit", + { capacityReason: "platform-running-limit" }, + "platform-running-limit", + ], + ["none", {}, undefined], + ["bogus", { capacityReason: "bogus" }, undefined], + ["a number", { capacityReason: 42 }, undefined], + ])( + "lists a worker that answered NO_CAPACITY with capacityReason %s as full, with the reason only when it is one of the four", + async (_label, details, expected) => { + const { clients, coordinator } = fleet(["wrk_a"]); + clients["wrk_a"]?.requestLeaseQueue.push(noCapacity(details)); + + await ask(coordinator); + + expect(listed(coordinator)).toEqual([ + { + reason: "full", + workerId: "wrk_a", + ...(expected === undefined ? {} : { capacityReason: expected }), + }, + ]); + }, + ); + + it.each([ + ["UNKNOWN_MODEL", "unknown-model"], + ["RUNTIME_MISSING", "runtime-missing"], + ["NO_DRIVER", "no-driver"], + ["CAPABILITY_MISSING", "capability-missing"], + ] as const)("lists a worker that answered %s before progress as %s", async (code, reason) => { + const { clients, coordinator } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a")); + clients["wrk_a"]?.requestLeaseQueue.push({ + error: new SimlockError(code, "domain", "cannot serve", {} as never), + kind: "error", + }); + + await ask(coordinator); + + expect(listed(coordinator)).toEqual([ + { reason, workerId: "wrk_a" }, + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, + ]); + }); + + it("lists a worker that granted another lease id than the one asked for as plain full", async () => { + const { clients, coordinator } = fleet(["wrk_a"]); + clients["wrk_a"]?.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { + ...grantFixture().lease, + id: "not-what-was-sent", + idChosenByRequester: true, + } as LeaseRecord, + }), + kind: "grant", + }); + + await ask(coordinator, REQUEST, { leaseId: "myid" } as never); + + expect(listed(coordinator)).toEqual([{ reason: "full", workerId: "wrk_a" }]); + }); + + it("after a NO_CAPACITY re-queue and before any view change, recomputes every entry: the refused worker is full with its reason, one drained meanwhile is drained, one that lost to it is not-chosen", async () => { + const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b", "wrk_c"], () => ({}), [ + "wrk_a", + ]); + clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "device-limit" })); + + await ask(coordinator); + expect(coordinator.waitingRequests()[0]).toMatchObject({ stage: "starting" }); + await workers.setDrained("wrk_b", true); + (clients["wrk_a"] as GatedClient).release(); + await tick(); + + expect(coordinator.waitingRequests()[0]).toMatchObject({ stage: "queued" }); + expect(listed(coordinator)).toEqual([ + { capacityReason: "device-limit", reason: "full", workerId: "wrk_a" }, + { reason: "drained", workerId: "wrk_b" }, + { reason: "not-chosen", workerId: "wrk_c" }, + ]); + }); + + it("follows the latest walk: a worker drained lists drained, and once undrained and refused with no detail at its RAM budget it lists full with ram-budget", async () => { + const { coordinator, workers } = fleet(["wrk_a", "wrk_b"], (id) => + id === "wrk_a" ? { capacity: atRamBudget() } : { capacity: saturatedIos() }, + ); + await workers.setDrained("wrk_a", true); + + await ask(coordinator); + expect(listed(coordinator)?.[0]).toEqual({ reason: "drained", workerId: "wrk_a" }); + + await workers.setDrained("wrk_a", false); + await tick(); + + expect(listed(coordinator)).toEqual([ + { capacityReason: "ram-budget", reason: "full", workerId: "wrk_a" }, + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, + ]); + }); + + it("lists no workers for a starting request", async () => { + const { clients, coordinator } = fleet(["wrk_a"]); + clients["wrk_a"]?.requestLeaseQueue.push({ kind: "hang" }); + + await ask(coordinator); + + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "starting" })]); + expect(coordinator.waitingRequests()[0]).not.toHaveProperty("workers"); + }); + + it("lists a worker drained after it refused the request as drained, not as its remembered refusal", async () => { + const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a")); + clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "ram-budget" })); + await ask(coordinator); + expect(listed(coordinator)?.[0]).toEqual({ + capacityReason: "ram-budget", + reason: "full", + workerId: "wrk_a", + }); + + await workers.setDrained("wrk_a", true); + await tick(); + + expect(listed(coordinator)?.[0]).toEqual({ reason: "drained", workerId: "wrk_a" }); + }); + + it("lists a refused worker whose view lost its free platform slot while the probe was in flight as full with platform-running-limit, not the reason it refused with", async () => { + const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a"), [ + "wrk_a", + ]); + clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "ram-budget" })); + await ask(coordinator); + workers.refresh("wrk_a", { capacity: saturatedIos() }); + + (clients["wrk_a"] as GatedClient).release(); + await tick(); + + expect(listed(coordinator)?.[0]).toEqual({ + capacityReason: "platform-running-limit", + reason: "full", + workerId: "wrk_a", + }); + }); + + it("lists a worker claimed by an earlier request in the same walk as not-chosen for the later one, even at its RAM budget", async () => { + const { clients, coordinator, workers } = fleet(["wrk_a"], () => ({ + capacity: saturatedIos(), + })); + clients["wrk_a"]?.requestLeaseQueue.push({ kind: "hang" }); + await ask(coordinator, REQUEST, { ownerId: "agent-1", requesterId: "agent-1" }); + await ask(coordinator, REQUEST, { ownerId: "agent-2", requesterId: "agent-2" }); + + workers.refresh("wrk_a", { capacity: atRamBudgetRoomy() }); + await tick(); + + expect(coordinator.waitingRequests().map((entry) => entry.stage)).toEqual([ + "starting", + "queued", + ]); + expect(listed(coordinator, 1)).toEqual([{ reason: "not-chosen", workerId: "wrk_a" }]); + + function atRamBudgetRoomy() { + const capacity = roomierIos(); + return { ...capacity, ios: { ...capacity.ios, atRamBudget: true } }; + } + }); + + it("lists no-free-physical for each worker of a physical request whose matching devices are not present, one entry per worker", async () => { + const notPresent = (id: string) => + physicalDeviceFixture(id, "ready", { presence: { present: false, reason: "not-present" } }); + const { coordinator } = fleet(["wrk_a", "wrk_b"], (id) => ({ + devices: [notPresent(id === "wrk_a" ? "dev_p" : "dev_q")], + })); + + await ask(coordinator, { physical: true, platform: "ios" }); + + expect(listed(coordinator)).toEqual([ + { reason: "no-free-physical", workerId: "wrk_a" }, + { reason: "no-free-physical", workerId: "wrk_b" }, + ]); + }); + + it("lists disconnected for a target found unreachable at the attempt, after the request is queued again", async () => { + const { coordinator, directory } = fleet(["wrk_a"]); + directory.unreachable.add("wrk_a"); + + await ask(coordinator); + + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); + expect(listed(coordinator)).toEqual([{ reason: "disconnected", workerId: "wrk_a" }]); + }); + + describe("a worker that answers PROVISION_FAILED", () => { + const provisionFailed = (message: string) => ({ + error: new SimlockError("PROVISION_FAILED", "domain", message, {}), + kind: "error" as const, + }); + + it("is walked past: the next worker is asked and its grant settles the request", async () => { + const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + clients["wrk_b"]?.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { outcome, state } = await ask(coordinator); + + expect(state.state).toBe("fulfilled"); + expect((await outcome).lease.worker?.id).toBe("wrk_b"); + }); + + it("rejects the request with PROVISION_FAILED and the last worker's message, and lists nothing, when every worker that could take it answered so", async () => { + const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + clients["wrk_b"]?.requestLeaseQueue.push(provisionFailed("worker b: boom")); + + const { ended, state } = await ask(coordinator); + + expect(state.state).toBe("rejected"); + expect(await ended).toMatchObject({ code: "PROVISION_FAILED", message: "worker b: boom" }); + expect(coordinator.waitingRequests()).toEqual([]); + expect(coordinator.queueDepth).toBe(0); + }); + + it("leaves the request queued when the other worker is only full", async () => { + const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + clients["wrk_b"]?.requestLeaseQueue.push(noCapacity()); + + const { state } = await ask(coordinator); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); + expect(listed(coordinator)).toEqual([ + { reason: "not-chosen", workerId: "wrk_a" }, + { reason: "full", workerId: "wrk_b" }, + ]); + }); + + it.each([ + { + build: () => { + const built = fleet(["wrk_a"]); + built.workers.connected("wrk_b", undefined, "0.3.0"); + return built; + }, + row: "not ready, having read no capacity or catalog yet", + }, + { + build: () => { + const built = fleet(["wrk_a", "wrk_b"]); + built.workers.refresh("wrk_b", { health: "starting" }); + return built; + }, + row: "unhealthy, still starting", + }, + { + build: () => fleet(["wrk_a", "wrk_b"], (id) => (id === "wrk_b" ? { queueDepth: 1 } : {})), + row: "busy with a queue of its own", + }, + { + build: () => { + const built = fleet(["wrk_a", "wrk_b"]); + built.workers.refresh("wrk_b", { + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + return built; + }, + row: "updating", + }, + ])("leaves the request queued when the other worker is $row", async ({ build }) => { + const { clients, coordinator } = build(); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + + const { state } = await ask(coordinator); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); + }); + + it.each([ + { + build: () => { + const built = fleet(["wrk_a", "wrk_b"]); + built.workers.disconnected("wrk_b"); + return built; + }, + row: "disconnected", + }, + { + build: async () => { + const built = fleet(["wrk_a", "wrk_b"]); + await built.workers.setDrained("wrk_b", true); + return built; + }, + row: "drained", + }, + ])( + "rejects the request when the other worker is $row, which cannot take it", + async ({ build }) => { + const { clients, coordinator } = await build(); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + + const { ended, state } = await ask(coordinator); + + expect(state.state).toBe("rejected"); + expect(await ended).toMatchObject({ code: "PROVISION_FAILED", message: "worker a: boom" }); + }, + ); + + it("leaves the request queued when the other worker is full in its view, so the gateway never asks it", async () => { + const { clients, coordinator } = fleet(["wrk_a", "wrk_b"], (id) => + id === "wrk_b" ? { capacity: saturatedIos() } : {}, + ); + clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); + const untouched = noCapacity(); + clients["wrk_b"]?.requestLeaseQueue.push(untouched); + + const { state } = await ask(coordinator); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); + expect(clients["wrk_b"]?.requestLeaseQueue).toEqual([untouched]); + }); + + it("leaves the request queued when the worker that is only full refused first and the one asked after a view change failed to provision", async () => { + const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"]); + clients["wrk_a"]?.requestLeaseQueue.push(noCapacity()); + clients["wrk_b"]?.requestLeaseQueue.push(provisionFailed("worker b: boom")); + const { state } = await ask(coordinator); + + workers.refresh("wrk_b", { capacity: roomierIos() }); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); + }); + }); +}); diff --git a/src/gateway/fleet-requests.test.ts b/src/gateway/fleet-requests.test.ts index b107c04d9..47167ff8a 100644 --- a/src/gateway/fleet-requests.test.ts +++ b/src/gateway/fleet-requests.test.ts @@ -1,18 +1,14 @@ import { describe, expect, it } from "vitest"; -import type { EventBus, EventMap } from "../bus/index.js"; +import type { EventMap } from "../bus/index.js"; import type { LeaseRecord } from "../admin/index.js"; import { fromWireError, SimlockError } from "../contract/index.js"; import { ScriptedProcessRunner } from "../ports/index.js"; import { promiseState } from "../test-support/promise-state.js"; -import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; import type { WorkerReportedLease } from "./lease-index.js"; import { RequesterAlreadyLeasedError } from "./queue.js"; import { catalogFixture, - deviceFixture, grantFixture, - leaseFixture, - physicalDeviceFixture, ScriptedWorkerClient, statusFixture, } from "./test-support.js"; @@ -530,1172 +526,3 @@ describe("FleetLeaseCoordinator caller-chosen lease IDs", () => { }); }); }); - -describe("FleetLeaseCoordinator records every fleet request's outcome (ADR 0021)", () => { - function oneWorker(overrides: Parameters[0] = {}) { - const fleet = buildFleet(overrides); - const client = new ScriptedWorkerClient(); - fleet.directory.add("wrk_a", client); - connectWorker(fleet.workers, "wrk_a"); - return { ...fleet, client }; - } - - function record( - eventBus: EventBus, - ...names: Name[] - ): Array<{ name: Name; payload: EventMap[Name] }> { - const seen: Array<{ name: Name; payload: EventMap[Name] }> = []; - for (const name of names) { - eventBus.subscribe(name, (envelope) => - seen.push({ name, payload: envelope.payload as EventMap[Name] }), - ); - } - return seen; - } - - async function start( - coordinator: FleetLeaseCoordinator, - overrides: Parameters[0] = {}, - ) { - let requestId: string | undefined; - const outcome = coordinator.request( - REQUEST, - requestOptions({ ...overrides, onAdmitted: (id) => (requestId = id) }), - ); - const ended = outcome.then( - (grant) => grant as unknown, - (error: unknown) => error, - ); - const state = promiseState(outcome); - await tick(); - return { ended, outcome, requestId: () => requestId, state }; - } - - it.each([ - ["a waiting caller", { noWait: false }], - ["a noWait caller", { noWait: true }], - ])("sends the gateway's request id as fleetRequestId, for %s", async (_label, caller) => { - const { client, coordinator } = oneWorker(); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { requestId } = await start(coordinator, caller); - - expect(requestId()).toMatch(/^req_/); - expect(client.lastRequestLeaseInput).toMatchObject({ - fleetRequestId: requestId(), - noWait: true, - }); - }); - - function twoWorkersAPreferred() { - const fleet = buildFleet(); - const preferred = new ScriptedWorkerClient(); - const other = new ScriptedWorkerClient(); - fleet.directory.add("wrk_a", preferred); - fleet.directory.add("wrk_b", other); - const capacity = statusFixture().capacity; - connectWorker(fleet.workers, "wrk_a", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, - }); - connectWorker(fleet.workers, "wrk_b"); - return { ...fleet, other, preferred }; - } - - it("sends the same fleetRequestId to the second worker when the first refuses NO_CAPACITY", async () => { - const { coordinator, other, preferred } = twoWorkersAPreferred(); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { requestId } = await start(coordinator, { noWait: true }); - - expect( - [preferred, other].map( - (client) => - (client.lastRequestLeaseInput as Record | undefined)?.fleetRequestId, - ), - ).toEqual([requestId(), requestId()]); - }); - - it("emits one lease.rejected worker-failed with the caller's error code for a request whose dispatch fails after a progress push", async () => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "lease.rejected", "request.granted"); - client.requestLeaseQueue.push({ - error: new SimlockError("INTERNAL", "domain", "the worker broke", {}), - kind: "error", - progress: [{ etaMs: 1_000, stage: "booting" }], - }); - - const { ended, requestId } = await start(coordinator); - - expect(await ended).toMatchObject({ code: "INTERNAL" }); - expect(events).toEqual([ - { - name: "lease.rejected", - payload: { - code: "INTERNAL", - reason: "worker-failed", - requestId: requestId(), - requestSpec: REQUEST, - requester: "agent-1", - worker: "wrk_a", - }, - }, - ]); - }); - - it.each([{ answer: null }, { answer: {} }, { answer: { lease: null } }])( - "ends a request whose worker answers a malformed grant ($answer) in lease.rejected worker-failed, with no unhandled rejection", - async ({ answer }) => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "lease.rejected"); - client.requestLeaseQueue.push({ grant: answer as never, kind: "grant" }); - const unhandled: unknown[] = []; - const listener = (reason: unknown): void => { - unhandled.push(reason); - }; - process.on("unhandledRejection", listener); - try { - const { ended, state } = await start(coordinator); - await new Promise((resolve) => setImmediate(resolve)); - expect(state.state).toBe("rejected"); - await expect(ended).resolves.toMatchObject({ code: "INTERNAL" }); - } finally { - process.off("unhandledRejection", listener); - } - - expect(unhandled).toEqual([]); - expect(events).toEqual([ - { name: "lease.rejected", payload: expect.objectContaining({ reason: "worker-failed" }) }, - ]); - }, - ); - - it.each([ - { - elapseMs: 0, - failure: "fails on a lost uplink", - script: (client: ScriptedWorkerClient) => - client.requestLeaseQueue.push({ - error: new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - "Daemon connection is closed", - {}, - ), - kind: "error", - }), - }, - { - elapseMs: 5_000, - failure: "times out", - script: (client: ScriptedWorkerClient) => client.requestLeaseQueue.push({ kind: "hang" }), - }, - ])( - "emits one lease.rejected worker-failed with code WORKER_UNREACHABLE, naming the worker, for a dispatch that $failure", - async ({ elapseMs, script }) => { - const { clock, client, coordinator, eventBus } = oneWorker({ leaseRequestTimeoutMs: 5_000 }); - const events = record(eventBus, "lease.rejected"); - script(client); - - const { ended } = await start(coordinator); - clock.advance(elapseMs); - await tick(); - - expect(await ended).toMatchObject({ code: "WORKER_UNREACHABLE" }); - expect(events).toEqual([ - { - name: "lease.rejected", - payload: expect.objectContaining({ - code: "WORKER_UNREACHABLE", - reason: "worker-failed", - worker: "wrk_a", - }), - }, - ]); - }, - ); - - it("carries the failing worker's id, not the one that only refused, as worker on a worker-failed rejection", async () => { - const { coordinator, eventBus, other } = twoWorkersAPreferred(); - const events = record(eventBus, "lease.rejected"); - other.requestLeaseQueue.push({ - error: new SimlockError("INTERNAL", "domain", "wrk_b broke", {}), - kind: "error", - progress: [{ etaMs: 1, stage: "booting" }], - }); - - await start(coordinator, { noWait: true }); - - expect(events).toEqual([ - { - name: "lease.rejected", - payload: expect.objectContaining({ reason: "worker-failed", worker: "wrk_b" }), - }, - ]); - }); - - it("ends a request never queued on a worker's cannot-serve refusal in one lease.rejected unresolvable-spec", async () => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "lease.rejected", "lease.queued"); - client.requestLeaseQueue.push({ - error: new SimlockError("RUNTIME_MISSING", "domain", "no such runtime", { - downloadable: false, - osVersion: "26.0", - platform: "ios", - }), - kind: "error", - }); - - const { ended, requestId } = await start(coordinator); - - expect(await ended).toMatchObject({ code: "RUNTIME_MISSING" }); - expect(events).toEqual([ - { - name: "lease.rejected", - payload: expect.objectContaining({ reason: "unresolvable-spec", requestId: requestId() }), - }, - ]); - }); - - it("emits no lease.rejected for a request one worker refuses NO_CAPACITY and another grants", async () => { - const { coordinator, eventBus, other } = twoWorkersAPreferred(); - const events = record(eventBus, "lease.rejected"); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { ended } = await start(coordinator, { noWait: true }); - - expect(await ended).toMatchObject({ lease: { worker: { id: "wrk_b" } } }); - expect(events).toEqual([]); - }); - - it("emits one request.granted with worker and both lease ids, and no workerId in its payload, for a settled grant", async () => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "request.granted", "lease.rejected"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { ended, requestId } = await start(coordinator); - - expect(await ended).toMatchObject({ lease: { id: "wrk_a.lse_1" } }); - expect(events).toEqual([ - { - name: "request.granted", - payload: { - leaseId: "wrk_a.lse_1", - requestId: requestId(), - worker: "wrk_a", - workerLeaseId: "lse_1", - }, - }, - ]); - expect(events[0]?.payload).not.toHaveProperty("workerId"); - }); - - it("emits lease.rejected lease-id-taken and no request.granted for a grant the lease index refuses", async () => { - const { client, coordinator, eventBus, leaseIndex } = oneWorker(); - const events = record(eventBus, "request.granted", "lease.rejected"); - client.requestLeaseQueue.push({ - beforeGrant: () => { - leaseIndex.rebuildFromWorker("wrk_b", [ - { - grantedAt: 1, - id: "myid", - idChosenByRequester: true, - ownerId: "agent-9", - requesterId: `${GATEWAY_PREFIX}agent-9`, - } as WorkerReportedLease, - ]); - }, - grant: grantFixture({ - lease: { ...grantFixture().lease, id: "myid", idChosenByRequester: true } as LeaseRecord, - }), - kind: "grant", - }); - - const { ended } = await start(coordinator, { leaseId: "myid" } as never); - - expect(await ended).toMatchObject({ code: "LEASE_ID_TAKEN" }); - expect(events).toEqual([ - { name: "lease.rejected", payload: expect.objectContaining({ reason: "lease-id-taken" }) }, - ]); - }); - - it("emits no request.granted for a grant that lands after dispose()", async () => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "request.granted", "lease.rejected"); - client.requestLeaseQueue.push({ - beforeGrant: () => coordinator.dispose(), - grant: grantFixture(), - kind: "grant", - }); - - const { ended } = await start(coordinator); - - expect(await ended).toMatchObject({ code: "DAEMON_STOPPING" }); - expect(events).toEqual([]); - }); - - it("emits no request.granted for a grant that lands after the dispatch timed out", async () => { - const { clock, client, coordinator, eventBus } = oneWorker({ leaseRequestTimeoutMs: 5_000 }); - const events = record(eventBus, "request.granted"); - client.requestLeaseQueue.push({ - beforeGrant: async () => { - await Promise.resolve(); - clock.advance(5_000); - }, - grant: grantFixture(), - kind: "grant", - }); - - const { ended } = await start(coordinator); - - expect(await ended).toMatchObject({ code: "WORKER_UNREACHABLE" }); - expect(events).toEqual([]); - }); - - it("emits no lease.rejected and no request.granted on dispose() with a queued and an in-flight request", async () => { - const { client, coordinator, directory, eventBus, workers } = oneWorker(); - const events = record(eventBus, "lease.rejected", "request.granted"); - client.requestLeaseQueue.push({ kind: "hang" }); - const inFlight = await start(coordinator); - const busy = new ScriptedWorkerClient(); - directory.add("wrk_b", busy); - connectWorker(workers, "wrk_b", { capacity: saturatedIos(), models: ["iPhone 18"] }); - const queued = await start(coordinator, { ownerId: "agent-2", requesterId: "agent-2" }); - expect(inFlight.state.state).toBe("pending"); - expect(coordinator.queueDepth).toBe(1); - - coordinator.dispose(); - await tick(); - - expect(await inFlight.ended).toMatchObject({ code: "DAEMON_STOPPING" }); - expect(await queued.ended).toMatchObject({ code: "DAEMON_STOPPING" }); - expect(events).toEqual([]); - }); - - it("emits no request.granted for a grant given back for a mismatched leaseId, and one for the retried grant", async () => { - const { client, coordinator, eventBus, workers } = oneWorker(); - const events = record(eventBus, "request.granted"); - const chosen = (leaseId: string) => - grantFixture({ - lease: { ...grantFixture().lease, id: leaseId, idChosenByRequester: true } as LeaseRecord, - }); - client.requestLeaseQueue.push( - { grant: chosen("not-what-was-sent"), kind: "grant" }, - { grant: chosen("myid"), kind: "grant" }, - ); - - const { ended, requestId } = await start(coordinator, { leaseId: "myid" } as never); - expect(events).toEqual([]); - - workers.refresh("wrk_a", { capacity: roomierIos() }); - await tick(); - - expect(await ended).toMatchObject({ lease: { id: "myid" } }); - expect(events).toEqual([ - { - name: "request.granted", - payload: { - leaseId: "myid", - requestId: requestId(), - worker: "wrk_a", - workerLeaseId: "myid", - }, - }, - ]); - }); - - it.each([ - [ - "REQUESTER_ALREADY_LEASED", - fromWireError("REQUESTER_ALREADY_LEASED", "already holds a lease", { requesterId: "gw" }), - ], - [ - "LEASE_ID_TAKEN", - fromWireError("LEASE_ID_TAKEN", "lease ID myid is already in use", { leaseId: "myid" }), - ], - ])( - "ends a request a worker refuses %s before any progress in lease.rejected worker-failed with that code, not the gateway's own lease-id-taken", - async (code, error) => { - const { client, coordinator, eventBus } = oneWorker(); - const events = record(eventBus, "lease.rejected"); - client.requestLeaseQueue.push({ error, kind: "error" }); - - const { ended } = await start(coordinator); - - expect(await ended).toMatchObject({ code }); - expect(events).toEqual([ - { - name: "lease.rejected", - payload: expect.objectContaining({ code, reason: "worker-failed", worker: "wrk_a" }), - }, - ]); - }, - ); -}); - -describe("FleetLeaseCoordinator: a requester may hold up to lease.maxPerRequester fleet leases (ADR 0023)", () => { - const held = (id: string, grantedAt: number, requester = "agent-1") => ({ - grantedAt, - id, - ownerId: requester, - requesterId: `${GATEWAY_PREFIX}${requester}`, - }); - const atLimit = (held: number, limit: number, leaseId: string) => - `Requester agent-1 already holds ${String(held)} leases and lease.maxPerRequester is ${String(limit)}; release one (\`simlock release ${leaseId}\`) before requesting another device`; - const waitingMessage = - "Requester agent-1 already has a waiting request; wait for it to be granted or cancel it before requesting another device"; - - async function withWaitingRequest(maxPerRequester?: number) { - const built = buildFleet(maxPerRequester === undefined ? {} : { maxPerRequester }); - connectWorker(built.workers, "wrk_a", { capacity: saturatedIos() }); - built.directory.add("wrk_a", new ScriptedWorkerClient()); - void built.coordinator.request(REQUEST, requestOptions()).catch(() => undefined); - await tick(); - expect(built.coordinator.queueDepth).toBe(1); - return built; - } - - it("with no limit set, a requester holding a fleet lease is refused with today's message, naming that lease", async () => { - const { coordinator, leaseIndex } = buildFleet(); - leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: "wrk_a.lse_1", - message: - "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with limit 2, a requester holding two fleet leases is refused, naming the earlier-granted one", async () => { - const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: 2 }); - leaseIndex.rebuildFromWorker("wrk_a", [held("lse_9", 10), held("lse_1", 20)]); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: "wrk_a.lse_9", - message: atLimit(2, 2, "wrk_a.lse_9"), - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with limit 2, a requester holding one fleet lease is not refused", async () => { - const { coordinator, directory, leaseIndex, workers } = buildFleet({ maxPerRequester: 2 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); - - await expect(coordinator.request(REQUEST, requestOptions())).resolves.toMatchObject({ - lease: expect.anything(), - }); - }); - - describe("per platform", () => { - const ANDROID_REQUEST = { - physical: false, - model: "Pixel 9", - osVersion: "26.0", - platform: "android" as const, - }; - const iosHeld = (): LeaseRecord => ({ - ...(leaseFixture("lse_1", "dev_1") as LeaseRecord), - requesterId: `${GATEWAY_PREFIX}agent-1`, - }); - const androidGrant = () => { - const base = grantFixture(); - return grantFixture({ - device: { - ...base.device, - id: "dev_2", - spec: { ...base.device.spec, model: "Pixel 9", platform: "android" }, - }, - lease: leaseFixture("lse_2", "dev_2") as LeaseRecord, - }); - }; - const iosRefusal = (leaseId: string) => - `Requester agent-1 already holds iOS lease ${leaseId}; release it (\`simlock release ${leaseId}\`) before requesting another iOS device`; - const twoWorkers = (built: ReturnType, held: LeaseRecord[] = []) => { - const ios = new ScriptedWorkerClient(); - const android = new ScriptedWorkerClient(); - built.directory.add("wrk_a", ios); - built.directory.add("wrk_b", android); - connectWorker(built.workers, "wrk_a", { - devices: held.length === 0 ? [] : [deviceFixture("dev_1", "leased")], - leases: held, - }); - connectWorker(built.workers, "wrk_b", { models: ["Pixel 9"], platform: "android" }); - return { android, ios }; - }; - - it("with lease.maxPerRequester per platform, a requester holding an iOS fleet lease is granted an Android one on another worker and refused a second iOS one, also once the index is rebuilt from worker views alone", async () => { - const limit = { ios: 1, android: 1 }; - const live = buildFleet({ maxPerRequester: limit }); - const liveWorkers = twoWorkers(live); - liveWorkers.ios.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - liveWorkers.android.requestLeaseQueue.push({ grant: androidGrant(), kind: "grant" }); - await live.coordinator.request(REQUEST, requestOptions()); - - const android = await live.coordinator.request(ANDROID_REQUEST, requestOptions()); - const second = await live.coordinator - .request(REQUEST, requestOptions()) - .catch((e: unknown) => e); - - expect(android.lease.id).toBe("wrk_b.lse_2"); - expect(second).toMatchObject({ - existingLeaseId: "wrk_a.lse_1", - message: iosRefusal("wrk_a.lse_1"), - name: "RequesterAlreadyLeasedError", - }); - - const rebuilt = buildFleet({ maxPerRequester: limit }); - const rebuiltWorkers = twoWorkers(rebuilt, [iosHeld()]); - rebuiltWorkers.android.requestLeaseQueue.push({ grant: androidGrant(), kind: "grant" }); - - const againIos = await rebuilt.coordinator - .request(REQUEST, requestOptions()) - .catch((e: unknown) => e); - const againAndroid = await rebuilt.coordinator.request(ANDROID_REQUEST, requestOptions()); - - expect(againIos).toMatchObject({ message: iosRefusal("wrk_a.lse_1") }); - expect(againAndroid.lease.id).toBe("wrk_b.lse_2"); - }); - }); - - it("with a numeric limit of 1, a requester holding an iOS and an Android fleet lease is refused with today's message, naming the earlier one", async () => { - const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: 1 }); - leaseIndex.rebuildFromWorker( - "wrk_a", - [ - { ...held("lse_2", 20), deviceId: "dev_2" }, - { ...held("lse_1", 10), deviceId: "dev_1" }, - ], - [ - { id: "dev_1", spec: { platform: "ios" } }, - { id: "dev_2", spec: { platform: "android" } }, - ], - ); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: - "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with lease.maxPerRequester per platform, a fleet lease whose device a worker view does not list counts against every platform", async () => { - const { coordinator, leaseIndex } = buildFleet({ maxPerRequester: { ios: 1, android: 1 } }); - leaseIndex.rebuildFromWorker("wrk_a", [{ ...held("lse_1", 10), deviceId: "dev_unlisted" }], []); - - const refusal = await coordinator - .request({ ...REQUEST, model: "Pixel 9", platform: "android" }, requestOptions()) - .catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: - "Requester agent-1 already holds Android lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another Android device", - name: "RequesterAlreadyLeasedError", - }); - }); - - describe("per platform, waiting", () => { - it("with lease.maxPerRequester per platform, a requester may wait for an iOS and an Android lease at once but not two of one platform, and lease.cancel names which one to cancel", async () => { - const built = buildFleet({ maxPerRequester: { ios: 1, android: 1 } }); - const capacity = statusFixture().capacity; - built.directory.add("wrk_a", new ScriptedWorkerClient()); - built.directory.add("wrk_b", new ScriptedWorkerClient()); - connectWorker(built.workers, "wrk_a", { capacity: saturatedIos() }); - connectWorker(built.workers, "wrk_b", { - capacity: { ...capacity, android: { ...capacity.android, maxRunning: 0 } }, - models: ["Pixel 9"], - platform: "android", - }); - const androidRequest = { - physical: false, - model: "Pixel 9", - osVersion: "26.0", - platform: "android" as const, - }; - void built.coordinator.request(REQUEST, requestOptions()).catch(() => undefined); - await tick(); - - const android = built.coordinator.request(androidRequest, requestOptions()); - void android.catch(() => undefined); - await tick(); - const secondIos = await Promise.race([ - built.coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e), - tick().then(() => "admitted"), - ]); - const platformsWaiting = () => - built.coordinator - .waitingRequests() - .map((waiting) => waiting.spec.platform) - .sort(); - - expect(platformsWaiting()).toEqual(["android", "ios"]); - expect(secondIos).toMatchObject({ - message: - "Requester agent-1 already has a waiting iOS request; wait for it to be granted or cancel it before requesting another iOS device", - name: "RequesterAlreadyLeasedError", - }); - await expect(built.coordinator.cancelPending("agent-1", "android")).resolves.toBe( - "cancelled", - ); - expect(platformsWaiting()).toEqual(["ios"]); - }); - }); - - it("with limit 2, a requester with a waiting fleet request is refused with the waiting-request message", async () => { - const { coordinator } = await withWaitingRequest(2); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: waitingMessage, - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with limit 2, a requester with a waiting fleet request that also holds two fleet leases gets the waiting-request message", async () => { - const { coordinator, leaseIndex } = await withWaitingRequest(2); - leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1), held("lse_2", 2)]); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - message: waitingMessage, - name: "RequesterAlreadyLeasedError", - }); - }); - - it("with no limit set, a requester that holds a fleet lease and has a waiting request gets today's message naming that lease", async () => { - const { coordinator, leaseIndex } = await withWaitingRequest(); - leaseIndex.rebuildFromWorker("wrk_a", [held("lse_1", 1)]); - - const refusal = await coordinator.request(REQUEST, requestOptions()).catch((e: unknown) => e); - - expect(refusal).toMatchObject({ - existingLeaseId: "wrk_a.lse_1", - message: - "Requester agent-1 already holds lease wrk_a.lse_1; release it (`simlock release wrk_a.lse_1`) before requesting another device", - }); - }); -}); - -describe("FleetLeaseCoordinator lists why each worker cannot take a waiting request (#637)", () => { - type Reasons = readonly { workerId: string; reason: string; capacityReason?: string }[]; - - // why: holds a `lease.request` until `release` runs, so a test can change the fleet while the gateway's probe is in flight. - class GatedClient extends ScriptedWorkerClient { - release: () => void = () => {}; - readonly #gate = new Promise((resolve) => { - this.release = resolve; - }); - - override async requestLease( - input: Parameters[0], - options?: Parameters[1], - ) { - await this.#gate; - return super.requestLease(input, options); - } - } - - function atRamBudget(): ReturnType["capacity"] { - const capacity = statusFixture().capacity; - return { ...capacity, ios: { ...capacity.ios, atRamBudget: true } }; - } - - function noCapacity(details?: unknown) { - return { - error: new SimlockError("NO_CAPACITY", "domain", "full", details as never), - kind: "error" as const, - }; - } - - function fleet( - ids: readonly string[], - setup: (id: string) => Parameters[2] = () => ({}), - gated: readonly string[] = [], - ) { - const fleetHarness = buildFleet(); - const clients: Record = {}; - for (const id of ids) { - const client = gated.includes(id) ? new GatedClient() : new ScriptedWorkerClient(); - clients[id] = client; - fleetHarness.directory.add(id, client); - connectWorker(fleetHarness.workers, id, setup(id)); - } - return { clients, ...fleetHarness }; - } - - async function ask( - coordinator: FleetLeaseCoordinator, - request: Parameters[0] = REQUEST, - options: Parameters[0] = {}, - ) { - const outcome = coordinator.request(request, requestOptions(options)); - const ended = outcome.catch((error: unknown) => error); - const state = promiseState(outcome); - await tick(); - return { ended, outcome, state }; - } - - const listed = (coordinator: FleetLeaseCoordinator, index = 0): Reasons | undefined => - (coordinator.waitingRequests()[index] as { workers?: Reasons } | undefined)?.workers; - - const saturatedExcept = (free: string) => (id: string) => - id === free ? {} : { capacity: saturatedIos() }; - - it("lists, in worker id order, drained for a drained worker and full with platform-running-limit for one with no free iOS slot, also when it has no free global slot either", async () => { - const capacity = statusFixture().capacity; - const bothFull = { - ...saturatedIos(), - global: { ...capacity.global, maxRunning: 0 }, - }; - const { coordinator, workers } = fleet(["wrk_c", "wrk_b", "wrk_a"], (id) => - id === "wrk_b" ? { capacity: saturatedIos() } : id === "wrk_c" ? { capacity: bothFull } : {}, - ); - await workers.setDrained("wrk_a", true); - - await ask(coordinator); - - expect(listed(coordinator)).toEqual([ - { reason: "drained", workerId: "wrk_a" }, - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_c" }, - ]); - }); - - it("lists full with global-running-limit for a worker with a free iOS slot and no free global slot", async () => { - const capacity = statusFixture().capacity; - const { coordinator } = fleet(["wrk_a", "wrk_b"], (id) => - id === "wrk_a" - ? { capacity: { ...capacity, global: { ...capacity.global, maxRunning: 0 } } } - : { capacity: saturatedIos() }, - ); - - await ask(coordinator); - - expect(listed(coordinator)?.[0]).toEqual({ - capacityReason: "global-running-limit", - reason: "full", - workerId: "wrk_a", - }); - }); - - it("lists one entry for every known view, a disconnected and an incompatible one included, and none for a worker removed", async () => { - const { coordinator, workers } = fleet(["wrk_a", "wrk_b"], (id) => - id === "wrk_a" ? { capacity: saturatedIos() } : {}, - ); - workers.incompatible( - "wrk_c", - undefined, - { gateway: { max: 20, min: 20 }, worker: { max: 4, min: 4 } }, - "0.2.0", - ); - workers.disconnected("wrk_b"); - - await ask(coordinator); - - expect(listed(coordinator)).toEqual([ - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_a" }, - { reason: "disconnected", workerId: "wrk_b" }, - { reason: "incompatible", workerId: "wrk_c" }, - ]); - - await workers.remove("wrk_b"); - await tick(); - - expect(listed(coordinator)?.map((entry) => entry.workerId)).toEqual(["wrk_a", "wrk_c"]); - }); - - it.each([ - [ - "platform-running-limit", - { capacityReason: "platform-running-limit" }, - "platform-running-limit", - ], - ["none", {}, undefined], - ["bogus", { capacityReason: "bogus" }, undefined], - ["a number", { capacityReason: 42 }, undefined], - ])( - "lists a worker that answered NO_CAPACITY with capacityReason %s as full, with the reason only when it is one of the four", - async (_label, details, expected) => { - const { clients, coordinator } = fleet(["wrk_a"]); - clients["wrk_a"]?.requestLeaseQueue.push(noCapacity(details)); - - await ask(coordinator); - - expect(listed(coordinator)).toEqual([ - { - reason: "full", - workerId: "wrk_a", - ...(expected === undefined ? {} : { capacityReason: expected }), - }, - ]); - }, - ); - - it.each([ - ["UNKNOWN_MODEL", "unknown-model"], - ["RUNTIME_MISSING", "runtime-missing"], - ["NO_DRIVER", "no-driver"], - ["CAPABILITY_MISSING", "capability-missing"], - ] as const)("lists a worker that answered %s before progress as %s", async (code, reason) => { - const { clients, coordinator } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a")); - clients["wrk_a"]?.requestLeaseQueue.push({ - error: new SimlockError(code, "domain", "cannot serve", {} as never), - kind: "error", - }); - - await ask(coordinator); - - expect(listed(coordinator)).toEqual([ - { reason, workerId: "wrk_a" }, - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, - ]); - }); - - it("lists a worker that granted another lease id than the one asked for as plain full", async () => { - const { clients, coordinator } = fleet(["wrk_a"]); - clients["wrk_a"]?.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { - ...grantFixture().lease, - id: "not-what-was-sent", - idChosenByRequester: true, - } as LeaseRecord, - }), - kind: "grant", - }); - - await ask(coordinator, REQUEST, { leaseId: "myid" } as never); - - expect(listed(coordinator)).toEqual([{ reason: "full", workerId: "wrk_a" }]); - }); - - it("after a NO_CAPACITY re-queue and before any view change, recomputes every entry: the refused worker is full with its reason, one drained meanwhile is drained, one that lost to it is not-chosen", async () => { - const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b", "wrk_c"], () => ({}), [ - "wrk_a", - ]); - clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "device-limit" })); - - await ask(coordinator); - expect(coordinator.waitingRequests()[0]).toMatchObject({ stage: "starting" }); - await workers.setDrained("wrk_b", true); - (clients["wrk_a"] as GatedClient).release(); - await tick(); - - expect(coordinator.waitingRequests()[0]).toMatchObject({ stage: "queued" }); - expect(listed(coordinator)).toEqual([ - { capacityReason: "device-limit", reason: "full", workerId: "wrk_a" }, - { reason: "drained", workerId: "wrk_b" }, - { reason: "not-chosen", workerId: "wrk_c" }, - ]); - }); - - it("follows the latest walk: a worker drained lists drained, and once undrained and refused with no detail at its RAM budget it lists full with ram-budget", async () => { - const { coordinator, workers } = fleet(["wrk_a", "wrk_b"], (id) => - id === "wrk_a" ? { capacity: atRamBudget() } : { capacity: saturatedIos() }, - ); - await workers.setDrained("wrk_a", true); - - await ask(coordinator); - expect(listed(coordinator)?.[0]).toEqual({ reason: "drained", workerId: "wrk_a" }); - - await workers.setDrained("wrk_a", false); - await tick(); - - expect(listed(coordinator)).toEqual([ - { capacityReason: "ram-budget", reason: "full", workerId: "wrk_a" }, - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, - ]); - }); - - it("lists no workers for a starting request", async () => { - const { clients, coordinator } = fleet(["wrk_a"]); - clients["wrk_a"]?.requestLeaseQueue.push({ kind: "hang" }); - - await ask(coordinator); - - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "starting" })]); - expect(coordinator.waitingRequests()[0]).not.toHaveProperty("workers"); - }); - - it("lists a worker drained after it refused the request as drained, not as its remembered refusal", async () => { - const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a")); - clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "ram-budget" })); - await ask(coordinator); - expect(listed(coordinator)?.[0]).toEqual({ - capacityReason: "ram-budget", - reason: "full", - workerId: "wrk_a", - }); - - await workers.setDrained("wrk_a", true); - await tick(); - - expect(listed(coordinator)?.[0]).toEqual({ reason: "drained", workerId: "wrk_a" }); - }); - - it("lists a refused worker whose view lost its free platform slot while the probe was in flight as full with platform-running-limit, not the reason it refused with", async () => { - const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"], saturatedExcept("wrk_a"), [ - "wrk_a", - ]); - clients["wrk_a"]?.requestLeaseQueue.push(noCapacity({ capacityReason: "ram-budget" })); - await ask(coordinator); - workers.refresh("wrk_a", { capacity: saturatedIos() }); - - (clients["wrk_a"] as GatedClient).release(); - await tick(); - - expect(listed(coordinator)?.[0]).toEqual({ - capacityReason: "platform-running-limit", - reason: "full", - workerId: "wrk_a", - }); - }); - - it("lists a worker claimed by an earlier request in the same walk as not-chosen for the later one, even at its RAM budget", async () => { - const { clients, coordinator, workers } = fleet(["wrk_a"], () => ({ - capacity: saturatedIos(), - })); - clients["wrk_a"]?.requestLeaseQueue.push({ kind: "hang" }); - await ask(coordinator, REQUEST, { ownerId: "agent-1", requesterId: "agent-1" }); - await ask(coordinator, REQUEST, { ownerId: "agent-2", requesterId: "agent-2" }); - - workers.refresh("wrk_a", { capacity: atRamBudgetRoomy() }); - await tick(); - - expect(coordinator.waitingRequests().map((entry) => entry.stage)).toEqual([ - "starting", - "queued", - ]); - expect(listed(coordinator, 1)).toEqual([{ reason: "not-chosen", workerId: "wrk_a" }]); - - function atRamBudgetRoomy() { - const capacity = roomierIos(); - return { ...capacity, ios: { ...capacity.ios, atRamBudget: true } }; - } - }); - - it("lists no-free-physical for each worker of a physical request whose matching devices are not present, one entry per worker", async () => { - const notPresent = (id: string) => - physicalDeviceFixture(id, "ready", { presence: { present: false, reason: "not-present" } }); - const { coordinator } = fleet(["wrk_a", "wrk_b"], (id) => ({ - devices: [notPresent(id === "wrk_a" ? "dev_p" : "dev_q")], - })); - - await ask(coordinator, { physical: true, platform: "ios" }); - - expect(listed(coordinator)).toEqual([ - { reason: "no-free-physical", workerId: "wrk_a" }, - { reason: "no-free-physical", workerId: "wrk_b" }, - ]); - }); - - it("lists disconnected for a target found unreachable at the attempt, after the request is queued again", async () => { - const { coordinator, directory } = fleet(["wrk_a"]); - directory.unreachable.add("wrk_a"); - - await ask(coordinator); - - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); - expect(listed(coordinator)).toEqual([{ reason: "disconnected", workerId: "wrk_a" }]); - }); - - describe("a worker that answers PROVISION_FAILED", () => { - const provisionFailed = (message: string) => ({ - error: new SimlockError("PROVISION_FAILED", "domain", message, {}), - kind: "error" as const, - }); - - it("is walked past: the next worker is asked and its grant settles the request", async () => { - const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - clients["wrk_b"]?.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { outcome, state } = await ask(coordinator); - - expect(state.state).toBe("fulfilled"); - expect((await outcome).lease.worker?.id).toBe("wrk_b"); - }); - - it("rejects the request with PROVISION_FAILED and the last worker's message, and lists nothing, when every worker that could take it answered so", async () => { - const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - clients["wrk_b"]?.requestLeaseQueue.push(provisionFailed("worker b: boom")); - - const { ended, state } = await ask(coordinator); - - expect(state.state).toBe("rejected"); - expect(await ended).toMatchObject({ code: "PROVISION_FAILED", message: "worker b: boom" }); - expect(coordinator.waitingRequests()).toEqual([]); - expect(coordinator.queueDepth).toBe(0); - }); - - it("leaves the request queued when the other worker is only full", async () => { - const { clients, coordinator } = fleet(["wrk_a", "wrk_b"]); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - clients["wrk_b"]?.requestLeaseQueue.push(noCapacity()); - - const { state } = await ask(coordinator); - - expect(state.state).toBe("pending"); - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); - expect(listed(coordinator)).toEqual([ - { reason: "not-chosen", workerId: "wrk_a" }, - { reason: "full", workerId: "wrk_b" }, - ]); - }); - - it.each([ - { - build: () => { - const built = fleet(["wrk_a"]); - built.workers.connected("wrk_b", undefined, "0.3.0"); - return built; - }, - row: "not ready, having read no capacity or catalog yet", - }, - { - build: () => { - const built = fleet(["wrk_a", "wrk_b"]); - built.workers.refresh("wrk_b", { health: "starting" }); - return built; - }, - row: "unhealthy, still starting", - }, - { - build: () => fleet(["wrk_a", "wrk_b"], (id) => (id === "wrk_b" ? { queueDepth: 1 } : {})), - row: "busy with a queue of its own", - }, - { - build: () => { - const built = fleet(["wrk_a", "wrk_b"]); - built.workers.refresh("wrk_b", { - update: { - forced: false, - from: "0.3.0", - stage: "installing", - startedAt: 1, - to: "0.4.0", - }, - }); - return built; - }, - row: "updating", - }, - ])("leaves the request queued when the other worker is $row", async ({ build }) => { - const { clients, coordinator } = build(); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - - const { state } = await ask(coordinator); - - expect(state.state).toBe("pending"); - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); - }); - - it.each([ - { - build: () => { - const built = fleet(["wrk_a", "wrk_b"]); - built.workers.disconnected("wrk_b"); - return built; - }, - row: "disconnected", - }, - { - build: async () => { - const built = fleet(["wrk_a", "wrk_b"]); - await built.workers.setDrained("wrk_b", true); - return built; - }, - row: "drained", - }, - ])( - "rejects the request when the other worker is $row, which cannot take it", - async ({ build }) => { - const { clients, coordinator } = await build(); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - - const { ended, state } = await ask(coordinator); - - expect(state.state).toBe("rejected"); - expect(await ended).toMatchObject({ code: "PROVISION_FAILED", message: "worker a: boom" }); - }, - ); - - it("leaves the request queued when the other worker is full in its view, so the gateway never asks it", async () => { - const { clients, coordinator } = fleet(["wrk_a", "wrk_b"], (id) => - id === "wrk_b" ? { capacity: saturatedIos() } : {}, - ); - clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); - const untouched = noCapacity(); - clients["wrk_b"]?.requestLeaseQueue.push(untouched); - - const { state } = await ask(coordinator); - - expect(state.state).toBe("pending"); - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); - expect(clients["wrk_b"]?.requestLeaseQueue).toEqual([untouched]); - }); - - it("leaves the request queued when the worker that is only full refused first and the one asked after a view change failed to provision", async () => { - const { clients, coordinator, workers } = fleet(["wrk_a", "wrk_b"]); - clients["wrk_a"]?.requestLeaseQueue.push(noCapacity()); - clients["wrk_b"]?.requestLeaseQueue.push(provisionFailed("worker b: boom")); - const { state } = await ask(coordinator); - - workers.refresh("wrk_b", { capacity: roomierIos() }); - await tick(); - - expect(state.state).toBe("pending"); - expect(coordinator.waitingRequests()).toEqual([expect.objectContaining({ stage: "queued" })]); - }); - }); -}); - -describe("FleetLeaseCoordinator: a lease.cancel sent right after its lease.request (#1492)", () => { - it.each([ - { fleetHas: "no room, so the request would queue", room: false }, - { fleetHas: "a free worker, so the request would be dispatched", room: true }, - ])( - "with $fleetHas, answers cancelled, rejects the request as cancelled, and leaves nothing waiting and nothing dispatched", - async ({ room }) => { - const fleet = buildFleet(); - const client = new ScriptedWorkerClient(); - fleet.directory.add("wrk_a", client); - connectWorker(fleet.workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - await fleet.coordinator.request(REQUEST, requestOptions()); - connectWorker(fleet.workers, "wrk_a", { capacity: room ? roomierIos() : saturatedIos() }); - client.requestLeaseQueue.push({ - grant: grantFixture({ lease: leaseFixture("lease-2", "dev_2") }), - kind: "grant", - }); - - const request = fleet.coordinator - .request(REQUEST, requestOptions({ ownerId: "agent-2", requesterId: "agent-2" })) - .then( - () => "granted", - (error: unknown) => error, - ); - const cancel = fleet.coordinator.cancelPending("agent-2"); - - await expect(cancel).resolves.toBe("cancelled"); - await expect(request).resolves.toMatchObject({ name: "RequestCancelledError" }); - await tick(); - expect(fleet.coordinator.waitingRequests()).toEqual([]); - expect( - client.calls.filter((call) => call === `lease.request:${GATEWAY_PREFIX}agent-2`), - ).toEqual([]); - }, - ); -}); From 56c585f03f389f1271ac7e1950d1d87e48f8c83e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:22:00 +0100 Subject: [PATCH 21/83] test(gateway): split worker-registry.test.ts by topic --- src/gateway/test-worker-registry.ts | 46 ++ ...orker-registry-load-and-disconnect.test.ts | 348 +++++++++++++ .../worker-registry-views-changed.test.ts | 99 ++++ src/gateway/worker-registry.test.ts | 486 +----------------- 4 files changed, 497 insertions(+), 482 deletions(-) create mode 100644 src/gateway/test-worker-registry.ts create mode 100644 src/gateway/worker-registry-load-and-disconnect.test.ts create mode 100644 src/gateway/worker-registry-views-changed.test.ts diff --git a/src/gateway/test-worker-registry.ts b/src/gateway/test-worker-registry.ts new file mode 100644 index 000000000..b44b685db --- /dev/null +++ b/src/gateway/test-worker-registry.ts @@ -0,0 +1,46 @@ +import { EventBus, type EventEnvelope } from "../bus/index.js"; +import { FakeClock, type Logger } from "../ports/index.js"; +import { MemoryDrainStore } from "./drain-store.js"; +import { WorkerRegistry } from "./worker-registry.js"; + +export const RETENTION_MS = 24 * 60 * 60_000; + +/** Matches `core/config.ts`'s own default, so a test that does not care about ADR 0005 §15's + * warning gets a cap no fixture's `ttlMs` ever comes near. */ +export const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; + +/** Matches `service.test.ts`'s `principal: "gw:instance-1"` -- ADR 0005 §14/§27's own-lease + * prefix is that principal plus a trailing `:`. */ +export const GATEWAY_REQUESTER_PREFIX = "gw:instance-1:"; + +export function registry( + options: { + readonly drainStore?: MemoryDrainStore; + /** Omit to get the default fixture prefix; pass `null` explicitly (H3) to build a registry + * with no `gatewayRequesterPrefix` at all, the way `dispatcher.test.ts`'s harness does. */ + readonly gatewayRequesterPrefix?: string | null; + /** ADR 0005 §15's own cap. Defaults to `core/config.ts`'s own default so a test that never + * reports a worker `lease.maxTtlMs` cannot accidentally trip the new warning. */ + readonly leaseMaxTtlMs?: number; + readonly logger?: Logger; + } = {}, +) { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const events: EventEnvelope[] = []; + eventBus.subscribeAll((envelope) => events.push(envelope)); + const gatewayRequesterPrefix = + options.gatewayRequesterPrefix === null + ? undefined + : (options.gatewayRequesterPrefix ?? GATEWAY_REQUESTER_PREFIX); + const workers = new WorkerRegistry({ + clock, + eventBus, + leaseMaxTtlMs: options.leaseMaxTtlMs ?? DEFAULT_LEASE_MAX_TTL_MS, + retentionMs: RETENTION_MS, + ...(gatewayRequesterPrefix === undefined ? {} : { gatewayRequesterPrefix }), + ...(options.drainStore === undefined ? {} : { drainStore: options.drainStore }), + ...(options.logger === undefined ? {} : { logger: options.logger }), + }); + return { clock, events, workers }; +} diff --git a/src/gateway/worker-registry-load-and-disconnect.test.ts b/src/gateway/worker-registry-load-and-disconnect.test.ts new file mode 100644 index 000000000..bb6a8a2ce --- /dev/null +++ b/src/gateway/worker-registry-load-and-disconnect.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, it } from "vitest"; +import { PROTOCOL_VERSION_RANGE } from "../contract/index.js"; +import { + catalogFixture, + deviceFixture, + hostFixture, + leaseFixture, + physicalDeviceFixture, + statusFixture, +} from "./test-support.js"; +import { RETENTION_MS, registry } from "./test-worker-registry.js"; + +describe("WorkerRegistry", () => { + describe("a worker's load on its view", () => { + const load = { + cpu: { busyPercent: 34, cores: 10, limited: false }, + readAt: 900, + }; + const incompatible = (workers: ReturnType["workers"], workerId: string) => + workers.incompatible( + workerId, + undefined, + { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, + "0.2.0", + ); + + it("the registry drops a worker's load when it disconnects, turns incompatible, or connects again", () => { + const { workers } = registry(); + for (const id of ["wrk_1", "wrk_2", "wrk_3"]) { + workers.connected(id, undefined, "0.3.0"); + workers.refresh(id, { load }); + } + expect(workers.view("wrk_1")?.load).toEqual(load); + + workers.disconnected("wrk_1"); + const turned = incompatible(workers, "wrk_2"); + const reconnected = workers.connected("wrk_3", undefined, "0.3.0"); + + expect(workers.view("wrk_1")).not.toHaveProperty("load"); + expect(turned).not.toHaveProperty("load"); + expect(workers.view("wrk_2")).not.toHaveProperty("load"); + expect(reconnected).not.toHaveProperty("load"); + expect(workers.view("wrk_3")).not.toHaveProperty("load"); + }); + + it("a refresh without load leaves the view without load, after one with load", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { load }); + expect(workers.view("wrk_1")?.load).toEqual(load); + + workers.refresh("wrk_1", { queueDepth: 1 }); + + expect(workers.view("wrk_1")?.queueDepth).toBe(1); + expect(workers.view("wrk_1")).not.toHaveProperty("load"); + }); + + it("a refresh that reaches a disconnected or an incompatible view leaves no load on it", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.connected("wrk_2", undefined, "0.3.0"); + workers.refresh("wrk_1", { load }); + workers.disconnected("wrk_1"); + incompatible(workers, "wrk_2"); + + workers.refresh("wrk_1", { load }); + workers.refresh("wrk_2", { load }); + + expect(workers.view("wrk_1")?.connection).toBe("disconnected"); + expect(workers.view("wrk_1")).not.toHaveProperty("load"); + expect(workers.view("wrk_2")?.connection).toBe("incompatible"); + expect(workers.view("wrk_2")).not.toHaveProperty("load"); + }); + }); + + it("drops a worker's installs from its view when it disconnects or turns incompatible", () => { + const { workers } = registry(); + const installs = [ + { + component: "26.4", + platform: "ios" as const, + since: 1, + state: "downloading" as const, + waiters: 1, + }, + ]; + workers.connected("wrk_1", undefined, "0.3.0"); + workers.connected("wrk_2", undefined, "0.3.0"); + workers.refresh("wrk_1", { installs }); + workers.refresh("wrk_2", { installs }); + + workers.disconnected("wrk_1"); + const incompatible = workers.incompatible( + "wrk_2", + undefined, + { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, + "0.2.0", + ); + + expect(workers.view("wrk_1")?.installs).toBeUndefined(); + expect(incompatible.installs).toBeUndefined(); + expect(workers.view("wrk_2")?.installs).toBeUndefined(); + }); + + describe("a worker's warm pool block", () => { + const warmPool = { + enabled: true, + reserveRunning: { android: 0, ios: 1 }, + targets: [], + }; + + it("is on the view after a refresh, and stays on a disconnected worker's view as it was last read", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, "0.3.0"); + + workers.refresh("wrk_1", { warmPool }); + workers.disconnected("wrk_1"); + + expect(workers.view("wrk_1")?.warmPool).toEqual(warmPool); + }); + + it("is absent for a worker whose status was never read", () => { + const { workers } = registry(); + + expect(workers.connected("wrk_1", undefined, "0.3.0").warmPool).toBeUndefined(); + }); + + it("is dropped from the view when the worker turns incompatible or starts over", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.connected("wrk_2", undefined, "0.3.0"); + workers.refresh("wrk_1", { warmPool }); + workers.refresh("wrk_2", { warmPool }); + + const incompatible = workers.incompatible( + "wrk_1", + undefined, + { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, + "0.2.0", + ); + workers.refresh("wrk_2", { health: "starting" }); + + expect(incompatible.warmPool).toBeUndefined(); + expect(workers.view("wrk_1")?.warmPool).toBeUndefined(); + expect(workers.view("wrk_2")?.warmPool).toBeUndefined(); + }); + }); + + it("drops a worker's waiting requests from its view when it disconnects or turns incompatible", () => { + const { workers } = registry(); + const waiting = [ + { + createdAt: 1, + id: "req_1", + queuePosition: 1, + requesterId: "local-agent", + spec: { model: "iPhone 17", platform: "ios" as const }, + stage: "queued" as const, + }, + ]; + workers.connected("wrk_1", undefined, "0.3.0"); + workers.connected("wrk_2", undefined, "0.3.0"); + workers.refresh("wrk_1", { waiting }); + workers.refresh("wrk_2", { waiting }); + expect(workers.view("wrk_1")?.waiting).toEqual(waiting); + + workers.disconnected("wrk_1"); + const incompatible = workers.incompatible( + "wrk_2", + undefined, + { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, + "0.2.0", + ); + + expect(workers.view("wrk_1")?.waiting).toBeUndefined(); + expect(incompatible.waiting).toBeUndefined(); + expect(workers.view("wrk_2")?.waiting).toBeUndefined(); + }); + + it("keeps a disconnected view with the leases it last reported", () => { + const { clock, events, workers } = registry(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); + + clock.advance(1_000); + workers.disconnected("wrk_1"); + + expect(workers.view("wrk_1")).toMatchObject({ + connection: "disconnected", + lastSeenAt: 2_000, + leases: [{ id: "lease_1" }], + }); + expect(events.at(-1)).toMatchObject({ + event: "worker.disconnected", + payload: { leaseCount: 1, label: "mac-mini-1", workerId: "wrk_1" }, + }); + }); + + it("reports no leases for a worker that disconnects while starting when none was ever read from it, and lists none on its view", () => { + const { events, workers } = registry(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + workers.disconnected("wrk_1"); + + expect(events.at(-1)).toMatchObject({ + event: "worker.disconnected", + payload: { leaseCount: 0, workerId: "wrk_1" }, + }); + expect(Object.keys(workers.view("wrk_1") ?? {})).not.toContain("leases"); + }); + + it("reports the leases last read for a worker that disconnects while starting, though its view lists none", () => { + const { events, workers } = registry(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + workers.refresh("wrk_1", { + leases: [leaseFixture("lease_1", "dev_1"), leaseFixture("lease_2", "dev_2")], + }); + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + workers.disconnected("wrk_1"); + + expect(events.at(-1)).toMatchObject({ + event: "worker.disconnected", + payload: { leaseCount: 2, workerId: "wrk_1" }, + }); + expect(Object.keys(workers.view("wrk_1") ?? {})).not.toContain("leases"); + }); + + describe("a starting refresh", () => { + const running = { + capacity: statusFixture().capacity, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + devices: [], + health: "running" as const, + host: hostFixture(), + installs: [], + leases: [leaseFixture("lease_1", "dev_1")], + physicalQueueDepth: 1, + queueDepth: 0, + waiting: [], + }; + + it("drops capacity, catalog, catalogReadAt, devices, installs, leases, queueDepth, physicalQueueDepth and waiting a running read filled in, on a worker that stays connected", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", running); + workers.refresh("wrk_1", { catalog: running.catalog }); + + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(Object.keys(workers.view("wrk_1") ?? {}).sort()).toEqual([ + "connection", + "drained", + "health", + "host", + "id", + "lastSeenAt", + ]); + }); + + it("keeps the last catalog for routing while the view leaves it out", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", running); + + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(workers.view("wrk_1")?.catalog).toBeUndefined(); + expect(workers.routingViews()[0]?.catalog).toEqual(running.catalog); + expect(workers.routingViews()[0]?.catalogReadAt).toBeUndefined(); + }); + + it("carries the physical devices last read from a starting worker in its routing view, and its views() entry lists no devices", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + const phone = physicalDeviceFixture("dev_p", "ready"); + workers.refresh("wrk_1", { + ...running, + devices: [phone, deviceFixture("dev_sim", "ready")], + }); + + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(workers.routingViews()[0]?.devices).toEqual([phone]); + expect(workers.view("wrk_1")?.devices).toBeUndefined(); + expect(workers.views()[0]).not.toHaveProperty("devices"); + }); + + it("drops a removed worker's last physical devices, so one that comes back starting has none in its routing view", async () => { + const { clock, workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { ...running, devices: [physicalDeviceFixture("dev_p", "ready")] }); + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + expect(workers.routingViews()[0]?.devices).toHaveLength(1); + workers.disconnected("wrk_1"); + clock.advance(RETENTION_MS); + await workers.pruneExpired(); + expect(workers.routingViews()).toEqual([]); + + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(workers.routingViews()[0]?.devices).toBeUndefined(); + }); + + it("has no catalog for routing from a worker whose catalog was never read", () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(Object.keys(workers.routingViews()[0] ?? {})).not.toContain("catalog"); + }); + + it("forgets the last catalog and leases with the view, so a worker that comes back starting is unknown", async () => { + const { clock, events, workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", running); + workers.disconnected("wrk_1"); + clock.advance(RETENTION_MS); + await workers.pruneExpired(); + expect(workers.view("wrk_1")).toBeUndefined(); + + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); + + expect(workers.routingViews()[0]?.catalog).toBeUndefined(); + workers.disconnected("wrk_1"); + expect(events.at(-1)).toMatchObject({ + event: "worker.disconnected", + payload: { leaseCount: 0 }, + }); + }); + }); + + it("says nothing when a view that is already disconnected disconnects again", () => { + const { events, workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + workers.disconnected("wrk_1"); + const before = events.length; + + workers.disconnected("wrk_1"); + + expect(events).toHaveLength(before); + }); +}); diff --git a/src/gateway/worker-registry-views-changed.test.ts b/src/gateway/worker-registry-views-changed.test.ts new file mode 100644 index 000000000..23ddd2285 --- /dev/null +++ b/src/gateway/worker-registry-views-changed.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { RETENTION_MS, registry } from "./test-worker-registry.js"; + +describe("WorkerRegistry", () => { + it("orders views by id, so two reads agree", () => { + const { workers } = registry(); + workers.connected("wrk_c", undefined, undefined); + workers.connected("wrk_a", undefined, undefined); + workers.connected("wrk_b", undefined, undefined); + + expect(workers.views().map((view) => view.id)).toEqual(["wrk_a", "wrk_b", "wrk_c"]); + }); + + // `./fleet-ports.ts`'s `FleetViews#onViewsChanged` -- #118's seam. This registry has no + // listeners of its own; these tests are the notifier's whole contract. + describe("onViewsChanged", () => { + it("fires once per committed change, after the view is committed", () => { + const { workers } = registry(); + const seen: Array = []; + workers.onViewsChanged(() => seen.push(workers.view("wrk_1")?.connection)); + + workers.connected("wrk_1", undefined, undefined); + + expect(seen).toEqual(["connected"]); + }); + + it("does not fire for a call that changes nothing", async () => { + const { workers } = registry(); + workers.connected("wrk_1", undefined, undefined); + let fired = 0; + workers.onViewsChanged(() => { + fired += 1; + }); + + // Draining an already-undrained-to-false worker: `setDrained` no-ops. + await workers.setDrained("wrk_1", false); + // A refresh for a worker with no view: dropped. + workers.refresh("wrk_missing", { queueDepth: 1 }); + // Disconnecting an already-disconnected view: no-op. + workers.disconnected("wrk_1"); + workers.disconnected("wrk_1"); + + expect(fired).toBe(1); // only the first `disconnected()` -- the second is the no-op repeat. + }); + + it("stops firing once unsubscribed", () => { + const { workers } = registry(); + let fired = 0; + const unsubscribe = workers.onViewsChanged(() => { + fired += 1; + }); + + workers.connected("wrk_1", undefined, undefined); + unsubscribe(); + workers.disconnected("wrk_1"); + + expect(fired).toBe(1); + }); + + it("does not let a throwing listener break the mutation or starve the other listeners", () => { + const { workers } = registry(); + let afterFired = 0; + workers.onViewsChanged(() => { + throw new Error("a subscriber's own bug"); + }); + workers.onViewsChanged(() => { + afterFired += 1; + }); + + expect(() => workers.connected("wrk_1", undefined, undefined)).not.toThrow(); + + // The mutation itself committed... + expect(workers.view("wrk_1")?.connection).toBe("connected"); + // ...and the listener registered after the throwing one still ran. + expect(afterFired).toBe(1); + }); + + it("fires for remove() and pruneExpired(), each forgetting a view through #forget", async () => { + const { clock, workers } = registry(); + let fired = 0; + workers.onViewsChanged(() => { + fired += 1; + }); + + workers.connected("wrk_1", undefined, undefined); // 1 + workers.disconnected("wrk_1"); // 2 + await workers.remove("wrk_1"); // 3 + + workers.connected("wrk_2", undefined, undefined); // 4 + workers.disconnected("wrk_2"); // 5 + clock.advance(RETENTION_MS + 1); + await workers.pruneExpired(); // 6 + + expect(fired).toBe(6); + expect(workers.view("wrk_1")).toBeUndefined(); + expect(workers.view("wrk_2")).toBeUndefined(); + }); + }); +}); diff --git a/src/gateway/worker-registry.test.ts b/src/gateway/worker-registry.test.ts index f6a297a75..7dded213f 100644 --- a/src/gateway/worker-registry.test.ts +++ b/src/gateway/worker-registry.test.ts @@ -1,26 +1,10 @@ import { describe, expect, it } from "vitest"; - -import { EventBus, type EventEnvelope } from "../bus/index.js"; +import { type EventEnvelope } from "../bus/index.js"; import { PROTOCOL_VERSION_RANGE } from "../contract/index.js"; -import { FakeClock, type Logger } from "../ports/index.js"; +import { type Logger } from "../ports/index.js"; import { MemoryDrainStore } from "./drain-store.js"; -import { - catalogFixture, - deviceFixture, - hostFixture, - leaseFixture, - physicalDeviceFixture, - statusFixture, -} from "./test-support.js"; -import { WorkerRegistry } from "./worker-registry.js"; - -const RETENTION_MS = 24 * 60 * 60_000; -/** Matches `core/config.ts`'s own default, so a test that does not care about ADR 0005 §15's - * warning gets a cap no fixture's `ttlMs` ever comes near. */ -const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; -/** Matches `service.test.ts`'s `principal: "gw:instance-1"` -- ADR 0005 §14/§27's own-lease - * prefix is that principal plus a trailing `:`. */ -const GATEWAY_REQUESTER_PREFIX = "gw:instance-1:"; +import { catalogFixture, hostFixture, leaseFixture } from "./test-support.js"; +import { RETENTION_MS, GATEWAY_REQUESTER_PREFIX, registry } from "./test-worker-registry.js"; /** Records every `warn` call, the same shape `service.test.ts`'s own `RecordingLogger` uses -- * §15's warning is asserted on directly rather than only on the view it leaves behind. */ @@ -38,38 +22,6 @@ class RecordingLogger implements Logger { } } -function registry( - options: { - readonly drainStore?: MemoryDrainStore; - /** Omit to get the default fixture prefix; pass `null` explicitly (H3) to build a registry - * with no `gatewayRequesterPrefix` at all, the way `dispatcher.test.ts`'s harness does. */ - readonly gatewayRequesterPrefix?: string | null; - /** ADR 0005 §15's own cap. Defaults to `core/config.ts`'s own default so a test that never - * reports a worker `lease.maxTtlMs` cannot accidentally trip the new warning. */ - readonly leaseMaxTtlMs?: number; - readonly logger?: Logger; - } = {}, -) { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const events: EventEnvelope[] = []; - eventBus.subscribeAll((envelope) => events.push(envelope)); - const gatewayRequesterPrefix = - options.gatewayRequesterPrefix === null - ? undefined - : (options.gatewayRequesterPrefix ?? GATEWAY_REQUESTER_PREFIX); - const workers = new WorkerRegistry({ - clock, - eventBus, - leaseMaxTtlMs: options.leaseMaxTtlMs ?? DEFAULT_LEASE_MAX_TTL_MS, - retentionMs: RETENTION_MS, - ...(gatewayRequesterPrefix === undefined ? {} : { gatewayRequesterPrefix }), - ...(options.drainStore === undefined ? {} : { drainStore: options.drainStore }), - ...(options.logger === undefined ? {} : { logger: options.logger }), - }); - return { clock, events, workers }; -} - /** A gateway-issued lease's `requesterId` (ADR 0005 §14/§27): the fixed prefix `pruneExpired` * scopes its retention hold to, so a fixture built with it exercises the real branch rather than * one that happens to pass because no scoping exists yet. */ @@ -388,341 +340,6 @@ describe("WorkerRegistry", () => { expect(view.protocol).toBeUndefined(); }); - describe("a worker's load on its view", () => { - const load = { - cpu: { busyPercent: 34, cores: 10, limited: false }, - readAt: 900, - }; - const incompatible = (workers: ReturnType["workers"], workerId: string) => - workers.incompatible( - workerId, - undefined, - { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, - "0.2.0", - ); - - it("the registry drops a worker's load when it disconnects, turns incompatible, or connects again", () => { - const { workers } = registry(); - for (const id of ["wrk_1", "wrk_2", "wrk_3"]) { - workers.connected(id, undefined, "0.3.0"); - workers.refresh(id, { load }); - } - expect(workers.view("wrk_1")?.load).toEqual(load); - - workers.disconnected("wrk_1"); - const turned = incompatible(workers, "wrk_2"); - const reconnected = workers.connected("wrk_3", undefined, "0.3.0"); - - expect(workers.view("wrk_1")).not.toHaveProperty("load"); - expect(turned).not.toHaveProperty("load"); - expect(workers.view("wrk_2")).not.toHaveProperty("load"); - expect(reconnected).not.toHaveProperty("load"); - expect(workers.view("wrk_3")).not.toHaveProperty("load"); - }); - - it("a refresh without load leaves the view without load, after one with load", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { load }); - expect(workers.view("wrk_1")?.load).toEqual(load); - - workers.refresh("wrk_1", { queueDepth: 1 }); - - expect(workers.view("wrk_1")?.queueDepth).toBe(1); - expect(workers.view("wrk_1")).not.toHaveProperty("load"); - }); - - it("a refresh that reaches a disconnected or an incompatible view leaves no load on it", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.connected("wrk_2", undefined, "0.3.0"); - workers.refresh("wrk_1", { load }); - workers.disconnected("wrk_1"); - incompatible(workers, "wrk_2"); - - workers.refresh("wrk_1", { load }); - workers.refresh("wrk_2", { load }); - - expect(workers.view("wrk_1")?.connection).toBe("disconnected"); - expect(workers.view("wrk_1")).not.toHaveProperty("load"); - expect(workers.view("wrk_2")?.connection).toBe("incompatible"); - expect(workers.view("wrk_2")).not.toHaveProperty("load"); - }); - }); - - it("drops a worker's installs from its view when it disconnects or turns incompatible", () => { - const { workers } = registry(); - const installs = [ - { - component: "26.4", - platform: "ios" as const, - since: 1, - state: "downloading" as const, - waiters: 1, - }, - ]; - workers.connected("wrk_1", undefined, "0.3.0"); - workers.connected("wrk_2", undefined, "0.3.0"); - workers.refresh("wrk_1", { installs }); - workers.refresh("wrk_2", { installs }); - - workers.disconnected("wrk_1"); - const incompatible = workers.incompatible( - "wrk_2", - undefined, - { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, - "0.2.0", - ); - - expect(workers.view("wrk_1")?.installs).toBeUndefined(); - expect(incompatible.installs).toBeUndefined(); - expect(workers.view("wrk_2")?.installs).toBeUndefined(); - }); - - describe("a worker's warm pool block", () => { - const warmPool = { - enabled: true, - reserveRunning: { android: 0, ios: 1 }, - targets: [], - }; - - it("is on the view after a refresh, and stays on a disconnected worker's view as it was last read", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, "0.3.0"); - - workers.refresh("wrk_1", { warmPool }); - workers.disconnected("wrk_1"); - - expect(workers.view("wrk_1")?.warmPool).toEqual(warmPool); - }); - - it("is absent for a worker whose status was never read", () => { - const { workers } = registry(); - - expect(workers.connected("wrk_1", undefined, "0.3.0").warmPool).toBeUndefined(); - }); - - it("is dropped from the view when the worker turns incompatible or starts over", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.connected("wrk_2", undefined, "0.3.0"); - workers.refresh("wrk_1", { warmPool }); - workers.refresh("wrk_2", { warmPool }); - - const incompatible = workers.incompatible( - "wrk_1", - undefined, - { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, - "0.2.0", - ); - workers.refresh("wrk_2", { health: "starting" }); - - expect(incompatible.warmPool).toBeUndefined(); - expect(workers.view("wrk_1")?.warmPool).toBeUndefined(); - expect(workers.view("wrk_2")?.warmPool).toBeUndefined(); - }); - }); - - it("drops a worker's waiting requests from its view when it disconnects or turns incompatible", () => { - const { workers } = registry(); - const waiting = [ - { - createdAt: 1, - id: "req_1", - queuePosition: 1, - requesterId: "local-agent", - spec: { model: "iPhone 17", platform: "ios" as const }, - stage: "queued" as const, - }, - ]; - workers.connected("wrk_1", undefined, "0.3.0"); - workers.connected("wrk_2", undefined, "0.3.0"); - workers.refresh("wrk_1", { waiting }); - workers.refresh("wrk_2", { waiting }); - expect(workers.view("wrk_1")?.waiting).toEqual(waiting); - - workers.disconnected("wrk_1"); - const incompatible = workers.incompatible( - "wrk_2", - undefined, - { gateway: PROTOCOL_VERSION_RANGE, worker: { min: 4, max: 4 } }, - "0.2.0", - ); - - expect(workers.view("wrk_1")?.waiting).toBeUndefined(); - expect(incompatible.waiting).toBeUndefined(); - expect(workers.view("wrk_2")?.waiting).toBeUndefined(); - }); - - it("keeps a disconnected view with the leases it last reported", () => { - const { clock, events, workers } = registry(); - workers.connected("wrk_1", "mac-mini-1", "0.3.0"); - workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); - - clock.advance(1_000); - workers.disconnected("wrk_1"); - - expect(workers.view("wrk_1")).toMatchObject({ - connection: "disconnected", - lastSeenAt: 2_000, - leases: [{ id: "lease_1" }], - }); - expect(events.at(-1)).toMatchObject({ - event: "worker.disconnected", - payload: { leaseCount: 1, label: "mac-mini-1", workerId: "wrk_1" }, - }); - }); - - it("reports no leases for a worker that disconnects while starting when none was ever read from it, and lists none on its view", () => { - const { events, workers } = registry(); - workers.connected("wrk_1", "mac-mini-1", "0.3.0"); - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - workers.disconnected("wrk_1"); - - expect(events.at(-1)).toMatchObject({ - event: "worker.disconnected", - payload: { leaseCount: 0, workerId: "wrk_1" }, - }); - expect(Object.keys(workers.view("wrk_1") ?? {})).not.toContain("leases"); - }); - - it("reports the leases last read for a worker that disconnects while starting, though its view lists none", () => { - const { events, workers } = registry(); - workers.connected("wrk_1", "mac-mini-1", "0.3.0"); - workers.refresh("wrk_1", { - leases: [leaseFixture("lease_1", "dev_1"), leaseFixture("lease_2", "dev_2")], - }); - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - workers.disconnected("wrk_1"); - - expect(events.at(-1)).toMatchObject({ - event: "worker.disconnected", - payload: { leaseCount: 2, workerId: "wrk_1" }, - }); - expect(Object.keys(workers.view("wrk_1") ?? {})).not.toContain("leases"); - }); - - describe("a starting refresh", () => { - const running = { - capacity: statusFixture().capacity, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - devices: [], - health: "running" as const, - host: hostFixture(), - installs: [], - leases: [leaseFixture("lease_1", "dev_1")], - physicalQueueDepth: 1, - queueDepth: 0, - waiting: [], - }; - - it("drops capacity, catalog, catalogReadAt, devices, installs, leases, queueDepth, physicalQueueDepth and waiting a running read filled in, on a worker that stays connected", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", running); - workers.refresh("wrk_1", { catalog: running.catalog }); - - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(Object.keys(workers.view("wrk_1") ?? {}).sort()).toEqual([ - "connection", - "drained", - "health", - "host", - "id", - "lastSeenAt", - ]); - }); - - it("keeps the last catalog for routing while the view leaves it out", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", running); - - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(workers.view("wrk_1")?.catalog).toBeUndefined(); - expect(workers.routingViews()[0]?.catalog).toEqual(running.catalog); - expect(workers.routingViews()[0]?.catalogReadAt).toBeUndefined(); - }); - - it("carries the physical devices last read from a starting worker in its routing view, and its views() entry lists no devices", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - const phone = physicalDeviceFixture("dev_p", "ready"); - workers.refresh("wrk_1", { - ...running, - devices: [phone, deviceFixture("dev_sim", "ready")], - }); - - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(workers.routingViews()[0]?.devices).toEqual([phone]); - expect(workers.view("wrk_1")?.devices).toBeUndefined(); - expect(workers.views()[0]).not.toHaveProperty("devices"); - }); - - it("drops a removed worker's last physical devices, so one that comes back starting has none in its routing view", async () => { - const { clock, workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { ...running, devices: [physicalDeviceFixture("dev_p", "ready")] }); - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - expect(workers.routingViews()[0]?.devices).toHaveLength(1); - workers.disconnected("wrk_1"); - clock.advance(RETENTION_MS); - await workers.pruneExpired(); - expect(workers.routingViews()).toEqual([]); - - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(workers.routingViews()[0]?.devices).toBeUndefined(); - }); - - it("has no catalog for routing from a worker whose catalog was never read", () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(Object.keys(workers.routingViews()[0] ?? {})).not.toContain("catalog"); - }); - - it("forgets the last catalog and leases with the view, so a worker that comes back starting is unknown", async () => { - const { clock, events, workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", running); - workers.disconnected("wrk_1"); - clock.advance(RETENTION_MS); - await workers.pruneExpired(); - expect(workers.view("wrk_1")).toBeUndefined(); - - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { health: "starting", host: hostFixture() }); - - expect(workers.routingViews()[0]?.catalog).toBeUndefined(); - workers.disconnected("wrk_1"); - expect(events.at(-1)).toMatchObject({ - event: "worker.disconnected", - payload: { leaseCount: 0 }, - }); - }); - }); - - it("says nothing when a view that is already disconnected disconnects again", () => { - const { events, workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - workers.disconnected("wrk_1"); - const before = events.length; - - workers.disconnected("wrk_1"); - - expect(events).toHaveLength(before); - }); - describe("retention", () => { it("forgets a disconnected view once the retention window passes", async () => { const { clock, events, workers } = registry(); @@ -1048,99 +665,4 @@ describe("WorkerRegistry", () => { }); }); }); - - it("orders views by id, so two reads agree", () => { - const { workers } = registry(); - workers.connected("wrk_c", undefined, undefined); - workers.connected("wrk_a", undefined, undefined); - workers.connected("wrk_b", undefined, undefined); - - expect(workers.views().map((view) => view.id)).toEqual(["wrk_a", "wrk_b", "wrk_c"]); - }); - - // `./fleet-ports.ts`'s `FleetViews#onViewsChanged` -- #118's seam. This registry has no - // listeners of its own; these tests are the notifier's whole contract. - describe("onViewsChanged", () => { - it("fires once per committed change, after the view is committed", () => { - const { workers } = registry(); - const seen: Array = []; - workers.onViewsChanged(() => seen.push(workers.view("wrk_1")?.connection)); - - workers.connected("wrk_1", undefined, undefined); - - expect(seen).toEqual(["connected"]); - }); - - it("does not fire for a call that changes nothing", async () => { - const { workers } = registry(); - workers.connected("wrk_1", undefined, undefined); - let fired = 0; - workers.onViewsChanged(() => { - fired += 1; - }); - - // Draining an already-undrained-to-false worker: `setDrained` no-ops. - await workers.setDrained("wrk_1", false); - // A refresh for a worker with no view: dropped. - workers.refresh("wrk_missing", { queueDepth: 1 }); - // Disconnecting an already-disconnected view: no-op. - workers.disconnected("wrk_1"); - workers.disconnected("wrk_1"); - - expect(fired).toBe(1); // only the first `disconnected()` -- the second is the no-op repeat. - }); - - it("stops firing once unsubscribed", () => { - const { workers } = registry(); - let fired = 0; - const unsubscribe = workers.onViewsChanged(() => { - fired += 1; - }); - - workers.connected("wrk_1", undefined, undefined); - unsubscribe(); - workers.disconnected("wrk_1"); - - expect(fired).toBe(1); - }); - - it("does not let a throwing listener break the mutation or starve the other listeners", () => { - const { workers } = registry(); - let afterFired = 0; - workers.onViewsChanged(() => { - throw new Error("a subscriber's own bug"); - }); - workers.onViewsChanged(() => { - afterFired += 1; - }); - - expect(() => workers.connected("wrk_1", undefined, undefined)).not.toThrow(); - - // The mutation itself committed... - expect(workers.view("wrk_1")?.connection).toBe("connected"); - // ...and the listener registered after the throwing one still ran. - expect(afterFired).toBe(1); - }); - - it("fires for remove() and pruneExpired(), each forgetting a view through #forget", async () => { - const { clock, workers } = registry(); - let fired = 0; - workers.onViewsChanged(() => { - fired += 1; - }); - - workers.connected("wrk_1", undefined, undefined); // 1 - workers.disconnected("wrk_1"); // 2 - await workers.remove("wrk_1"); // 3 - - workers.connected("wrk_2", undefined, undefined); // 4 - workers.disconnected("wrk_2"); // 5 - clock.advance(RETENTION_MS + 1); - await workers.pruneExpired(); // 6 - - expect(fired).toBe(6); - expect(workers.view("wrk_1")).toBeUndefined(); - expect(workers.view("wrk_2")).toBeUndefined(); - }); - }); }); From 4f1596b4ebfefeef0abb432af22037504633b056 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:23:22 +0100 Subject: [PATCH 22/83] test(gateway): split relay.test.ts by topic --- src/gateway/relay-failures.test.ts | 430 +++++++++++++++++++ src/gateway/relay.test.ts | 637 +---------------------------- src/gateway/test-relay.ts | 211 ++++++++++ 3 files changed, 656 insertions(+), 622 deletions(-) create mode 100644 src/gateway/relay-failures.test.ts create mode 100644 src/gateway/test-relay.ts diff --git a/src/gateway/relay-failures.test.ts b/src/gateway/relay-failures.test.ts new file mode 100644 index 000000000..49338b888 --- /dev/null +++ b/src/gateway/relay-failures.test.ts @@ -0,0 +1,430 @@ +import { describe, expect, it, vi } from "vitest"; +import type { LinkChannelErrorReason } from "../link/index.js"; +import { + WORKER_ID, + KIB, + sleep, + FakeChannel, + FakeLink, + setup, + call, + sentFrame, + outcome, + relayIdOf, + readFailure, + openedChannel, + openTwoWay, +} from "./test-relay.js"; + +async function readAll(response: Response) { + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("no body"); + let bytes = 0; + try { + for (;;) { + const step = await reader.read(); + if (step.done) return { bytes, state: "ended" as const }; + bytes += step.value.length; + } + } catch (error: unknown) { + return { + bytes, + code: (error as { code?: unknown } | null)?.code, + message: error instanceof Error ? error.message : String(error), + state: "failed" as const, + }; + } +} + +describe("GatewayRelay bounding the head and the body", () => { + it.each([ + { bound: "the deadline", deadlineMs: 100, idleMs: 10_000 }, + { bound: "the idle bound", deadlineMs: 10_000, idleMs: 100 }, + ])( + "keeps relaying a body whose bytes keep coming past $bound, and ends it at the channel's end", + async ({ deadlineMs, idleMs }) => { + const { connection, relay } = await setup({ deadlineMs, idleMs }); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + connection.reply(request.id, {}); + + const reading = readAll(response); + for (let piece = 0; piece < 12 && !channel.aborted; piece += 1) { + await sleep(30); + if (!channel.aborted) channel.feed(new Uint8Array(KIB)); + } + if (!channel.aborted) channel.end(); + + expect(await reading).toEqual({ bytes: 12 * KIB, state: "ended" }); + }, + ); + + it("ends a body that sends nothing for the idle bound with WORKER_UNREACHABLE, sends relay.cancel and aborts its channel", async () => { + const { connection, relay } = await setup({ deadlineMs: 10_000, idleMs: 100 }); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + connection.reply(request.id, {}); + + channel.feed(new Uint8Array(KIB)); + const result = await Promise.race([ + readAll(response), + sleep(2_000).then(() => ({ state: "still open" as const })), + ]); + + expect(result).toEqual({ + bytes: KIB, + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} sent nothing on the relayed response for 100 ms`, + state: "failed", + }); + const cancel = await sentFrame(connection, "relay.cancel"); + expect(relayIdOf(cancel)).toBe(relayIdOf(request)); + expect(channel.aborted).toBe(true); + }); + + it("fails a relay whose head never arrives at the deadline, not at the shorter idle bound", async () => { + const { connection, relay } = await setup({ deadlineMs: 150, idleMs: 50 }); + const pending = outcome(relay.relay(WORKER_ID, call())); + await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + const failure = await pending; + + expect(failure).toEqual({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} did not finish the relayed request within 150 ms`, + state: "rejected", + }); + expect(channel.aborted).toBe(true); + }); + + it("ends a two-way relay with WORKER_UNREACHABLE after 15 s with no byte from the worker, however much the gateway writes, and keeps it open while bytes come or the worker's direction is ended", async () => { + const silent = await openTwoWay(); + await silent.writer.write(new Uint8Array(8)); + silent.clock.advance(10_000); + await silent.writer.write(new Uint8Array(8)); + silent.clock.advance(4_999); + const openAt14 = silent.channel.aborted; + silent.clock.advance(1); + const abortedAt15 = silent.channel.aborted; + const silentRead = outcome(silent.reader.read()); + const writeFailure = outcome(silent.writer.write(new Uint8Array(1))); + + const chatty = await openTwoWay(); + const steps: number[] = []; + for (let at = 0; at < 12; at += 1) { + chatty.clock.advance(5_000); + chatty.channel.feed(new Uint8Array(1)); + steps.push((await chatty.reader.read()).value?.length ?? 0); + } + const chattyAborted = chatty.channel.aborted; + + const ended = await openTwoWay(); + ended.channel.end(); + const endStep = await ended.reader.read(); + ended.clock.advance(60_000); + await ended.writer.write(new Uint8Array(3)); + + const slowReader = await openTwoWay(); + slowReader.channel.feed(new Uint8Array(70 * KIB)); + await vi.waitFor(() => expect(slowReader.clock.pendingTimerCount).toBe(0)); + slowReader.clock.advance(60_000); + const slowAborted = slowReader.channel.aborted; + const slowStep = await slowReader.reader.read(); + slowReader.clock.advance(15_000); + const slowFailure = await outcome(slowReader.reader.read()); + + const message = `worker ${WORKER_ID} sent nothing on the relayed response for 15000 ms`; + expect(openAt14).toBe(false); + expect(abortedAt15).toBe(true); + expect(await silentRead).toMatchObject({ + code: "WORKER_UNREACHABLE", + message, + state: "rejected", + }); + expect(await writeFailure).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + expect(silent.channel.aborted).toBe(true); + expect(silent.connection.lastSentOf("relay.cancel")).toBeDefined(); + expect(steps).toEqual(Array.from({ length: 12 }, () => 1)); + expect(chattyAborted).toBe(false); + expect(endStep.done).toBe(true); + expect(ended.channel.aborted).toBe(false); + expect(ended.channel.written.map((chunk) => chunk.length)).toEqual([3]); + expect(slowAborted).toBe(false); + expect(slowStep.value?.length).toBe(70 * KIB); + expect(slowFailure).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + }); +}); + +describe("GatewayRelay mapping a channel failure", () => { + const before: readonly { + readonly reason: LinkChannelErrorReason; + readonly code: string; + readonly message: string; + }[] = [ + { + code: "INTERNAL", + message: `worker ${WORKER_ID} sent an invalid relay response`, + reason: "invalid", + }, + { + code: "INTERNAL", + message: `worker ${WORKER_ID} did not read the relayed request body for 60 s`, + reason: "stalled", + }, + { + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} did not take the relayed request within 30 s`, + reason: "not-accepted", + }, + ]; + + it.each(before)( + "fails $reason before the head with $code and the message the table names", + async ({ code, message, reason }) => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + channel.fail(reason); + + expect(await pending).toEqual({ code, message, state: "rejected" }); + }, + ); + + it("fails closed before the head with WORKER_UNREACHABLE", async () => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + channel.fail("closed"); + + expect(await pending).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + }); + + it("lets the relay.request answer decide when the peer aborts before the head", async () => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + channel.fail("aborted"); + connection.fail(request.id, "INTERNAL", "worker side exploded"); + + expect(await pending).toEqual({ + code: "INTERNAL", + message: "worker side exploded", + state: "rejected", + }); + }); + + it("fails with the deadline error when the peer aborts before the head and no answer comes", async () => { + const { connection, relay } = await setup({ deadlineMs: 150 }); + const pending = outcome(relay.relay(WORKER_ID, call())); + await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + + channel.fail("aborted"); + + expect(await pending).toEqual({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} did not finish the relayed request within 150 ms`, + state: "rejected", + }); + }); + + const after: readonly { + readonly name: string; + readonly run: (connection: FakeLink, channel: FakeChannel, id: number | string) => void; + readonly expected: Record; + }[] = [ + { + expected: { + message: `worker ${WORKER_ID} sent an invalid relay response`, + state: "rejected", + }, + name: "invalid", + run: (_connection, channel) => channel.fail("invalid"), + }, + { + expected: { code: "WORKER_UNREACHABLE", state: "rejected" }, + name: "closed", + run: (_connection, channel) => channel.fail("closed"), + }, + { + expected: { message: expect.stringMatching(/fell behind for 60 s/), state: "rejected" }, + name: "stalled", + run: (_connection, channel) => channel.fail("stalled"), + }, + { + expected: { message: "worker side exploded", state: "rejected" }, + name: "aborted by the peer, with a failed relay.request answer", + run: (connection, channel, id) => { + channel.fail("aborted"); + connection.fail(id, "INTERNAL", "worker side exploded"); + }, + }, + { + expected: { + message: `worker ${WORKER_ID} ended the relayed response early`, + state: "rejected", + }, + name: "aborted by the peer, with a relay.request that settles without an error", + run: (connection, channel, id) => { + channel.fail("aborted"); + connection.reply(id, {}); + }, + }, + ]; + + it.each(after)("errors the caller's body for $name after the head", async ({ expected, run }) => { + const { connection, relay } = await setup(); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + + run(connection, channel, request.id); + const read = await readFailure(response); + + expect(read).toMatchObject(expected); + }); +}); + +describe("GatewayRelay against a worker that breaks the protocol", () => { + it("fails the relay with INTERNAL naming the worker, and sends relay.cancel, on a relay-head with status 101", async () => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + const request = await sentFrame(connection, "relay.request"); + + connection.push("relay-head", { headers: [], requestId: request.id, status: 101 }); + + expect(await pending).toEqual({ + code: "INTERNAL", + message: `worker ${WORKER_ID} sent an invalid relay response`, + state: "rejected", + }); + const cancel = await sentFrame(connection, "relay.cancel"); + expect(relayIdOf(cancel)).toBe(relayIdOf(request)); + }); + + it("errors the caller's body with INTERNAL naming the worker, and sends relay.cancel, on a second relay-head", async () => { + const { connection, relay } = await setup(); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const read = await readFailure(response); + + expect(read).toEqual({ + code: "INTERNAL", + message: `worker ${WORKER_ID} sent an invalid relay response`, + state: "rejected", + }); + const cancel = await sentFrame(connection, "relay.cancel"); + expect(relayIdOf(cancel)).toBe(relayIdOf(request)); + }); + + it("fails the relay with INTERNAL naming the worker, and sends relay.cancel, on a relay-head whose header value `Headers` refuses", async () => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + const request = await sentFrame(connection, "relay.request"); + + connection.push("relay-head", { + headers: [["x-note", "€"]], + requestId: request.id, + status: 200, + }); + + expect(await pending).toEqual({ + code: "INTERNAL", + message: `worker ${WORKER_ID} sent an invalid relay response`, + state: "rejected", + }); + const cancel = await sentFrame(connection, "relay.cancel"); + expect(relayIdOf(cancel)).toBe(relayIdOf(request)); + }); + + it.each([ + { answer: {}, name: "{} with no head" }, + { answer: { cancelled: true }, name: "{ cancelled: true } when no cancel was sent" }, + ])( + "fails the relay with INTERNAL saying the worker ended it without a response when relay.request answers $name", + async ({ answer }) => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + const request = await sentFrame(connection, "relay.request"); + + connection.reply(request.id, answer); + + expect(await pending).toEqual({ + code: "INTERNAL", + message: `worker ${WORKER_ID} ended the relay without a response`, + state: "rejected", + }); + }, + ); + + it("errors the caller's body stream with the worker's message on an INTERNAL after the head", async () => { + const { connection, relay } = await setup(); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const response = await pending; + + connection.fail(request.id, "INTERNAL", "worker side exploded"); + + const read = await readFailure(response); + expect(read).toMatchObject({ message: "worker side exploded", state: "rejected" }); + }); + + it("reads DAEMON_STOPPING from a worker as a lost uplink: WORKER_UNREACHABLE", async () => { + const { connection, relay } = await setup(); + const pending = outcome(relay.relay(WORKER_ID, call())); + const request = await sentFrame(connection, "relay.request"); + + connection.fail(request.id, "DAEMON_STOPPING", "The daemon is stopping"); + + expect(await pending).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + }); + + it("drops every hop-by-hop and framing header of a worker's relay head, in any case, and keeps an ordinary header, on the response it hands the caller", async () => { + const { connection, relay } = await setup(); + const pending = relay.relay(WORKER_ID, call()); + const request = await sentFrame(connection, "relay.request"); + await openedChannel(connection); + + connection.push("relay-head", { + headers: [ + ["connection", "close"], + ["Content-Length", "5"], + ["keep-alive", "timeout=5"], + ["proxy-authenticate", "Basic realm=worker"], + ["proxy-authorization", "Basic worker-secret"], + ["te", "trailers"], + ["Te", "custom-checksum"], + ["trailer", "X-Note"], + ["TRANSFER-ENCODING", "chunked"], + ["upgrade", "h2c"], + ["x-kept", "yes"], + ], + requestId: request.id, + status: 200, + }); + const response = await pending; + + expect([...response.headers.entries()]).toEqual([["x-kept", "yes"]]); + }); +}); diff --git a/src/gateway/relay.test.ts b/src/gateway/relay.test.ts index 7214ba415..cd7d1b4e5 100644 --- a/src/gateway/relay.test.ts +++ b/src/gateway/relay.test.ts @@ -1,234 +1,20 @@ import { describe, expect, it, vi } from "vitest"; - import { LinkChannelError } from "../link/index.js"; -import type { LinkChannel, LinkChannelErrorReason, LinkConnection } from "../link/index.js"; -import { FakeClock, type Clock } from "../ports/index.js"; -import { connectSimlockClient } from "../simlock-client/client.js"; -import { completeHello, ScriptedConnection } from "../simlock-client/test-support.js"; -import type { SentFrame } from "../simlock-client/test-support.js"; -import type { WorkerDirectory } from "./fleet-ports.js"; -import { GatewayRelay, type TwoWayRelay } from "./index.js"; -import type { GatewayWorkerClient } from "./worker-client.js"; - -const WORKER_ID = "wrk_a"; -const KIB = 1024; - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class FakeChannel implements LinkChannel { - readonly readable: ReadableStream; - readonly writable: WritableStream; - readonly written: Uint8Array[] = []; - readonly credit = 0; - writableClosed = false; - aborted = false; - abortReason: string | undefined; - #readController!: ReadableStreamDefaultController; - #writeController!: WritableStreamDefaultController; - - constructor(readonly id: string) { - this.readable = new ReadableStream({ - start: (controller) => { - this.#readController = controller; - }, - }); - this.writable = new WritableStream({ - close: () => { - this.writableClosed = true; - }, - start: (controller) => { - this.#writeController = controller; - }, - write: (chunk) => { - this.written.push(chunk); - }, - }); - } - - whenCredit(): Promise { - return Promise.resolve(); - } - - abort(reason?: string): void { - this.aborted = true; - this.abortReason = reason; - try { - this.#readController.error(new LinkChannelError("aborted")); - } catch { - // why: the readable had already ended or failed. - } - } - - feed(bytes: Uint8Array): void { - this.#readController.enqueue(bytes); - } - - end(): void { - this.#readController.close(); - } - - fail(reason: LinkChannelErrorReason): void { - try { - this.#readController.error(new LinkChannelError(reason)); - } catch { - // why: already ended. - } - try { - this.#writeController.error(new LinkChannelError(reason)); - } catch { - // why: already closed. - } - } - - // why: the real channel errors its writable at once on a peer's STOP (src/link/channel.ts onStop → #failOutgoing). - stopWriting(): void { - this.#writeController.error(new LinkChannelError("stopped")); - } - - failWritable(reason: LinkChannelErrorReason): void { - this.#writeController.error(new LinkChannelError(reason)); - } -} - -class FakeLink extends ScriptedConnection implements LinkConnection { - readonly channels: FakeChannel[] = []; - holdOpen = false; - - openChannel(options: { readonly signal: AbortSignal }): Promise { - if (this.holdOpen) { - return new Promise((_resolve, reject) => { - const fail = () => reject(options.signal.reason as Error); - if (options.signal.aborted) fail(); - else options.signal.addEventListener("abort", fail, { once: true }); - }); - } - const channel = new FakeChannel(`chan-${String(this.channels.length + 1)}`); - this.channels.push(channel); - return Promise.resolve(channel); - } - - acceptChannel(id: string): LinkChannel { - throw new Error(`the gateway accepts no channel: ${id}`); - } -} - -async function setup( - options: { - readonly deadlineMs?: number; - readonly holdOpen?: boolean; - readonly idleMs?: number; - readonly clock?: Clock; - readonly proxyAuthHeader?: string; - } = {}, -) { - const connection = new FakeLink(); - connection.holdOpen = options.holdOpen ?? false; - const connecting = connectSimlockClient({ connection, credential: "secret" }, true); - await vi.waitFor(() => expect(connection.lastSentOf("hello")).toBeDefined()); - completeHello(connection, { role: "admin" }); - const client = (await connecting) as GatewayWorkerClient; - const directory: WorkerDirectory = { - target: (workerId) => - workerId === WORKER_ID - ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } - : undefined, - }; - return { - connection, - relay: new GatewayRelay({ - directory, - ...(options.deadlineMs === undefined ? {} : { deadlineMs: options.deadlineMs }), - ...(options.idleMs === undefined ? {} : { idleMs: options.idleMs }), - ...(options.clock === undefined ? {} : { clock: options.clock }), - ...(options.proxyAuthHeader === undefined - ? {} - : { proxyAuthHeader: options.proxyAuthHeader }), - }), - }; -} - -function call( - body?: ReadableStream, - signal?: AbortSignal, -): { request: Request; target: string; tenant: string } { - return { - request: new Request("http://gateway.test:4321/agent-device/rpc", { - method: body === undefined ? "GET" : "PUT", - ...(body === undefined ? {} : { body, duplex: "half" }), - ...(signal === undefined ? {} : { signal }), - } as RequestInit), - target: "echo", - tenant: "gw:g1:alice", - }; -} - -async function sentFrame(connection: ScriptedConnection, type: string): Promise { - await vi.waitFor(() => expect(connection.lastSentOf(type)).toBeDefined()); - return connection.lastSentOf(type) as SentFrame; -} - -async function outcome(promise: Promise) { - return promise.then( - (value) => ({ state: "fulfilled" as const, value }), - (error: unknown) => ({ - code: (error as { code?: unknown } | null)?.code, - message: error instanceof Error ? error.message : String(error), - state: "rejected" as const, - }), - ); -} - -function relayIdOf(frame: SentFrame): unknown { - return (frame.payload as { relayId?: unknown }).relayId; -} - -async function readFailure(response: Response) { - return outcome(response.body?.getReader().read() ?? Promise.reject(new Error("no body"))); -} - -async function readAll(response: Response) { - const reader = response.body?.getReader(); - if (reader === undefined) throw new Error("no body"); - let bytes = 0; - try { - for (;;) { - const step = await reader.read(); - if (step.done) return { bytes, state: "ended" as const }; - bytes += step.value.length; - } - } catch (error: unknown) { - return { - bytes, - code: (error as { code?: unknown } | null)?.code, - message: error instanceof Error ? error.message : String(error), - state: "failed" as const, - }; - } -} - -async function openedChannel(connection: FakeLink): Promise { - await vi.waitFor(() => expect(connection.channels).toHaveLength(1)); - return connection.channels[0] as FakeChannel; -} - -async function openTwoWay(signal?: AbortSignal) { - const clock = new FakeClock(); - const { connection, relay } = await setup({ clock }); - const pending = relay.relayTwoWay(WORKER_ID, call(undefined, signal)); - const request = await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const opened = (await pending) as TwoWayRelay; - return { - channel, - clock, - connection, - reader: opened.readable.getReader(), - writer: opened.writable.getWriter(), - }; -} +import { FakeClock } from "../ports/index.js"; +import { + WORKER_ID, + KIB, + sleep, + FakeLink, + setup, + call, + sentFrame, + outcome, + relayIdOf, + readFailure, + openedChannel, + openTwoWay, +} from "./test-relay.js"; describe("GatewayRelay on a byte channel", () => { it("with http.proxyAuth.header configured, relays neither that header nor cookie, keeps authorization and x-agent-device-token dropped, and passes other headers", async () => { @@ -729,396 +515,3 @@ describe("GatewayRelay when the worker resets a two-way relay's channel", () => }, ); }); - -describe("GatewayRelay bounding the head and the body", () => { - it.each([ - { bound: "the deadline", deadlineMs: 100, idleMs: 10_000 }, - { bound: "the idle bound", deadlineMs: 10_000, idleMs: 100 }, - ])( - "keeps relaying a body whose bytes keep coming past $bound, and ends it at the channel's end", - async ({ deadlineMs, idleMs }) => { - const { connection, relay } = await setup({ deadlineMs, idleMs }); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const response = await pending; - connection.reply(request.id, {}); - - const reading = readAll(response); - for (let piece = 0; piece < 12 && !channel.aborted; piece += 1) { - await sleep(30); - if (!channel.aborted) channel.feed(new Uint8Array(KIB)); - } - if (!channel.aborted) channel.end(); - - expect(await reading).toEqual({ bytes: 12 * KIB, state: "ended" }); - }, - ); - - it("ends a body that sends nothing for the idle bound with WORKER_UNREACHABLE, sends relay.cancel and aborts its channel", async () => { - const { connection, relay } = await setup({ deadlineMs: 10_000, idleMs: 100 }); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const response = await pending; - connection.reply(request.id, {}); - - channel.feed(new Uint8Array(KIB)); - const result = await Promise.race([ - readAll(response), - sleep(2_000).then(() => ({ state: "still open" as const })), - ]); - - expect(result).toEqual({ - bytes: KIB, - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} sent nothing on the relayed response for 100 ms`, - state: "failed", - }); - const cancel = await sentFrame(connection, "relay.cancel"); - expect(relayIdOf(cancel)).toBe(relayIdOf(request)); - expect(channel.aborted).toBe(true); - }); - - it("fails a relay whose head never arrives at the deadline, not at the shorter idle bound", async () => { - const { connection, relay } = await setup({ deadlineMs: 150, idleMs: 50 }); - const pending = outcome(relay.relay(WORKER_ID, call())); - await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - - const failure = await pending; - - expect(failure).toEqual({ - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} did not finish the relayed request within 150 ms`, - state: "rejected", - }); - expect(channel.aborted).toBe(true); - }); - - it("ends a two-way relay with WORKER_UNREACHABLE after 15 s with no byte from the worker, however much the gateway writes, and keeps it open while bytes come or the worker's direction is ended", async () => { - const silent = await openTwoWay(); - await silent.writer.write(new Uint8Array(8)); - silent.clock.advance(10_000); - await silent.writer.write(new Uint8Array(8)); - silent.clock.advance(4_999); - const openAt14 = silent.channel.aborted; - silent.clock.advance(1); - const abortedAt15 = silent.channel.aborted; - const silentRead = outcome(silent.reader.read()); - const writeFailure = outcome(silent.writer.write(new Uint8Array(1))); - - const chatty = await openTwoWay(); - const steps: number[] = []; - for (let at = 0; at < 12; at += 1) { - chatty.clock.advance(5_000); - chatty.channel.feed(new Uint8Array(1)); - steps.push((await chatty.reader.read()).value?.length ?? 0); - } - const chattyAborted = chatty.channel.aborted; - - const ended = await openTwoWay(); - ended.channel.end(); - const endStep = await ended.reader.read(); - ended.clock.advance(60_000); - await ended.writer.write(new Uint8Array(3)); - - const slowReader = await openTwoWay(); - slowReader.channel.feed(new Uint8Array(70 * KIB)); - await vi.waitFor(() => expect(slowReader.clock.pendingTimerCount).toBe(0)); - slowReader.clock.advance(60_000); - const slowAborted = slowReader.channel.aborted; - const slowStep = await slowReader.reader.read(); - slowReader.clock.advance(15_000); - const slowFailure = await outcome(slowReader.reader.read()); - - const message = `worker ${WORKER_ID} sent nothing on the relayed response for 15000 ms`; - expect(openAt14).toBe(false); - expect(abortedAt15).toBe(true); - expect(await silentRead).toMatchObject({ - code: "WORKER_UNREACHABLE", - message, - state: "rejected", - }); - expect(await writeFailure).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - expect(silent.channel.aborted).toBe(true); - expect(silent.connection.lastSentOf("relay.cancel")).toBeDefined(); - expect(steps).toEqual(Array.from({ length: 12 }, () => 1)); - expect(chattyAborted).toBe(false); - expect(endStep.done).toBe(true); - expect(ended.channel.aborted).toBe(false); - expect(ended.channel.written.map((chunk) => chunk.length)).toEqual([3]); - expect(slowAborted).toBe(false); - expect(slowStep.value?.length).toBe(70 * KIB); - expect(slowFailure).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - }); -}); - -describe("GatewayRelay mapping a channel failure", () => { - const before: readonly { - readonly reason: LinkChannelErrorReason; - readonly code: string; - readonly message: string; - }[] = [ - { - code: "INTERNAL", - message: `worker ${WORKER_ID} sent an invalid relay response`, - reason: "invalid", - }, - { - code: "INTERNAL", - message: `worker ${WORKER_ID} did not read the relayed request body for 60 s`, - reason: "stalled", - }, - { - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} did not take the relayed request within 30 s`, - reason: "not-accepted", - }, - ]; - - it.each(before)( - "fails $reason before the head with $code and the message the table names", - async ({ code, message, reason }) => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - - channel.fail(reason); - - expect(await pending).toEqual({ code, message, state: "rejected" }); - }, - ); - - it("fails closed before the head with WORKER_UNREACHABLE", async () => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - - channel.fail("closed"); - - expect(await pending).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - }); - - it("lets the relay.request answer decide when the peer aborts before the head", async () => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - const request = await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - - channel.fail("aborted"); - connection.fail(request.id, "INTERNAL", "worker side exploded"); - - expect(await pending).toEqual({ - code: "INTERNAL", - message: "worker side exploded", - state: "rejected", - }); - }); - - it("fails with the deadline error when the peer aborts before the head and no answer comes", async () => { - const { connection, relay } = await setup({ deadlineMs: 150 }); - const pending = outcome(relay.relay(WORKER_ID, call())); - await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - - channel.fail("aborted"); - - expect(await pending).toEqual({ - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} did not finish the relayed request within 150 ms`, - state: "rejected", - }); - }); - - const after: readonly { - readonly name: string; - readonly run: (connection: FakeLink, channel: FakeChannel, id: number | string) => void; - readonly expected: Record; - }[] = [ - { - expected: { - message: `worker ${WORKER_ID} sent an invalid relay response`, - state: "rejected", - }, - name: "invalid", - run: (_connection, channel) => channel.fail("invalid"), - }, - { - expected: { code: "WORKER_UNREACHABLE", state: "rejected" }, - name: "closed", - run: (_connection, channel) => channel.fail("closed"), - }, - { - expected: { message: expect.stringMatching(/fell behind for 60 s/), state: "rejected" }, - name: "stalled", - run: (_connection, channel) => channel.fail("stalled"), - }, - { - expected: { message: "worker side exploded", state: "rejected" }, - name: "aborted by the peer, with a failed relay.request answer", - run: (connection, channel, id) => { - channel.fail("aborted"); - connection.fail(id, "INTERNAL", "worker side exploded"); - }, - }, - { - expected: { - message: `worker ${WORKER_ID} ended the relayed response early`, - state: "rejected", - }, - name: "aborted by the peer, with a relay.request that settles without an error", - run: (connection, channel, id) => { - channel.fail("aborted"); - connection.reply(id, {}); - }, - }, - ]; - - it.each(after)("errors the caller's body for $name after the head", async ({ expected, run }) => { - const { connection, relay } = await setup(); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - const channel = await openedChannel(connection); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const response = await pending; - - run(connection, channel, request.id); - const read = await readFailure(response); - - expect(read).toMatchObject(expected); - }); -}); - -describe("GatewayRelay against a worker that breaks the protocol", () => { - it("fails the relay with INTERNAL naming the worker, and sends relay.cancel, on a relay-head with status 101", async () => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - const request = await sentFrame(connection, "relay.request"); - - connection.push("relay-head", { headers: [], requestId: request.id, status: 101 }); - - expect(await pending).toEqual({ - code: "INTERNAL", - message: `worker ${WORKER_ID} sent an invalid relay response`, - state: "rejected", - }); - const cancel = await sentFrame(connection, "relay.cancel"); - expect(relayIdOf(cancel)).toBe(relayIdOf(request)); - }); - - it("errors the caller's body with INTERNAL naming the worker, and sends relay.cancel, on a second relay-head", async () => { - const { connection, relay } = await setup(); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - await openedChannel(connection); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const response = await pending; - - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const read = await readFailure(response); - - expect(read).toEqual({ - code: "INTERNAL", - message: `worker ${WORKER_ID} sent an invalid relay response`, - state: "rejected", - }); - const cancel = await sentFrame(connection, "relay.cancel"); - expect(relayIdOf(cancel)).toBe(relayIdOf(request)); - }); - - it("fails the relay with INTERNAL naming the worker, and sends relay.cancel, on a relay-head whose header value `Headers` refuses", async () => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - const request = await sentFrame(connection, "relay.request"); - - connection.push("relay-head", { - headers: [["x-note", "€"]], - requestId: request.id, - status: 200, - }); - - expect(await pending).toEqual({ - code: "INTERNAL", - message: `worker ${WORKER_ID} sent an invalid relay response`, - state: "rejected", - }); - const cancel = await sentFrame(connection, "relay.cancel"); - expect(relayIdOf(cancel)).toBe(relayIdOf(request)); - }); - - it.each([ - { answer: {}, name: "{} with no head" }, - { answer: { cancelled: true }, name: "{ cancelled: true } when no cancel was sent" }, - ])( - "fails the relay with INTERNAL saying the worker ended it without a response when relay.request answers $name", - async ({ answer }) => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - const request = await sentFrame(connection, "relay.request"); - - connection.reply(request.id, answer); - - expect(await pending).toEqual({ - code: "INTERNAL", - message: `worker ${WORKER_ID} ended the relay without a response`, - state: "rejected", - }); - }, - ); - - it("errors the caller's body stream with the worker's message on an INTERNAL after the head", async () => { - const { connection, relay } = await setup(); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); - const response = await pending; - - connection.fail(request.id, "INTERNAL", "worker side exploded"); - - const read = await readFailure(response); - expect(read).toMatchObject({ message: "worker side exploded", state: "rejected" }); - }); - - it("reads DAEMON_STOPPING from a worker as a lost uplink: WORKER_UNREACHABLE", async () => { - const { connection, relay } = await setup(); - const pending = outcome(relay.relay(WORKER_ID, call())); - const request = await sentFrame(connection, "relay.request"); - - connection.fail(request.id, "DAEMON_STOPPING", "The daemon is stopping"); - - expect(await pending).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - }); - - it("drops every hop-by-hop and framing header of a worker's relay head, in any case, and keeps an ordinary header, on the response it hands the caller", async () => { - const { connection, relay } = await setup(); - const pending = relay.relay(WORKER_ID, call()); - const request = await sentFrame(connection, "relay.request"); - await openedChannel(connection); - - connection.push("relay-head", { - headers: [ - ["connection", "close"], - ["Content-Length", "5"], - ["keep-alive", "timeout=5"], - ["proxy-authenticate", "Basic realm=worker"], - ["proxy-authorization", "Basic worker-secret"], - ["te", "trailers"], - ["Te", "custom-checksum"], - ["trailer", "X-Note"], - ["TRANSFER-ENCODING", "chunked"], - ["upgrade", "h2c"], - ["x-kept", "yes"], - ], - requestId: request.id, - status: 200, - }); - const response = await pending; - - expect([...response.headers.entries()]).toEqual([["x-kept", "yes"]]); - }); -}); diff --git a/src/gateway/test-relay.ts b/src/gateway/test-relay.ts new file mode 100644 index 000000000..93ceb1164 --- /dev/null +++ b/src/gateway/test-relay.ts @@ -0,0 +1,211 @@ +import { expect, vi } from "vitest"; +import { LinkChannelError } from "../link/index.js"; +import type { LinkChannel, LinkChannelErrorReason, LinkConnection } from "../link/index.js"; +import { FakeClock, type Clock } from "../ports/index.js"; +import { connectSimlockClient } from "../simlock-client/client.js"; +import { completeHello, ScriptedConnection } from "../simlock-client/test-support.js"; +import type { SentFrame } from "../simlock-client/test-support.js"; +import type { WorkerDirectory } from "./fleet-ports.js"; +import { GatewayRelay, type TwoWayRelay } from "./index.js"; +import type { GatewayWorkerClient } from "./worker-client.js"; + +export const WORKER_ID = "wrk_a"; + +export const KIB = 1024; + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export class FakeChannel implements LinkChannel { + readonly readable: ReadableStream; + readonly writable: WritableStream; + readonly written: Uint8Array[] = []; + readonly credit = 0; + writableClosed = false; + aborted = false; + abortReason: string | undefined; + #readController!: ReadableStreamDefaultController; + #writeController!: WritableStreamDefaultController; + + constructor(readonly id: string) { + this.readable = new ReadableStream({ + start: (controller) => { + this.#readController = controller; + }, + }); + this.writable = new WritableStream({ + close: () => { + this.writableClosed = true; + }, + start: (controller) => { + this.#writeController = controller; + }, + write: (chunk) => { + this.written.push(chunk); + }, + }); + } + + whenCredit(): Promise { + return Promise.resolve(); + } + + abort(reason?: string): void { + this.aborted = true; + this.abortReason = reason; + try { + this.#readController.error(new LinkChannelError("aborted")); + } catch { + // why: the readable had already ended or failed. + } + } + + feed(bytes: Uint8Array): void { + this.#readController.enqueue(bytes); + } + + end(): void { + this.#readController.close(); + } + + fail(reason: LinkChannelErrorReason): void { + try { + this.#readController.error(new LinkChannelError(reason)); + } catch { + // why: already ended. + } + try { + this.#writeController.error(new LinkChannelError(reason)); + } catch { + // why: already closed. + } + } + + // why: the real channel errors its writable at once on a peer's STOP (src/link/channel.ts onStop → #failOutgoing). + stopWriting(): void { + this.#writeController.error(new LinkChannelError("stopped")); + } + + failWritable(reason: LinkChannelErrorReason): void { + this.#writeController.error(new LinkChannelError(reason)); + } +} + +export class FakeLink extends ScriptedConnection implements LinkConnection { + readonly channels: FakeChannel[] = []; + holdOpen = false; + + openChannel(options: { readonly signal: AbortSignal }): Promise { + if (this.holdOpen) { + return new Promise((_resolve, reject) => { + const fail = () => reject(options.signal.reason as Error); + if (options.signal.aborted) fail(); + else options.signal.addEventListener("abort", fail, { once: true }); + }); + } + const channel = new FakeChannel(`chan-${String(this.channels.length + 1)}`); + this.channels.push(channel); + return Promise.resolve(channel); + } + + acceptChannel(id: string): LinkChannel { + throw new Error(`the gateway accepts no channel: ${id}`); + } +} + +export async function setup( + options: { + readonly deadlineMs?: number; + readonly holdOpen?: boolean; + readonly idleMs?: number; + readonly clock?: Clock; + readonly proxyAuthHeader?: string; + } = {}, +) { + const connection = new FakeLink(); + connection.holdOpen = options.holdOpen ?? false; + const connecting = connectSimlockClient({ connection, credential: "secret" }, true); + await vi.waitFor(() => expect(connection.lastSentOf("hello")).toBeDefined()); + completeHello(connection, { role: "admin" }); + const client = (await connecting) as GatewayWorkerClient; + const directory: WorkerDirectory = { + target: (workerId) => + workerId === WORKER_ID + ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } + : undefined, + }; + return { + connection, + relay: new GatewayRelay({ + directory, + ...(options.deadlineMs === undefined ? {} : { deadlineMs: options.deadlineMs }), + ...(options.idleMs === undefined ? {} : { idleMs: options.idleMs }), + ...(options.clock === undefined ? {} : { clock: options.clock }), + ...(options.proxyAuthHeader === undefined + ? {} + : { proxyAuthHeader: options.proxyAuthHeader }), + }), + }; +} + +export function call( + body?: ReadableStream, + signal?: AbortSignal, +): { request: Request; target: string; tenant: string } { + return { + request: new Request("http://gateway.test:4321/agent-device/rpc", { + method: body === undefined ? "GET" : "PUT", + ...(body === undefined ? {} : { body, duplex: "half" }), + ...(signal === undefined ? {} : { signal }), + } as RequestInit), + target: "echo", + tenant: "gw:g1:alice", + }; +} + +export async function sentFrame(connection: ScriptedConnection, type: string): Promise { + await vi.waitFor(() => expect(connection.lastSentOf(type)).toBeDefined()); + return connection.lastSentOf(type) as SentFrame; +} + +export async function outcome(promise: Promise) { + return promise.then( + (value) => ({ state: "fulfilled" as const, value }), + (error: unknown) => ({ + code: (error as { code?: unknown } | null)?.code, + message: error instanceof Error ? error.message : String(error), + state: "rejected" as const, + }), + ); +} + +export function relayIdOf(frame: SentFrame): unknown { + return (frame.payload as { relayId?: unknown }).relayId; +} + +export async function readFailure(response: Response) { + return outcome(response.body?.getReader().read() ?? Promise.reject(new Error("no body"))); +} + +export async function openedChannel(connection: FakeLink): Promise { + await vi.waitFor(() => expect(connection.channels).toHaveLength(1)); + return connection.channels[0] as FakeChannel; +} + +export async function openTwoWay(signal?: AbortSignal) { + const clock = new FakeClock(); + const { connection, relay } = await setup({ clock }); + const pending = relay.relayTwoWay(WORKER_ID, call(undefined, signal)); + const request = await sentFrame(connection, "relay.request"); + const channel = await openedChannel(connection); + connection.push("relay-head", { headers: [], requestId: request.id, status: 200 }); + const opened = (await pending) as TwoWayRelay; + return { + channel, + clock, + connection, + reader: opened.readable.getReader(), + writer: opened.writable.getWriter(), + }; +} From 3f4c82137fec81886fb9b5fde0c0a77e9da18d40 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:24:25 +0100 Subject: [PATCH 23/83] test(gateway): split aggregate.test.ts by topic --- src/gateway/aggregate-catalog.test.ts | 580 +++++++++++++++++++++++++ src/gateway/aggregate.test.ts | 591 +------------------------- src/gateway/test-aggregate.ts | 13 + 3 files changed, 596 insertions(+), 588 deletions(-) create mode 100644 src/gateway/aggregate-catalog.test.ts create mode 100644 src/gateway/test-aggregate.ts diff --git a/src/gateway/aggregate-catalog.test.ts b/src/gateway/aggregate-catalog.test.ts new file mode 100644 index 000000000..388072967 --- /dev/null +++ b/src/gateway/aggregate-catalog.test.ts @@ -0,0 +1,580 @@ +import { describe, expect, it } from "vitest"; +import { OPERATIONS } from "../contract/index.js"; +import { aggregateCatalog } from "./aggregate.js"; +import type { WorkerView } from "./worker-registry.js"; +import { view } from "./test-aggregate.js"; + +// why: pure logic with many cases; the gateway barrel does not export aggregateCatalog, so its cases are asserted here through aggregate.ts. +describe("aggregateCatalog", () => { + const iosOnA = { + defaultRuntime: "26.0", + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["26.0"] }, + models: ["iPhone 17"], + platform: "ios" as const, + runtimes: ["26.0"], + }; + // B has 25.4 installed, but only the iPad pairs with it there. + const iosOnB = { + defaultRuntime: "26.0", + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPad Pro": ["25.4", "26.0"], "iPhone 17": ["26.0"] }, + models: ["iPhone 17", "iPad Pro"], + platform: "ios" as const, + runtimes: ["26.0", "25.4"], + }; + + it("leaves a worker with no catalog read, such as one that is starting, out of the union", () => { + const starting: WorkerView = { + connection: "connected", + drained: false, + health: "starting", + id: "wrk_s", + lastSeenAt: 1_000, + }; + + const catalog = aggregateCatalog([starting, view({ catalog: [iosOnA], id: "wrk_a" })]); + + expect(catalog.platforms).toHaveLength(1); + expect(catalog.platforms[0]?.modelWorkers).toEqual({ "iPhone 17": ["wrk_a"] }); + }); + + it("unions the connected workers' catalogs and annotates every entry", () => { + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [iosOnB], id: "wrk_b" }), + ]); + + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + expect(catalog.platforms).toHaveLength(1); + expect(catalog.platforms[0]).toMatchObject({ + models: ["iPad Pro", "iPhone 17"], + modelWorkers: { "iPad Pro": ["wrk_b"], "iPhone 17": ["wrk_a", "wrk_b"] }, + platform: "ios", + runtimes: ["25.4", "26.0"], + runtimeWorkers: { "25.4": ["wrk_b"], "26.0": ["wrk_a", "wrk_b"] }, + }); + }); + + it("carries the union of the workers' slim categories, and no default mode", () => { + const slimming = (slimCategories: string[], defaultMode: "slim" | "full") => ({ + ...iosOnA, + defaultMode, + slimCategories, + }); + + const catalog = aggregateCatalog([ + view({ catalog: [slimming(["store", "mdm"], "slim")], id: "wrk_a" }), + view({ catalog: [slimming(["store", "widgets"], "full")], id: "wrk_b" }), + ]); + + expect(catalog.platforms[0]?.slimCategories).toEqual(["mdm", "store", "widgets"]); + expect(catalog.platforms[0]).not.toHaveProperty("defaultMode"); + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + }); + + it("carries the union of the connected workers' model classes, and keeps the class of the first worker in id order when two disagree", () => { + const classed = (modelClasses: Record) => ({ + ...iosOnB, + modelClasses, + }); + + // Listed with the later id first, so the answer cannot come from input order. + const catalog = aggregateCatalog([ + view({ catalog: [classed({ "iPad Pro": "phone", "iPhone 17": "tv" })], id: "wrk_b" }), + view({ catalog: [classed({ "iPhone 17": "phone" })], id: "wrk_a" }), + view({ + catalog: [classed({ "iPad Pro": "tablet" })], + connection: "disconnected", + id: "wrk_0", + }), + ]); + + expect(catalog.platforms[0]?.modelClasses).toEqual({ + "iPad Pro": "phone", + "iPhone 17": "phone", + }); + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + }); + + it("drops a model class for a name the worker does not list", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [{ ...iosOnB, modelClasses: { Ghost: "tv", "iPhone 17": "phone" } }], + id: "wrk_a", + }), + ]); + + expect(catalog.platforms[0]?.modelClasses).toEqual({ "iPhone 17": "phone" }); + }); + + it("carries a class default only when every connected worker reports the same one", () => { + // Every default names a model its worker lists, so a disagreement reaches agreedClassDefaults. + const withDefaults = ( + classDefaults: Record, + base: typeof iosOnA | typeof iosOnB = iosOnA, + ) => ({ + ...base, + classDefaults, + models: [...new Set([...base.models, "iPad Pro", "iPad (A16)"])], + }); + + const agreed = aggregateCatalog([ + view({ catalog: [withDefaults({ phone: "iPhone 17", tablet: "iPad Pro" })], id: "wrk_a" }), + view({ + catalog: [withDefaults({ phone: "iPhone 17", tablet: "iPad (A16)" }, iosOnB)], + id: "wrk_b", + }), + ]); + + expect(agreed.platforms[0]?.classDefaults).toEqual({ phone: "iPhone 17" }); + expect(() => OPERATIONS["catalog.get"].output.parse(agreed)).not.toThrow(); + }); + + it("drops a class default when a connected worker has none for that class", () => { + const catalog = aggregateCatalog([ + view({ catalog: [{ ...iosOnA, classDefaults: { phone: "iPhone 17" } }], id: "wrk_a" }), + view({ catalog: [{ ...iosOnB, classDefaults: {} }], id: "wrk_b" }), + ]); + + expect(catalog.platforms[0]?.classDefaults).toEqual({}); + }); + + it("drops a class default naming a model the worker does not list", () => { + const catalog = aggregateCatalog([ + view({ catalog: [{ ...iosOnB, classDefaults: { phone: "Ghost" } }], id: "wrk_a" }), + ]); + + expect(catalog.platforms[0]?.classDefaults).toEqual({}); + }); + + it("keeps a class default a lone connected worker reports, and ignores a disconnected worker's", () => { + const catalog = aggregateCatalog([ + view({ catalog: [{ ...iosOnA, classDefaults: { phone: "iPhone 17" } }], id: "wrk_a" }), + view({ + catalog: [{ ...iosOnB, classDefaults: { phone: "iPhone 16" } }], + connection: "disconnected", + id: "wrk_b", + }), + ]); + + expect(catalog.platforms[0]?.classDefaults).toEqual({ phone: "iPhone 17" }); + }); + + it("keeps a default runtime only when every worker agrees on it", () => { + const agreed = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [iosOnB], id: "wrk_b" }), + ]); + expect(agreed.platforms[0]?.defaultRuntime).toBe("26.0"); + + const disagreed = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [{ ...iosOnB, defaultRuntime: "25.4" }], id: "wrk_b" }), + ]); + // Picking one at random would make `simlock lease` non-deterministic across an unchanged + // fleet, so a fleet that disagrees has no default. + expect(disagreed.platforms[0]?.defaultRuntime).toBeUndefined(); + }); + + it("ignores a disconnected worker: a machine nobody can reach can lease nothing", () => { + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], connection: "disconnected", id: "wrk_a" }), + ]); + + expect(catalog.platforms).toEqual([]); + }); + + it("keeps a drained worker, whose models are still installed", () => { + const catalog = aggregateCatalog([view({ catalog: [iosOnA], drained: true, id: "wrk_a" })]); + + expect(catalog.platforms[0]?.models).toEqual(["iPhone 17"]); + }); + + it("pairs a model with a runtime when at least one connected worker pairs them", () => { + const iosOnC = { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["25.4"] }, + models: ["iPhone 17"], + platform: "ios" as const, + runtimes: ["25.4"], + }; + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [iosOnC], id: "wrk_c" }), + ]); + + expect(catalog.platforms[0]?.modelRuntimes).toEqual({ "iPhone 17": ["25.4", "26.0"] }); + }); + + it("does not pair a model with a runtime that only another model has", () => { + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [iosOnB], id: "wrk_b" }), + ]); + + // 25.4 is in the fleet's runtimes, and iPhone 17 is in its models, but no worker pairs them. + expect(catalog.platforms[0]?.runtimes).toContain("25.4"); + expect(catalog.platforms[0]?.modelRuntimes).toEqual({ + "iPad Pro": ["25.4", "26.0"], + "iPhone 17": ["26.0"], + }); + }); + + it("ignores the pairings of disconnected and incompatible workers", () => { + const pairsOld = { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["25.4"] }, + models: ["iPhone 17"], + platform: "ios" as const, + runtimes: ["25.4"], + }; + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [pairsOld], connection: "disconnected", id: "wrk_gone" }), + view({ catalog: [pairsOld], connection: "incompatible", id: "wrk_old" }), + ]); + + expect(catalog.platforms[0]?.modelRuntimes).toEqual({ "iPhone 17": ["26.0"] }); + }); + + it("gives every model in the fleet catalog a modelRuntimes entry, empty when nothing pairs", () => { + const unpaired = { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone XS": [] }, + models: ["iPhone XS"], + platform: "ios" as const, + runtimes: ["26.0"], + }; + const catalog = aggregateCatalog([ + view({ catalog: [iosOnB], id: "wrk_b" }), + view({ catalog: [unpaired], id: "wrk_x" }), + ]); + + const platform = catalog.platforms[0]; + expect(Object.keys(platform?.modelRuntimes ?? {}).sort()).toEqual( + [...(platform?.models ?? [])].sort(), + ); + expect(platform?.modelRuntimes["iPhone XS"]).toEqual([]); + }); + + it("treats a model a worker names after an Object.prototype member like any other", () => { + // Wire input: a worker that lists `constructor` without pairing it must not make the fleet + // catalog read the inherited function. + const odd = OPERATIONS["catalog.get"].output.parse({ + platforms: [ + { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: {}, + models: ["constructor"], + platform: "ios", + runtimes: [], + }, + ], + }).platforms; + + const catalog = aggregateCatalog([view({ catalog: odd, id: "wrk_a" })]); + + expect(catalog.platforms[0]?.modelRuntimes).toEqual({ constructor: [] }); + }); + + it("drops a pairing with a runtime the worker does not list itself, even when another worker has it", () => { + const claimsMore = { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["25.4", "26.0"] }, + models: ["iPhone 17"], + platform: "ios" as const, + runtimes: ["26.0"], + }; + const catalog = aggregateCatalog([ + view({ catalog: [claimsMore], id: "wrk_a" }), + view({ catalog: [iosOnB], id: "wrk_b" }), + ]); + + // 25.4 is in the fleet (on B, for the iPad only); A's claim to pair it is not A's to make. + expect(catalog.platforms[0]?.modelRuntimes["iPhone 17"]).toEqual(["26.0"]); + }); + + it("filters to one platform when asked", () => { + const catalog = aggregateCatalog( + [ + view({ + catalog: [ + iosOnA, + { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "Pixel 9": ["35"] }, + models: ["Pixel 9"], + platform: "android", + runtimes: ["35"], + }, + ], + id: "wrk_a", + }), + ], + "android", + ); + + expect(catalog.platforms.map((entry) => entry.platform)).toEqual(["android"]); + }); + + describe("other names and images", () => { + const androidOn = ( + overrides: Partial<{ + models: string[]; + modelAliases: Record; + images: { runtime: string; tag: string; abi: string }[]; + runtimes: string[]; + customModels: string[]; + classDefaults: {}; + modelClasses: Record; + }>, + ) => { + const models = overrides.models ?? ["Pixel 8"]; + const runtimes = overrides.runtimes ?? ["34", "35"]; + return { + ...(overrides.customModels === undefined ? {} : { customModels: overrides.customModels }), + ...(overrides.images === undefined ? {} : { images: overrides.images }), + modelAliases: overrides.modelAliases ?? {}, + classDefaults: {}, + modelClasses: overrides.modelClasses ?? {}, + modelRuntimes: Object.fromEntries(models.map((model) => [model, runtimes])), + models, + platform: "android" as const, + runtimes, + }; + }; + + it("unions other names per model, once per spelling ignoring case", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [androidOn({ modelAliases: { "Pixel 8": ["pixel_8"] } })], + id: "wrk_a", + }), + view({ + catalog: [ + androidOn({ + modelAliases: { "Pixel 8": ["PIXEL_8", "pixel8"], "Pixel 9": ["pixel_9"] }, + models: ["Pixel 8", "Pixel 9"], + }), + ], + id: "wrk_b", + }), + ]); + + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + expect(catalog.platforms[0]?.modelAliases).toEqual({ + "Pixel 8": ["pixel8", "pixel_8"], + "Pixel 9": ["pixel_9"], + }); + }); + + it("lists each distinct image once", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [ + androidOn({ + images: [ + { abi: "x86_64", runtime: "34", tag: "default" }, + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + ], + }), + ], + id: "wrk_a", + }), + view({ + catalog: [ + androidOn({ + images: [ + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + { abi: "x86_64", runtime: "35", tag: "google_apis" }, + ], + }), + ], + id: "wrk_b", + }), + ]); + + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + expect(catalog.platforms[0]?.images).toEqual([ + { abi: "x86_64", runtime: "34", tag: "default" }, + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + { abi: "x86_64", runtime: "35", tag: "google_apis" }, + ]); + }); + + it("omits images when no worker reports any", () => { + const catalog = aggregateCatalog([ + view({ catalog: [iosOnA], id: "wrk_a" }), + view({ catalog: [iosOnB], id: "wrk_b" }), + ]); + + expect(catalog.platforms[0]).not.toHaveProperty("images"); + }); + + it("does not list a model's own name, in any letter case, as another name for it", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [androidOn({ modelAliases: { "Pixel 8": ["PIXEL 8", "pixel_8"] } })], + id: "wrk_a", + }), + ]); + + expect(catalog.platforms[0]?.modelAliases).toEqual({ "Pixel 8": ["pixel_8"] }); + }); + + it("sorts the fleet's images whatever order the workers report them in", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [ + androidOn({ + images: [ + { abi: "x86_64", runtime: "35", tag: "google_apis" }, + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + { abi: "x86_64", runtime: "34", tag: "default" }, + ], + }), + ], + id: "wrk_a", + }), + ]); + + expect(catalog.platforms[0]?.images).toEqual([ + { abi: "x86_64", runtime: "34", tag: "default" }, + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + { abi: "x86_64", runtime: "35", tag: "google_apis" }, + ]); + }); + + it("keeps the fleet catalog inside the contract's bounds when the union of valid workers is not", () => { + const names = (prefix: string, count: number) => + Array.from({ length: count }, (_, index) => `${prefix}${index}`); + const worker = (prefix: string) => + androidOn({ + images: names(prefix, 1024).map((tag) => ({ abi: "x86_64", runtime: "34", tag })), + modelAliases: { + ...Object.fromEntries(names(`${prefix}m`, 4095).map((model) => [model, ["x"]])), + "Pixel 8": names(prefix, 32), + }, + customModels: ["Pixel 8", ...names(`${prefix}m`, 4095)], + classDefaults: {}, + modelClasses: Object.fromEntries( + ["Pixel 8", ...names(`${prefix}m`, 4095)].map((model) => [model, "phone" as const]), + ), + models: ["Pixel 8", ...names(`${prefix}m`, 4095)], + }); + const valid = [worker("a"), worker("b")]; + for (const entry of valid) { + expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).not.toThrow(); + } + + // Worker b connects first, so what survives the cut is decided by sort order, not arrival. + const catalog = aggregateCatalog([ + view({ catalog: [valid[1]!], id: "wrk_b" }), + view({ catalog: [valid[0]!], id: "wrk_a" }), + ]); + + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + const platform = catalog.platforms[0]; + expect(platform?.images).toHaveLength(1024); + expect(platform?.images?.every((image) => image.tag.startsWith("a"))).toBe(true); + const aliasedModels = Object.keys(platform?.modelAliases ?? {}); + expect(aliasedModels).toHaveLength(4096); + expect(aliasedModels.filter((model) => model.startsWith("bm"))).toEqual([]); + expect(platform?.modelAliases["Pixel 8"]).toEqual(names("a", 32).sort()); + expect(platform?.customModels).toHaveLength(4096); + expect(platform?.customModels?.filter((model) => model.startsWith("bm"))).toEqual([]); + }); + + it("cuts the fleet's model classes by sort order, not arrival order, when the union is over the bound", () => { + const names = (prefix: string) => + Array.from({ length: 4096 }, (_, index) => `${prefix}${index}`); + const worker = (prefix: string) => + androidOn({ + classDefaults: {}, + modelClasses: Object.fromEntries(names(prefix).map((model) => [model, "phone" as const])), + models: names(prefix), + }); + + // The worker first in id order holds the names that sort last. + const catalog = aggregateCatalog([ + view({ catalog: [worker("zm")], id: "wrk_a" }), + view({ catalog: [worker("am")], id: "wrk_b" }), + ]); + + const classed = Object.keys(catalog.platforms[0]?.modelClasses ?? {}); + expect(classed).toHaveLength(4096); + expect(classed.every((model) => model.startsWith("am"))).toBe(true); + }); + + it("marks a model custom when one worker marks it and another lists it as built-in", () => { + const catalog = aggregateCatalog([ + view({ catalog: [androidOn({ models: ["Pixel 8", "My Tablet"] })], id: "wrk_a" }), + view({ + catalog: [androidOn({ customModels: ["My Tablet"], models: ["My Tablet"] })], + id: "wrk_b", + }), + ]); + + expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); + expect(catalog.platforms[0]?.customModels).toEqual(["My Tablet"]); + }); + + it("drops a custom name the worker does not list in models", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [androidOn({ customModels: ["My Tablet", "Ghost"], models: ["My Tablet"] })], + id: "wrk_a", + }), + // Another worker lists Ghost, but only the worker that marks a name can make it custom. + view({ catalog: [androidOn({ models: ["Ghost"] })], id: "wrk_b" }), + ]); + + expect(catalog.platforms[0]?.customModels).toEqual(["My Tablet"]); + }); + + it("omits customModels when no worker marks a model it lists", () => { + const catalog = aggregateCatalog([ + view({ catalog: [androidOn({ customModels: ["Ghost"] })], id: "wrk_a" }), + ]); + + expect(catalog.platforms[0]).not.toHaveProperty("customModels"); + }); + + it("drops an image whose runtime the reporting worker does not list", () => { + const catalog = aggregateCatalog([ + view({ + catalog: [ + androidOn({ + images: [ + { abi: "x86_64", runtime: "34", tag: "default" }, + { abi: "x86_64", runtime: "99", tag: "default" }, + ], + }), + ], + id: "wrk_a", + }), + ]); + + expect(catalog.platforms[0]?.images).toEqual([ + { abi: "x86_64", runtime: "34", tag: "default" }, + ]); + }); + }); +}); diff --git a/src/gateway/aggregate.test.ts b/src/gateway/aggregate.test.ts index 2fc86bd8b..89a9fa1a2 100644 --- a/src/gateway/aggregate.test.ts +++ b/src/gateway/aggregate.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from "vitest"; - import { OPERATIONS } from "../contract/index.js"; -import { aggregateCatalog, aggregateStatus } from "./aggregate.js"; +import { aggregateStatus } from "./aggregate.js"; import { FleetLeaseIndex } from "./lease-index.js"; import { deviceFixture, hostFixture, leaseFixture } from "./test-support.js"; import type { WorkerView } from "./worker-registry.js"; +import { view } from "./test-aggregate.js"; function capacity(running: number, limit: number) { return { @@ -34,18 +34,7 @@ function capacity(running: number, limit: number) { const GATEWAY_HOST = { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] }; -function view(overrides: Partial & { readonly id: string }): WorkerView { - return { - catalog: [], - connection: "connected", - devices: [], - drained: false, - lastSeenAt: 1_000, - leases: [], - ...overrides, - }; -} - +// why: pure logic with many cases; the gateway barrel does not export aggregateStatus, so its cases are asserted here through aggregate.ts. describe("aggregateStatus", () => { it("returns the same shape a worker returns, with the gateway named in the daemon block", () => { const status = aggregateStatus([], { @@ -386,580 +375,6 @@ describe("aggregateStatus", () => { }); }); -describe("aggregateCatalog", () => { - const iosOnA = { - defaultRuntime: "26.0", - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["26.0"] }, - models: ["iPhone 17"], - platform: "ios" as const, - runtimes: ["26.0"], - }; - // B has 25.4 installed, but only the iPad pairs with it there. - const iosOnB = { - defaultRuntime: "26.0", - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPad Pro": ["25.4", "26.0"], "iPhone 17": ["26.0"] }, - models: ["iPhone 17", "iPad Pro"], - platform: "ios" as const, - runtimes: ["26.0", "25.4"], - }; - - it("leaves a worker with no catalog read, such as one that is starting, out of the union", () => { - const starting: WorkerView = { - connection: "connected", - drained: false, - health: "starting", - id: "wrk_s", - lastSeenAt: 1_000, - }; - - const catalog = aggregateCatalog([starting, view({ catalog: [iosOnA], id: "wrk_a" })]); - - expect(catalog.platforms).toHaveLength(1); - expect(catalog.platforms[0]?.modelWorkers).toEqual({ "iPhone 17": ["wrk_a"] }); - }); - - it("unions the connected workers' catalogs and annotates every entry", () => { - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [iosOnB], id: "wrk_b" }), - ]); - - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - expect(catalog.platforms).toHaveLength(1); - expect(catalog.platforms[0]).toMatchObject({ - models: ["iPad Pro", "iPhone 17"], - modelWorkers: { "iPad Pro": ["wrk_b"], "iPhone 17": ["wrk_a", "wrk_b"] }, - platform: "ios", - runtimes: ["25.4", "26.0"], - runtimeWorkers: { "25.4": ["wrk_b"], "26.0": ["wrk_a", "wrk_b"] }, - }); - }); - - it("carries the union of the workers' slim categories, and no default mode", () => { - const slimming = (slimCategories: string[], defaultMode: "slim" | "full") => ({ - ...iosOnA, - defaultMode, - slimCategories, - }); - - const catalog = aggregateCatalog([ - view({ catalog: [slimming(["store", "mdm"], "slim")], id: "wrk_a" }), - view({ catalog: [slimming(["store", "widgets"], "full")], id: "wrk_b" }), - ]); - - expect(catalog.platforms[0]?.slimCategories).toEqual(["mdm", "store", "widgets"]); - expect(catalog.platforms[0]).not.toHaveProperty("defaultMode"); - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - }); - - it("carries the union of the connected workers' model classes, and keeps the class of the first worker in id order when two disagree", () => { - const classed = (modelClasses: Record) => ({ - ...iosOnB, - modelClasses, - }); - - // Listed with the later id first, so the answer cannot come from input order. - const catalog = aggregateCatalog([ - view({ catalog: [classed({ "iPad Pro": "phone", "iPhone 17": "tv" })], id: "wrk_b" }), - view({ catalog: [classed({ "iPhone 17": "phone" })], id: "wrk_a" }), - view({ - catalog: [classed({ "iPad Pro": "tablet" })], - connection: "disconnected", - id: "wrk_0", - }), - ]); - - expect(catalog.platforms[0]?.modelClasses).toEqual({ - "iPad Pro": "phone", - "iPhone 17": "phone", - }); - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - }); - - it("drops a model class for a name the worker does not list", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [{ ...iosOnB, modelClasses: { Ghost: "tv", "iPhone 17": "phone" } }], - id: "wrk_a", - }), - ]); - - expect(catalog.platforms[0]?.modelClasses).toEqual({ "iPhone 17": "phone" }); - }); - - it("carries a class default only when every connected worker reports the same one", () => { - // Every default names a model its worker lists, so a disagreement reaches agreedClassDefaults. - const withDefaults = ( - classDefaults: Record, - base: typeof iosOnA | typeof iosOnB = iosOnA, - ) => ({ - ...base, - classDefaults, - models: [...new Set([...base.models, "iPad Pro", "iPad (A16)"])], - }); - - const agreed = aggregateCatalog([ - view({ catalog: [withDefaults({ phone: "iPhone 17", tablet: "iPad Pro" })], id: "wrk_a" }), - view({ - catalog: [withDefaults({ phone: "iPhone 17", tablet: "iPad (A16)" }, iosOnB)], - id: "wrk_b", - }), - ]); - - expect(agreed.platforms[0]?.classDefaults).toEqual({ phone: "iPhone 17" }); - expect(() => OPERATIONS["catalog.get"].output.parse(agreed)).not.toThrow(); - }); - - it("drops a class default when a connected worker has none for that class", () => { - const catalog = aggregateCatalog([ - view({ catalog: [{ ...iosOnA, classDefaults: { phone: "iPhone 17" } }], id: "wrk_a" }), - view({ catalog: [{ ...iosOnB, classDefaults: {} }], id: "wrk_b" }), - ]); - - expect(catalog.platforms[0]?.classDefaults).toEqual({}); - }); - - it("drops a class default naming a model the worker does not list", () => { - const catalog = aggregateCatalog([ - view({ catalog: [{ ...iosOnB, classDefaults: { phone: "Ghost" } }], id: "wrk_a" }), - ]); - - expect(catalog.platforms[0]?.classDefaults).toEqual({}); - }); - - it("keeps a class default a lone connected worker reports, and ignores a disconnected worker's", () => { - const catalog = aggregateCatalog([ - view({ catalog: [{ ...iosOnA, classDefaults: { phone: "iPhone 17" } }], id: "wrk_a" }), - view({ - catalog: [{ ...iosOnB, classDefaults: { phone: "iPhone 16" } }], - connection: "disconnected", - id: "wrk_b", - }), - ]); - - expect(catalog.platforms[0]?.classDefaults).toEqual({ phone: "iPhone 17" }); - }); - - it("keeps a default runtime only when every worker agrees on it", () => { - const agreed = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [iosOnB], id: "wrk_b" }), - ]); - expect(agreed.platforms[0]?.defaultRuntime).toBe("26.0"); - - const disagreed = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [{ ...iosOnB, defaultRuntime: "25.4" }], id: "wrk_b" }), - ]); - // Picking one at random would make `simlock lease` non-deterministic across an unchanged - // fleet, so a fleet that disagrees has no default. - expect(disagreed.platforms[0]?.defaultRuntime).toBeUndefined(); - }); - - it("ignores a disconnected worker: a machine nobody can reach can lease nothing", () => { - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], connection: "disconnected", id: "wrk_a" }), - ]); - - expect(catalog.platforms).toEqual([]); - }); - - it("keeps a drained worker, whose models are still installed", () => { - const catalog = aggregateCatalog([view({ catalog: [iosOnA], drained: true, id: "wrk_a" })]); - - expect(catalog.platforms[0]?.models).toEqual(["iPhone 17"]); - }); - - it("pairs a model with a runtime when at least one connected worker pairs them", () => { - const iosOnC = { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["25.4"] }, - models: ["iPhone 17"], - platform: "ios" as const, - runtimes: ["25.4"], - }; - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [iosOnC], id: "wrk_c" }), - ]); - - expect(catalog.platforms[0]?.modelRuntimes).toEqual({ "iPhone 17": ["25.4", "26.0"] }); - }); - - it("does not pair a model with a runtime that only another model has", () => { - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [iosOnB], id: "wrk_b" }), - ]); - - // 25.4 is in the fleet's runtimes, and iPhone 17 is in its models, but no worker pairs them. - expect(catalog.platforms[0]?.runtimes).toContain("25.4"); - expect(catalog.platforms[0]?.modelRuntimes).toEqual({ - "iPad Pro": ["25.4", "26.0"], - "iPhone 17": ["26.0"], - }); - }); - - it("ignores the pairings of disconnected and incompatible workers", () => { - const pairsOld = { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["25.4"] }, - models: ["iPhone 17"], - platform: "ios" as const, - runtimes: ["25.4"], - }; - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [pairsOld], connection: "disconnected", id: "wrk_gone" }), - view({ catalog: [pairsOld], connection: "incompatible", id: "wrk_old" }), - ]); - - expect(catalog.platforms[0]?.modelRuntimes).toEqual({ "iPhone 17": ["26.0"] }); - }); - - it("gives every model in the fleet catalog a modelRuntimes entry, empty when nothing pairs", () => { - const unpaired = { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone XS": [] }, - models: ["iPhone XS"], - platform: "ios" as const, - runtimes: ["26.0"], - }; - const catalog = aggregateCatalog([ - view({ catalog: [iosOnB], id: "wrk_b" }), - view({ catalog: [unpaired], id: "wrk_x" }), - ]); - - const platform = catalog.platforms[0]; - expect(Object.keys(platform?.modelRuntimes ?? {}).sort()).toEqual( - [...(platform?.models ?? [])].sort(), - ); - expect(platform?.modelRuntimes["iPhone XS"]).toEqual([]); - }); - - it("treats a model a worker names after an Object.prototype member like any other", () => { - // Wire input: a worker that lists `constructor` without pairing it must not make the fleet - // catalog read the inherited function. - const odd = OPERATIONS["catalog.get"].output.parse({ - platforms: [ - { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: {}, - models: ["constructor"], - platform: "ios", - runtimes: [], - }, - ], - }).platforms; - - const catalog = aggregateCatalog([view({ catalog: odd, id: "wrk_a" })]); - - expect(catalog.platforms[0]?.modelRuntimes).toEqual({ constructor: [] }); - }); - - it("drops a pairing with a runtime the worker does not list itself, even when another worker has it", () => { - const claimsMore = { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["25.4", "26.0"] }, - models: ["iPhone 17"], - platform: "ios" as const, - runtimes: ["26.0"], - }; - const catalog = aggregateCatalog([ - view({ catalog: [claimsMore], id: "wrk_a" }), - view({ catalog: [iosOnB], id: "wrk_b" }), - ]); - - // 25.4 is in the fleet (on B, for the iPad only); A's claim to pair it is not A's to make. - expect(catalog.platforms[0]?.modelRuntimes["iPhone 17"]).toEqual(["26.0"]); - }); - - it("filters to one platform when asked", () => { - const catalog = aggregateCatalog( - [ - view({ - catalog: [ - iosOnA, - { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "Pixel 9": ["35"] }, - models: ["Pixel 9"], - platform: "android", - runtimes: ["35"], - }, - ], - id: "wrk_a", - }), - ], - "android", - ); - - expect(catalog.platforms.map((entry) => entry.platform)).toEqual(["android"]); - }); - - describe("other names and images", () => { - const androidOn = ( - overrides: Partial<{ - models: string[]; - modelAliases: Record; - images: { runtime: string; tag: string; abi: string }[]; - runtimes: string[]; - customModels: string[]; - classDefaults: {}; - modelClasses: Record; - }>, - ) => { - const models = overrides.models ?? ["Pixel 8"]; - const runtimes = overrides.runtimes ?? ["34", "35"]; - return { - ...(overrides.customModels === undefined ? {} : { customModels: overrides.customModels }), - ...(overrides.images === undefined ? {} : { images: overrides.images }), - modelAliases: overrides.modelAliases ?? {}, - classDefaults: {}, - modelClasses: overrides.modelClasses ?? {}, - modelRuntimes: Object.fromEntries(models.map((model) => [model, runtimes])), - models, - platform: "android" as const, - runtimes, - }; - }; - - it("unions other names per model, once per spelling ignoring case", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [androidOn({ modelAliases: { "Pixel 8": ["pixel_8"] } })], - id: "wrk_a", - }), - view({ - catalog: [ - androidOn({ - modelAliases: { "Pixel 8": ["PIXEL_8", "pixel8"], "Pixel 9": ["pixel_9"] }, - models: ["Pixel 8", "Pixel 9"], - }), - ], - id: "wrk_b", - }), - ]); - - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - expect(catalog.platforms[0]?.modelAliases).toEqual({ - "Pixel 8": ["pixel8", "pixel_8"], - "Pixel 9": ["pixel_9"], - }); - }); - - it("lists each distinct image once", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [ - androidOn({ - images: [ - { abi: "x86_64", runtime: "34", tag: "default" }, - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - ], - }), - ], - id: "wrk_a", - }), - view({ - catalog: [ - androidOn({ - images: [ - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - { abi: "x86_64", runtime: "35", tag: "google_apis" }, - ], - }), - ], - id: "wrk_b", - }), - ]); - - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - expect(catalog.platforms[0]?.images).toEqual([ - { abi: "x86_64", runtime: "34", tag: "default" }, - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - { abi: "x86_64", runtime: "35", tag: "google_apis" }, - ]); - }); - - it("omits images when no worker reports any", () => { - const catalog = aggregateCatalog([ - view({ catalog: [iosOnA], id: "wrk_a" }), - view({ catalog: [iosOnB], id: "wrk_b" }), - ]); - - expect(catalog.platforms[0]).not.toHaveProperty("images"); - }); - - it("does not list a model's own name, in any letter case, as another name for it", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [androidOn({ modelAliases: { "Pixel 8": ["PIXEL 8", "pixel_8"] } })], - id: "wrk_a", - }), - ]); - - expect(catalog.platforms[0]?.modelAliases).toEqual({ "Pixel 8": ["pixel_8"] }); - }); - - it("sorts the fleet's images whatever order the workers report them in", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [ - androidOn({ - images: [ - { abi: "x86_64", runtime: "35", tag: "google_apis" }, - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - { abi: "x86_64", runtime: "34", tag: "default" }, - ], - }), - ], - id: "wrk_a", - }), - ]); - - expect(catalog.platforms[0]?.images).toEqual([ - { abi: "x86_64", runtime: "34", tag: "default" }, - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - { abi: "x86_64", runtime: "35", tag: "google_apis" }, - ]); - }); - - it("keeps the fleet catalog inside the contract's bounds when the union of valid workers is not", () => { - const names = (prefix: string, count: number) => - Array.from({ length: count }, (_, index) => `${prefix}${index}`); - const worker = (prefix: string) => - androidOn({ - images: names(prefix, 1024).map((tag) => ({ abi: "x86_64", runtime: "34", tag })), - modelAliases: { - ...Object.fromEntries(names(`${prefix}m`, 4095).map((model) => [model, ["x"]])), - "Pixel 8": names(prefix, 32), - }, - customModels: ["Pixel 8", ...names(`${prefix}m`, 4095)], - classDefaults: {}, - modelClasses: Object.fromEntries( - ["Pixel 8", ...names(`${prefix}m`, 4095)].map((model) => [model, "phone" as const]), - ), - models: ["Pixel 8", ...names(`${prefix}m`, 4095)], - }); - const valid = [worker("a"), worker("b")]; - for (const entry of valid) { - expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).not.toThrow(); - } - - // Worker b connects first, so what survives the cut is decided by sort order, not arrival. - const catalog = aggregateCatalog([ - view({ catalog: [valid[1]!], id: "wrk_b" }), - view({ catalog: [valid[0]!], id: "wrk_a" }), - ]); - - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - const platform = catalog.platforms[0]; - expect(platform?.images).toHaveLength(1024); - expect(platform?.images?.every((image) => image.tag.startsWith("a"))).toBe(true); - const aliasedModels = Object.keys(platform?.modelAliases ?? {}); - expect(aliasedModels).toHaveLength(4096); - expect(aliasedModels.filter((model) => model.startsWith("bm"))).toEqual([]); - expect(platform?.modelAliases["Pixel 8"]).toEqual(names("a", 32).sort()); - expect(platform?.customModels).toHaveLength(4096); - expect(platform?.customModels?.filter((model) => model.startsWith("bm"))).toEqual([]); - }); - - it("cuts the fleet's model classes by sort order, not arrival order, when the union is over the bound", () => { - const names = (prefix: string) => - Array.from({ length: 4096 }, (_, index) => `${prefix}${index}`); - const worker = (prefix: string) => - androidOn({ - classDefaults: {}, - modelClasses: Object.fromEntries(names(prefix).map((model) => [model, "phone" as const])), - models: names(prefix), - }); - - // The worker first in id order holds the names that sort last. - const catalog = aggregateCatalog([ - view({ catalog: [worker("zm")], id: "wrk_a" }), - view({ catalog: [worker("am")], id: "wrk_b" }), - ]); - - const classed = Object.keys(catalog.platforms[0]?.modelClasses ?? {}); - expect(classed).toHaveLength(4096); - expect(classed.every((model) => model.startsWith("am"))).toBe(true); - }); - - it("marks a model custom when one worker marks it and another lists it as built-in", () => { - const catalog = aggregateCatalog([ - view({ catalog: [androidOn({ models: ["Pixel 8", "My Tablet"] })], id: "wrk_a" }), - view({ - catalog: [androidOn({ customModels: ["My Tablet"], models: ["My Tablet"] })], - id: "wrk_b", - }), - ]); - - expect(() => OPERATIONS["catalog.get"].output.parse(catalog)).not.toThrow(); - expect(catalog.platforms[0]?.customModels).toEqual(["My Tablet"]); - }); - - it("drops a custom name the worker does not list in models", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [androidOn({ customModels: ["My Tablet", "Ghost"], models: ["My Tablet"] })], - id: "wrk_a", - }), - // Another worker lists Ghost, but only the worker that marks a name can make it custom. - view({ catalog: [androidOn({ models: ["Ghost"] })], id: "wrk_b" }), - ]); - - expect(catalog.platforms[0]?.customModels).toEqual(["My Tablet"]); - }); - - it("omits customModels when no worker marks a model it lists", () => { - const catalog = aggregateCatalog([ - view({ catalog: [androidOn({ customModels: ["Ghost"] })], id: "wrk_a" }), - ]); - - expect(catalog.platforms[0]).not.toHaveProperty("customModels"); - }); - - it("drops an image whose runtime the reporting worker does not list", () => { - const catalog = aggregateCatalog([ - view({ - catalog: [ - androidOn({ - images: [ - { abi: "x86_64", runtime: "34", tag: "default" }, - { abi: "x86_64", runtime: "99", tag: "default" }, - ], - }), - ], - id: "wrk_a", - }), - ]); - - expect(catalog.platforms[0]?.images).toEqual([ - { abi: "x86_64", runtime: "34", tag: "default" }, - ]); - }); - }); -}); - describe("aggregateStatus at-RAM-budget flag", () => { const withBudget = (ios: boolean, android: boolean) => { const base = capacity(1, 2); diff --git a/src/gateway/test-aggregate.ts b/src/gateway/test-aggregate.ts new file mode 100644 index 000000000..3efc1658b --- /dev/null +++ b/src/gateway/test-aggregate.ts @@ -0,0 +1,13 @@ +import type { WorkerView } from "./worker-registry.js"; + +export function view(overrides: Partial & { readonly id: string }): WorkerView { + return { + catalog: [], + connection: "connected", + devices: [], + drained: false, + lastSeenAt: 1_000, + leases: [], + ...overrides, + }; +} From 182bc1dd30978ba56d660fe047aecb0c7589188a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:23:54 +0100 Subject: [PATCH 24/83] test(ios): split index.test.ts by topic --- .../ios/index-component-install.test.ts | 393 ++ src/drivers/ios/index-device-set.test.ts | 641 ++ src/drivers/ios/index-list-components.test.ts | 274 + src/drivers/ios/index-passthrough.test.ts | 297 + src/drivers/ios/index-provenance-root.test.ts | 358 + .../ios/index-remove-component.test.ts | 852 +++ .../ios/index-save-source-lifecycle.test.ts | 575 ++ .../ios/index-screen-tool-versions.test.ts | 170 + src/drivers/ios/index-seed-provision.test.ts | 460 ++ src/drivers/ios/index-seed-sources.test.ts | 644 ++ src/drivers/ios/index-simulator-audio.test.ts | 167 + src/drivers/ios/index-slim-mode.test.ts | 601 ++ .../ios/index-slim-resolve-advisories.test.ts | 457 ++ src/drivers/ios/index.test.ts | 6009 +---------------- src/drivers/ios/test-index.ts | 360 + src/source-hygiene.test.ts | 1 - 16 files changed, 6410 insertions(+), 5849 deletions(-) create mode 100644 src/drivers/ios/index-component-install.test.ts create mode 100644 src/drivers/ios/index-device-set.test.ts create mode 100644 src/drivers/ios/index-list-components.test.ts create mode 100644 src/drivers/ios/index-passthrough.test.ts create mode 100644 src/drivers/ios/index-provenance-root.test.ts create mode 100644 src/drivers/ios/index-remove-component.test.ts create mode 100644 src/drivers/ios/index-save-source-lifecycle.test.ts create mode 100644 src/drivers/ios/index-screen-tool-versions.test.ts create mode 100644 src/drivers/ios/index-seed-provision.test.ts create mode 100644 src/drivers/ios/index-seed-sources.test.ts create mode 100644 src/drivers/ios/index-simulator-audio.test.ts create mode 100644 src/drivers/ios/index-slim-mode.test.ts create mode 100644 src/drivers/ios/index-slim-resolve-advisories.test.ts create mode 100644 src/drivers/ios/test-index.ts diff --git a/src/drivers/ios/index-component-install.test.ts b/src/drivers/ios/index-component-install.test.ts new file mode 100644 index 000000000..d8759b1fc --- /dev/null +++ b/src/drivers/ios/index-component-install.test.ts @@ -0,0 +1,393 @@ +import { afterEach, expect, describe, it } from "vitest"; +import { RuntimeMissingError, DriverCrashError } from "../../core/index.js"; +import { ScriptedProcessRunner, FakeClock, MemoryFilesystem } from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { deviceRoot, simctl, instanceId } from "./test-slim.js"; +import { + UNSCOPED_LEGACY_CALLS, + createDriver, + listFixture, + listFixtureAfterDownload, + listInvocation, + pairingFixture, + scopedRunners, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + describe("component install", () => { + const runtimeListInvocation = simctl("runtime", "list", "-j"); + const imagesBefore = [ + { build: "22E238", id: "IMG-184", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }, + { build: "23F77", id: "IMG-265", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, + ]; + const imagesAfter = [ + ...imagesBefore, + { build: "22G86", id: "IMG-186", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-6" }, + ]; + + function listed(stdout: string) { + return { match: listInvocation, result: { code: 0, stderr: "", stdout } }; + } + + function imagesListed(images: readonly { build: string; id: string; runtime: string }[]) { + const stdout = JSON.stringify( + Object.fromEntries( + images.map((image) => [ + image.id, + { + build: image.build, + identifier: image.id, + runtimeIdentifier: image.runtime, + state: "Ready", + }, + ]), + ), + ); + return { match: runtimeListInvocation, result: { code: 0, stderr: "", stdout } }; + } + + function xcodebuild(args: readonly string[], extra: object = {}) { + return { match: { args: [...args], command: "xcodebuild" }, ...extra }; + } + + function install( + driver: IosSimctlDriver, + component: string, + options: { onProgress?: (progress: unknown) => void; signal?: AbortSignal } = {}, + ) { + return driver.installComponent(component, { + onProgress: options.onProgress ?? (() => undefined), + signal: options.signal ?? new AbortController().signal, + }); + } + + it("installs an exact version with -buildVersion and no timeout of its own, verifies it in a fresh read, and reports it installed with its image and build", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"]), + listed(listFixtureAfterDownload), + imagesListed(imagesAfter), + ]); + const driver = await createDriver(runner); + + await expect(install(driver, "18.6")).resolves.toEqual({ + outcome: "installed", + receipt: { build: "22G86", image: "IMG-186" }, + version: "18.6", + }); + const xcodebuildCall = runner.calls.find((call) => call.command === "xcodebuild"); + expect(xcodebuildCall?.options).toEqual({ lineEnd: "carriage-return-too" }); + }); + + it("reports already-installed when the runtime the installer leaves was there before it ran", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS"]), + listed(listFixture), + imagesListed(imagesBefore), + ]); + const driver = await createDriver(runner); + + await expect(install(driver, "latest")).resolves.toEqual({ + outcome: "already-installed", + receipt: { build: "23F77", image: "IMG-265" }, + version: "26.5", + }); + }); + + it("installs a bare major with -buildVersion and reports the newest runtime of it", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18"]), + listed(listFixtureAfterDownload), + imagesListed(imagesAfter), + ]); + const driver = await createDriver(runner); + + await expect(install(driver, "18")).resolves.toMatchObject({ + outcome: "installed", + version: "18.6", + }); + }); + + it("refuses a version below the download floor without running anything", async () => { + const runner = new ScriptedProcessRunner([]); + const driver = await createDriver(runner); + + const result = await install(driver, "13.0").catch((error: unknown) => error); + + expect(result).toBeInstanceOf(RuntimeMissingError); + expect(result).toMatchObject({ + downloadable: false, + message: expect.stringContaining("16.0"), + }); + expect(runner.calls).toEqual([]); + }); + + it("fails when xcodebuild exits 0 but the runtime is still not installed", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"]), + listed(listFixture), + imagesListed(imagesBefore), + ]); + const driver = await createDriver(runner); + + const result = await install(driver, "18.6").catch((error: unknown) => error); + + expect(result).toBeInstanceOf(DriverCrashError); + expect((result as Error).message).toContain("iOS 18.6 is still not installed"); + }); + + it("fails with xcodebuild's own output when it exits non-zero", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { + result: { code: 1, stderr: "no network", stdout: "" }, + }), + ]); + const driver = await createDriver(runner); + + const result = await install(driver, "18.6").catch((error: unknown) => error); + + expect(result).toBeInstanceOf(DriverCrashError); + expect((result as Error).message).toContain("no network"); + }); + + it("fails with a non-downloadable RuntimeMissingError when Apple no longer offers the version", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { + result: { code: 70, stderr: "iOS 18.6 is not available for download.", stdout: "" }, + }), + ]); + const driver = await createDriver(runner); + + const result = await install(driver, "18.6").catch((error: unknown) => error); + + expect(result).toBeInstanceOf(RuntimeMissingError); + expect(result).toMatchObject({ downloadable: false }); + }); + + it("reports each percentage xcodebuild prints as progress, and nothing for a line without one", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { + stdoutLines: [ + "Downloading iOS 18.6 Simulator (22G86): 12.5% (1 GB of 8 GB)", + "Verifying download", + "Downloading iOS 18.6 Simulator (22G86): 100.0% (8 GB of 8 GB)", + ], + }), + listed(listFixtureAfterDownload), + imagesListed(imagesAfter), + ]); + const driver = await createDriver(runner); + const progress: unknown[] = []; + + await install(driver, "18.6", { onProgress: (report) => progress.push(report) }); + + expect(progress).toEqual([ + { percent: 12.5, stage: "downloading" }, + { percent: 100, stage: "downloading" }, + ]); + }); + + it("ends xcodebuild when the signal fires, and fails the install", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { hangs: true }), + ]); + const driver = await createDriver(runner); + const controller = new AbortController(); + + let error: unknown; + void install(driver, "18.6", { signal: controller.signal }).catch((caught: unknown) => { + error = caught; + }); + for (let tick = 0; tick < 50 && runner.handles.length < 3; tick += 1) { + await Promise.resolve(); + } + expect(runner.handles).toHaveLength(3); + controller.abort(); + for (let tick = 0; tick < 50 && error === undefined; tick += 1) { + await Promise.resolve(); + } + + // why: Settled within a few microtasks of the signal: the kill ended it, not a timer. + expect(error).toBeInstanceOf(DriverCrashError); + expect((error as Error).message).toContain("was ended before it finished"); + }); + + it.each([ + [ + "an exact version", + { physical: false, model: "iPhone 7", osVersion: "13.0", platform: "ios" as const }, + "13.0", + ], + ["no version", { model: "iPhone 7", platform: "ios" as const }, "15"], + ])( + "reports a missing runtime below the download floor as not downloadable, for %s", + async (_case, request, named) => { + const runner = new ScriptedProcessRunner([listed(pairingFixture)]); + const driver = await createDriver(runner); + + const result = await driver.resolveSpec(request).catch((error: unknown) => error); + + expect(result).toBeInstanceOf(RuntimeMissingError); + expect(result).toMatchObject({ + component: undefined, + downloadable: false, + message: expect.stringContaining(`iOS ${named} predates`), + }); + }, + ); + + it("tells the requester to name an exact release when the download of a bare major fails", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18"], { + result: { code: 1, stderr: "no such build", stdout: "" }, + }), + ]); + const driver = await createDriver(runner); + + const result = await install(driver, "18").catch((error: unknown) => error); + + expect(result).toBeInstanceOf(DriverCrashError); + expect((result as Error).message).toMatch(/tried 18.*no such build.*pass --os /); + }); + + it("names the image whose build matches the runtime's when two images provide one runtime identifier", async () => { + const twoBuilds = JSON.stringify({ + devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, + runtimes: [ + { + buildversion: "23F79", + identifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-5", + isAvailable: true, + name: "iOS 26.5", + supportedDeviceTypes: [], + version: "26.5", + }, + ], + }); + const runner = new ScriptedProcessRunner([ + listed(twoBuilds), + imagesListed([ + { build: "23F77", id: "IMG-OLD", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, + { build: "23F79", id: "IMG-NEW", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, + ]), + ]); + const driver = await createDriver(runner); + + await expect(driver.findComponent("26.5")).resolves.toEqual({ + receipt: { build: "23F79", image: "IMG-NEW" }, + version: "26.5", + }); + }); + + it("kills xcodebuild with SIGKILL when it ignores the SIGTERM its signal sent", async () => { + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { + hangs: true, + ignoresSigterm: true, + }), + ]); + const driver = await createDriver(runner, clock); + const controller = new AbortController(); + + let error: unknown; + void install(driver, "18.6", { signal: controller.signal }).catch((caught: unknown) => { + error = caught; + }); + for (let tick = 0; tick < 50 && runner.handles.length < 3; tick += 1) { + await Promise.resolve(); + } + controller.abort(); + for (let tick = 0; tick < 50; tick += 1) await Promise.resolve(); + expect(error).toBeUndefined(); + + clock.advance(10_000); + for (let tick = 0; tick < 50 && error === undefined; tick += 1) { + await Promise.resolve(); + } + expect(error).toBeInstanceOf(DriverCrashError); + }); + + it("finds an installed version with its receipt, and nothing for latest, a bare major or a missing version", async () => { + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(imagesBefore), + listed(listFixture), + imagesListed(imagesBefore), + listed(listFixture), + imagesListed(imagesBefore), + listed(listFixture), + imagesListed(imagesBefore), + ]); + const driver = await createDriver(runner); + + await expect(driver.findComponent("26.5")).resolves.toEqual({ + receipt: { build: "23F77", image: "IMG-265" }, + version: "26.5", + }); + await expect(driver.findComponent("latest")).resolves.toBeUndefined(); + await expect(driver.findComponent("26")).resolves.toBeUndefined(); + await expect(driver.findComponent("18.6")).resolves.toBeUndefined(); + expect(runner.calls.map((call) => call.command)).not.toContain("xcodebuild"); + }); + + it("states 8 GiB on the configured CoreSimulator volume as the footprint of one install", async () => { + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot: "/Users/agent/Library/Developer/CoreSimulator", + driverConfig: { deviceRoot }, + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + simlockHome: "/home/.simlock", + }); + + expect(driver.componentFootprint).toEqual({ + bytes: 8 * 1024 ** 3, + path: "/Users/agent/Library/Developer/CoreSimulator", + }); + }); + }); +}); diff --git a/src/drivers/ios/index-device-set.test.ts b/src/drivers/ios/index-device-set.test.ts new file mode 100644 index 000000000..1a526fb5d --- /dev/null +++ b/src/drivers/ios/index-device-set.test.ts @@ -0,0 +1,641 @@ +import { describe, it, expect } from "vitest"; +import { + OWNED_ROOT_MARKER_FILE, + OwnedRootError, + PassthroughRefusedError, +} from "../../core/index.js"; +import { + type Filesystem, + ScriptedProcessRunner, + MemoryFilesystem, + FakeClock, + FakeBytePipeLauncher, +} from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { + driverData, + deviceRoot, + simctlArgs, + deviceHubEdit, + plantManagedDevice, + instanceId, +} from "./test-slim.js"; +import { + PHYSICAL_DEVICE, + listFixture, + prepareFull, + seedUdid, + seededDatabase, + simctlPassthrough, + spec, +} from "./test-index.js"; + +describe("IosSimctlDriver device set", () => { + const coreSimulatorRoot = "/CoreSim"; + const runtimeImage = { + build: "23E244", + identifier: "IMG-26.4-23E244", + platformIdentifier: "com.apple.platform.iphonesimulator", + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-4", + sizeBytes: 1_000, + state: "Ready", + version: "26.4", + }; + const imageList = JSON.stringify({ [runtimeImage.identifier]: runtimeImage }); + const emptyDevices = JSON.stringify({ devices: {} }); + + interface Step { + readonly args: readonly string[]; + readonly scoped: boolean; + readonly sources?: true; + readonly stdout?: string; + } + + async function plantDevice(filesystem: Filesystem, devicesDir: string): Promise { + await filesystem.mkdirp(`${devicesDir}/${driverData.udid}/data`); + } + + it.each([ + ["unset", {}, deviceRoot, true, `${deviceRoot}/sources`, false], + ["simlock", { deviceSet: "simlock" }, deviceRoot, true, `${deviceRoot}/sources`, false], + [ + "default", + { deviceSet: "default" }, + `${coreSimulatorRoot}/Devices`, + false, + "/home/.simlock/devices/ios-sources", + true, + ], + ] as const)( + "the driver given deviceSet %s selects simctl scope and whether a ready device needs reboot before grant", + async (_case, config, devicesDir, scoped, sourcesSet, rebootBeforeGrant) => { + const steps: Step[] = [ + { args: ["list", "-j", "devicetypes", "runtimes"], scoped: true, stdout: listFixture }, + { + args: ["create", "simlock-seed-device-1", driverData.deviceTypeId, driverData.runtimeId], + scoped: true, + sources: true, + stdout: `${seedUdid}\n`, + }, + { args: ["boot", seedUdid], scoped: true, sources: true }, + { args: ["bootstatus", seedUdid, "-b"], scoped: true, sources: true }, + { args: ["listapps", seedUdid], scoped: true, sources: true }, + { args: ["shutdown", seedUdid], scoped: true, sources: true }, + { + args: ["clone", seedUdid, driverData.name, devicesDir], + scoped: true, + sources: true, + stdout: `${driverData.udid}\n`, + }, + { args: ["boot", driverData.udid], scoped: true }, + { args: ["bootstatus", driverData.udid, "-b"], scoped: true }, + { args: ["listapps", driverData.udid], scoped: true }, + { args: ["shutdown", driverData.udid], scoped: true }, + { args: ["delete", driverData.udid], scoped: true }, + { args: ["list", "-j", "devices"], scoped: true, stdout: emptyDevices }, + { args: ["runtime", "list", "-j"], scoped: true, stdout: imageList }, + { args: ["list", "-j", "devices"], scoped: false, stdout: emptyDevices }, + { args: ["runtime", "delete", runtimeImage.identifier], scoped: true }, + { args: ["runtime", "list", "-j"], scoped: true, stdout: "{}" }, + { args: ["runtime", "list", "-j"], scoped: true, stdout: "{}" }, + { args: ["list", "-j", "devices"], scoped: false, stdout: emptyDevices }, + ]; + const argvOf = (step: Step): string[] => + step.sources === true + ? ["simctl", "--set", sourcesSet, ...step.args] + : step.scoped && scoped + ? simctlArgs(...step.args) + : ["simctl", ...step.args]; + const runner = new ScriptedProcessRunner( + steps.flatMap((step) => [ + ...(!scoped && step.args[0] === "boot" && step.sources !== true + ? deviceHubEdit( + `Set :DevicePreferences:${driverData.udid}:pasteboardSyncEnabled false`, + ).map((match) => ({ match })) + : []), + { + match: { args: argvOf(step), command: "xcrun" }, + ...(step.stdout === undefined + ? {} + : { result: { code: 0, stderr: "", stdout: step.stdout } }), + }, + ...(!scoped && step.args[0] === "delete" + ? deviceHubEdit(`Delete :DevicePreferences:${driverData.udid}`).map((match) => ({ + match, + })) + : []), + ]), + ); + const filesystem = new MemoryFilesystem(); + await plantDevice(filesystem, devicesDir); + if (scoped) await plantManagedDevice(filesystem); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: scoped ? { deviceRoot, ...config } : config, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + expect(driver.rebootReadyBeforeGrant).toBe(rebootBeforeGrant); + const device = { address: driverData.udid, deviceId: driverData.udid, driverData }; + + await expect( + driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }), + ).resolves.toBeDefined(); + await driver.provision(spec); + await expect(filesystem.exists(`${sourcesSet}/${OWNED_ROOT_MARKER_FILE}`)).resolves.toBe( + !scoped, + ); + await driver.makeReady(device, prepareFull); + await driver.destroy(device); + await driver.listManaged(); + await driver.removeComponent( + { build: runtimeImage.build, image: runtimeImage.identifier }, + { signal: new AbortController().signal }, + ); + + expect( + runner.calls.filter((call) => call.command === "xcrun").map((call) => call.args), + ).toEqual(steps.map(argvOf)); + }, + ); + + it.each([ + ["Default", { deviceSet: "Default" }, "Default", 'but it is "Default"'], + ["the empty string", { deviceSet: "" }, "", 'but it is ""'], + ["a boolean", { deviceSet: true }, "true", "but it is true"], + [ + "default with a deviceRoot", + { deviceSet: "default", deviceRoot: "/Volumes/sims" }, + "default", + 'drivers.ios.deviceRoot names Simlock\'s own set, but drivers.ios.deviceSet is "default". Remove one of them.', + ], + ] as const)( + "refuses iOS with invalid-device-set when deviceSet is %s", + async (_case, config, root, detail) => { + const create = async (driverConfig: Record) => + IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig, + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + simlockHome: "/home/.simlock", + }); + + await expect(create({ deviceSet: "default" })).resolves.toBeInstanceOf(IosSimctlDriver); + const failure = await create(config) + .then(() => undefined) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(OwnedRootError); + expect(failure).toMatchObject({ + message: expect.stringContaining(detail), + path: root, + platform: "ios", + reason: "invalid-device-set", + }); + expect((failure as Error).message).toMatch(/^Refusing the ios device set: /); + }, + ); + + it("an operator who switches with leftovers gets device-set-changed, and an unreadable folder gets unreadable", async () => { + const ownRoot = "/home/.simlock/devices/ios"; + const xcodeDevices = `${coreSimulatorRoot}/Devices`; + const [first, second, third] = [1, 2, 3].map( + (n) => `00000000-0000-0000-0000-00000000000${String(n)}`, + ); + const mark = (owner: string) => JSON.stringify({ instanceId: owner, token: "t", udid: "u" }); + const steps: readonly [ + string, + Record, + (fs: MemoryFilesystem) => Promise, + { readonly message?: string; readonly path?: string; readonly reason: string } | undefined, + ][] = [ + [ + "default with two devices in Simlock's set", + { deviceSet: "default" }, + async (fs) => { + await fs.mkdirp(`${ownRoot}/${second}`); + await fs.mkdirp(`${ownRoot}/${first}`); + await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); + }, + { + message: `Refusing the ios device set: Simlock's own set at ${ownRoot} still holds 2 simulator(s) (${first}, ${second}). Set drivers.ios.deviceSet back to simlock, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot} delete \`, or, for a seed in the sources set, \`xcrun simctl --set ${ownRoot}/sources delete \`.`, + path: ownRoot, + reason: "device-set-changed", + }, + ], + [ + "default with a seed in the sources set Simlock's own set held", + { deviceSet: "default" }, + async (fs) => { + await fs.mkdirp(`${ownRoot}/sources/${third}`); + await fs.mkdirp(`${ownRoot}/${first}`); + await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); + }, + { + message: `Refusing the ios device set: Simlock's own set at ${ownRoot} still holds 2 simulator(s) (${first}, ${third}). Set drivers.ios.deviceSet back to simlock, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot} delete \`, or, for a seed in the sources set, \`xcrun simctl --set ${ownRoot}/sources delete \`.`, + path: ownRoot, + reason: "device-set-changed", + }, + ], + [ + "simlock with a seed in the sources set deviceSet default used", + { deviceSet: "simlock" }, + async (fs) => { + await fs.mkdirp(`${ownRoot}-sources/${second}`); + }, + { + message: `Refusing the ios device set: Simlock's sources set at ${ownRoot}-sources still holds 1 simulator(s) (${second}). Set drivers.ios.deviceSet back to default, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot}-sources delete \`. Run \`simlock nuke --delete-devices\` again afterwards, so Simlock drops its record of them.`, + path: `${ownRoot}-sources`, + reason: "device-set-changed", + }, + ], + [ + "default with Simlock's set empty", + { deviceSet: "default" }, + async (fs) => { + await fs.mkdirp(ownRoot); + await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); + }, + undefined, + ], + ["default with no root folder", { deviceSet: "default" }, async () => undefined, undefined], + [ + "simlock with a device of this instance in Xcode's set", + { deviceSet: "simlock" }, + async (fs) => { + await fs.mkdirp(`${xcodeDevices}/${third}`); + await fs.writeFileAtomic(`${xcodeDevices}/${third}/simlock-mark.json`, mark(instanceId)); + }, + { + message: `Refusing the ios device set: Xcode's device set still holds 1 of this Simlock's simulator(s) (${third}). Set drivers.ios.deviceSet back to default, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl delete \`.`, + path: xcodeDevices, + reason: "device-set-changed", + }, + ], + [ + "simlock with only unmarked, other-instance and corrupt devices in Xcode's set", + { deviceSet: "simlock" }, + async (fs) => { + await fs.mkdirp(`${xcodeDevices}/${first}`); + await fs.mkdirp(`${xcodeDevices}/${second}`); + await fs.writeFileAtomic(`${xcodeDevices}/${second}/simlock-mark.json`, mark("other")); + await fs.mkdirp(`${xcodeDevices}/${third}`); + await fs.writeFileAtomic(`${xcodeDevices}/${third}/simlock-mark.json`, "{broken"); + }, + undefined, + ], + [ + "default with Simlock's folder unreadable", + { deviceSet: "default" }, + async (fs) => { + await fs.mkdirp(ownRoot); + fs.defineFailure(ownRoot, "EACCES"); + }, + { path: ownRoot, reason: "unreadable" }, + ], + [ + "simlock with Xcode's folder unreadable", + { deviceSet: "simlock" }, + async (fs) => { + await fs.mkdirp(xcodeDevices); + fs.defineFailure(xcodeDevices, "EACCES"); + }, + { path: xcodeDevices, reason: "unreadable" }, + ], + ]; + + for (const [name, config, plant, expected] of steps) { + const filesystem = new MemoryFilesystem(); + await plant(filesystem); + const runner = new ScriptedProcessRunner([]); + const failure = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: config, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }) + .then(() => undefined) + .catch((error: unknown) => error); + + expect(runner.calls, name).toEqual([]); + if (expected === undefined) { + expect(failure, name).toBeUndefined(); + } else { + expect(failure, name).toBeInstanceOf(OwnedRootError); + expect(failure, name).toMatchObject({ ...expected, platform: "ios" }); + } + } + }); + + it.each([ + ["simlock", { deviceRoot, deviceSet: "simlock" }, deviceRoot], + ["default", { deviceSet: "default" }, `${coreSimulatorRoot}/Devices`], + ] as const)( + "a new simulator's marks name this instance under deviceSet %s", + async (setting, config, devicesDir) => { + const scoped = setting === "simlock"; + const argv = (...args: string[]) => (scoped ? simctlArgs(...args) : ["simctl", ...args]); + const marks = [ + `${devicesDir}/${driverData.udid}/data/simlock-mark.json`, + `${devicesDir}/${driverData.udid}/simlock-mark.json`, + ]; + const driverFor = async (steps: readonly (readonly [string[], string?])[]) => { + const runner = new ScriptedProcessRunner( + steps.map(([args, stdout]) => ({ + match: { args, command: "xcrun" }, + ...(stdout === undefined ? {} : { result: { code: 0, stderr: "", stdout } }), + })), + ); + const filesystem = new MemoryFilesystem(); + if (scoped) await plantManagedDevice(filesystem); + else await filesystem.mkdirp(`${devicesDir}/${driverData.udid}/data`); + const driver = await IosSimctlDriver.create({ + database: await seededDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: config, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + return { driver, filesystem, runner }; + }; + const created = [ + argv("list", "-j", "devicetypes", "runtimes"), + [ + "simctl", + "--set", + scoped ? `${deviceRoot}/sources` : "/home/.simlock/devices/ios-sources", + "clone", + seedUdid, + driverData.name, + devicesDir, + ], + ]; + + const { driver, filesystem } = await driverFor([ + [created[0] ?? [], listFixture], + [created[1] ?? [], `${driverData.udid}\n`], + ]); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + await driver.provision(spec); + for (const path of marks) { + expect(JSON.parse(await filesystem.readFile(path)), path).toMatchObject({ instanceId }); + } + + if (scoped) return; + const failing = await driverFor([ + [created[0] ?? [], listFixture], + [created[1] ?? [], `${driverData.udid}\n`], + [argv("delete", driverData.udid)], + ]); + await failing.filesystem.rm(`${devicesDir}/${driverData.udid}/data`); + await failing.driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + await expect(failing.driver.provision(spec)).rejects.toThrow(); + expect(failing.runner.calls.at(-1)).toEqual({ + args: argv("delete", driverData.udid), + command: "xcrun", + options: { timeoutMs: 30_000 }, + }); + }, + ); + + it("under deviceSet default listManaged reports only devices whose durable mark names this instance", async () => { + const devicesDir = `${coreSimulatorRoot}/Devices`; + const udid = (n: number) => `00000000-0000-0000-0000-00000000010${String(n)}`; + const mark = (owner?: string) => + JSON.stringify({ ...(owner === undefined ? {} : { instanceId: owner }), token: "t" }); + const filesystem = new MemoryFilesystem(); + const cases = [ + { durable: mark(instanceId), erasable: mark(instanceId), listed: true, name: "simlock-0" }, + { durable: mark(instanceId), erasable: undefined, listed: true, name: "simlock-9" }, + { durable: undefined, erasable: mark(instanceId), listed: false, name: "no-durable" }, + { durable: mark(), erasable: mark(), listed: false, name: "no-instance" }, + { durable: mark("other"), erasable: mark("other"), listed: false, name: "another" }, + { durable: undefined, erasable: undefined, listed: false, name: "simlock-1" }, + ]; + for (const [index, entry] of cases.entries()) { + await filesystem.mkdirp(`${devicesDir}/${udid(index)}/data`); + if (entry.durable !== undefined) { + await filesystem.writeFileAtomic( + `${devicesDir}/${udid(index)}/simlock-mark.json`, + entry.durable, + ); + } + if (entry.erasable !== undefined) { + await filesystem.writeFileAtomic( + `${devicesDir}/${udid(index)}/data/simlock-mark.json`, + entry.erasable, + ); + } + } + const runner = new ScriptedProcessRunner([ + { + match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, + result: { + code: 0, + stderr: "", + stdout: JSON.stringify({ + devices: { + "com.apple.CoreSimulator.SimRuntime.iOS-26-5": cases.map((entry, index) => ({ + dataPath: `${devicesDir}/${udid(index)}/data`, + name: entry.name, + state: "Shutdown", + udid: udid(index), + })), + }, + }), + }, + }, + ]); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: { deviceSet: "default" }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + + const reality = await driver.listManaged(); + + expect(reality.devices.map((device) => [device.deviceId, device.mark?.erasable])).toEqual([ + [udid(0), "t"], + [udid(1), undefined], + ]); + await expect(driver.findLegacy(udid(0))).resolves.toBeUndefined(); + expect(runner.calls).toHaveLength(1); + }); + + it("under deviceSet default foreignDevices and the removal's unused count take only devices this instance's mark does not name", async () => { + const devicesDir = `${coreSimulatorRoot}/Devices`; + const udids = ["A", "B", "C"].map((n) => `0000000${n}-0000-0000-0000-000000000000`); + const devices = (used: boolean) => + JSON.stringify({ + devices: { + [runtimeImage.runtimeIdentifier]: udids.map((udid) => ({ + ...(used ? { lastUsedAt: "2026-10-01T12:00:00Z" } : {}), + name: "d", + state: "Shutdown", + udid, + })), + }, + }); + const driverWith = async (steps: readonly (readonly [readonly string[], string?])[]) => { + const filesystem = new MemoryFilesystem(); + const marked = [instanceId, undefined, "other"] as const; + for (const [index, udid] of udids.entries()) { + await filesystem.mkdirp(`${devicesDir}/${udid}`); + const owner = marked[index]; + if (owner !== undefined) { + await filesystem.writeFileAtomic( + `${devicesDir}/${udid}/simlock-mark.json`, + JSON.stringify({ instanceId: owner, token: "t" }), + ); + } + } + return IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: { deviceSet: "default" }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner( + steps.map(([args, stdout]) => ({ + match: { args: [...args], command: "xcrun" }, + ...(stdout === undefined ? {} : { result: { code: 0, stderr: "", stdout } }), + })), + ), + simlockHome: "/home/.simlock", + }); + }; + const runtimeList = [["simctl", "runtime", "list", "-j"], imageList] as const; + const deviceList = (used: boolean) => + [["simctl", "list", "-j", "devices"], devices(used)] as const; + + const listing = await driverWith([runtimeList, deviceList(true)]); + const [component] = await listing.listComponents(); + expect(component?.foreignDevices).toBe(2); + + const removing = await driverWith([ + runtimeList, + deviceList(false), + [["simctl", "runtime", "delete", runtimeImage.identifier]], + [["simctl", "runtime", "list", "-j"], "{}"], + [["simctl", "runtime", "list", "-j"], "{}"], + deviceList(false), + ]); + const removal = await removing.removeComponent( + { build: runtimeImage.build, image: runtimeImage.identifier }, + { signal: new AbortController().signal }, + ); + expect(removal.residue).toContain("2 never-used simulators"); + }); + + it.each([ + ["simlock", { deviceRoot, deviceSet: "simlock" }, { SIMLOCK_IOS_DEVICE_SET: deviceRoot }], + ["default", { deviceSet: "default" }, {}], + ] as const)( + "a lease holder and the screen capture get the device set only under deviceSet %s", + async (_setting, config, leaseEnvironment) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/dist/bin"); + await filesystem.writeFileAtomic("/dist/bin/simlock-screen-capture", ""); + const launcher = new FakeBytePipeLauncher(); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: config, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + screenCapture: { + baseEnvironment: { PATH: "/usr/bin", SIMLOCK_IOS_DEVICE_SET: "/from/the/daemon" }, + command: "/dist/bin/simlock-screen-capture", + launcher, + }, + simlockHome: "/home/.simlock", + }); + const capture = driver.screenCapture; + if (capture === undefined) throw new Error("the driver offers no screen capture"); + + await capture.open( + { address: driverData.udid, deviceId: driverData.udid, driverData, physical: false }, + { bitrateKbps: 1_500, fps: 30, maxDimension: 720 }, + new AbortController().signal, + ); + + expect(driver.virtualLeaseEnvironment()).toEqual(leaseEnvironment); + expect( + driver.leaseEnvironment({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + }), + ).toEqual(leaseEnvironment); + expect(driver.leaseEnvironment(PHYSICAL_DEVICE)).toEqual({ + SIMLOCK_DEVICE_UDID: PHYSICAL_DEVICE.deviceId, + }); + expect(launcher.starts[0]?.env).toEqual({ + PATH: "/usr/bin", + ...leaseEnvironment, + }); + }, + ); + + it("simlock simctl under deviceSet default runs plain xcrun simctl and keeps its refusals", async () => { + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + coreSimulatorRoot, + driverConfig: { deviceSet: "default" }, + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + simlockHome: "/home/.simlock", + }); + + await expect( + simctlPassthrough(driver, ["install", driverData.udid, "app.app"]), + ).resolves.toEqual({ + args: ["simctl", "install", driverData.udid, "app.app"], + command: "xcrun", + env: {}, + }); + for (const refused of [ + ["erase", driverData.udid], + ["shutdown", "all"], + ["--set", "/tmp", "list"], + ]) { + await expect(simctlPassthrough(driver, refused), refused.join(" ")).rejects.toBeInstanceOf( + PassthroughRefusedError, + ); + } + }); +}); diff --git a/src/drivers/ios/index-list-components.test.ts b/src/drivers/ios/index-list-components.test.ts new file mode 100644 index 000000000..df102be51 --- /dev/null +++ b/src/drivers/ios/index-list-components.test.ts @@ -0,0 +1,274 @@ +import { afterEach, expect, it, describe } from "vitest"; +import { DriverCrashError } from "../../core/index.js"; +import { ScriptedProcessRunner } from "../../ports/index.js"; +import { deviceRoot } from "./test-slim.js"; +import { + Image, + UNSCOPED_LEGACY_CALLS, + createDriver, + defaultSetInvocation, + defaultSetListed, + image, + imagesListed, + listFixture, + listFixtureAfterDownload, + listInvocation, + runtimeListInvocation, + scopedRunners, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver listComponents()", () => { + it("lists one entry per iOS runtime image with its version, size and build, and no entry for another platform's runtime", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([ + image("18.4", "22E238", { sizeBytes: 8_963_458_461 }), + image("26.0", "23A343", { sizeBytes: 9_100_000_000 }), + image("11.4", "22T250", { + platform: "com.apple.platform.watchsimulator", + runtime: "com.apple.CoreSimulator.SimRuntime.watchOS-11-4", + }), + ]), + defaultSetListed({}), + ]); + const driver = await createDriver(runner); + + await expect(driver.listComponents()).resolves.toEqual([ + { + foreignDevices: 0, + receipt: { build: "22E238", image: "IMG-18.4-22E238" }, + sizeBytes: 8_963_458_461, + variant: "22E238", + version: "18.4", + }, + { + foreignDevices: 0, + receipt: { build: "23A343", image: "IMG-26.0-23A343" }, + sizeBytes: 9_100_000_000, + variant: "23A343", + version: "26.0", + }, + ]); + }); + + it("counts a device in the default device set that has lastUsedAt and uses an image's runtime as foreign, and reads no device in Simlock's own set for it", async () => { + const used = image("18.4", "22E238"); + const unused = image("26.0", "23A343"); + // why: The scripted runner refuses any call it was not given, so a `simctl --set list` + // would fail this test: only the default set is read. + const runner = new ScriptedProcessRunner([ + imagesListed([used, unused]), + defaultSetListed({ [used.runtime]: 2, "com.apple.CoreSimulator.SimRuntime.iOS-17-5": 1 }), + ]); + const driver = await createDriver(runner); + + const listed = await driver.listComponents(); + + expect(listed.map(({ foreignDevices, version }) => ({ foreignDevices, version }))).toEqual([ + { foreignDevices: 2, version: "18.4" }, + { foreignDevices: 0, version: "26.0" }, + ]); + expect(runner.calls.map((call) => call.args)).toEqual([ + runtimeListInvocation.args, + defaultSetInvocation.args, + ]); + }); + + it("does not count a device in the default device set without lastUsedAt, such as the ones macOS creates when a runtime install ends (#241)", async () => { + const fresh = image("18.6", "22G86"); + const runner = new ScriptedProcessRunner([ + imagesListed([fresh]), + // why: What a runtime install leaves on Xcode 26.4.1: about eleven never-booted simulators of + // it. One more the user has booted is the only one that counts. + defaultSetListed({ [fresh.runtime]: { unused: 11, used: 1 } }), + ]); + const driver = await createDriver(runner); + + const listed = await driver.listComponents(); + + expect(listed.map(({ foreignDevices }) => foreignDevices)).toEqual([1]); + }); + + it("counts a default-set entry that is not a device object as a user, so an unreadable entry never unblocks a removal (#241)", async () => { + const fresh = image("18.6", "22G86"); + const runner = new ScriptedProcessRunner([ + imagesListed([fresh]), + { + match: defaultSetInvocation, + result: { + code: 0, + stderr: "", + stdout: JSON.stringify({ + devices: { [fresh.runtime]: ["garbled", null, { name: "fresh", udid: "U-1" }] }, + }), + }, + }, + ]); + const driver = await createDriver(runner); + + const listed = await driver.listComponents(); + + expect(listed.map(({ foreignDevices }) => foreignDevices)).toEqual([2]); + }); + + it("lists a runtime with the receipt installComponent returned for it, and runs no installer to list", async () => { + const before = [ + { build: "22E238", id: "IMG-184", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }, + ]; + const after = [ + ...before, + { build: "22G86", id: "IMG-186", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-6" }, + ]; + const asImages = (entries: typeof after): Image[] => + entries.map((entry) => ({ + ...entry, + version: entry.runtime.endsWith("18-6") ? "18.6" : "18.4", + })); + const listed = (stdout: string) => ({ + match: listInvocation, + result: { code: 0, stderr: "", stdout }, + }); + const runner = new ScriptedProcessRunner([ + listed(listFixture), + imagesListed(asImages(before)), + { + match: { + args: ["-downloadPlatform", "iOS", "-buildVersion", "18.6"], + command: "xcodebuild", + }, + }, + listed(listFixtureAfterDownload), + imagesListed(asImages(after)), + imagesListed(asImages(after)), + defaultSetListed({}), + ]); + const driver = await createDriver(runner); + const installed = await driver.installComponent("18.6", { + onProgress: () => undefined, + signal: new AbortController().signal, + }); + const callsBeforeListing = runner.calls.length; + + const components = await driver.listComponents(); + + expect(components.find((component) => component.version === "18.6")?.receipt).toEqual( + installed.receipt, + ); + expect(runner.calls.slice(callsBeforeListing).map((call) => call.command)).toEqual([ + "xcrun", + "xcrun", + ]); + }); + + it("leaves out an image with no version or no platform, leaves out a size that is not a byte count, and counts no device under a runtime that lists none", async () => { + const ios = "com.apple.platform.iphonesimulator"; + const runtimeList = { + "IMG-A": { + build: "22E238", + identifier: "IMG-A", + platformIdentifier: ios, + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-18-4", + sizeBytes: -1, + version: "18.4", + }, + "IMG-B": { + build: "23A343", + identifier: "IMG-B", + platformIdentifier: ios, + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-0", + sizeBytes: 1.5, + }, + "IMG-C": { + build: "23A344", + identifier: "IMG-C", + runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-1", + sizeBytes: 10, + version: "26.1", + }, + }; + const runner = new ScriptedProcessRunner([ + { + match: runtimeListInvocation, + result: { code: 0, stderr: "", stdout: JSON.stringify(runtimeList) }, + }, + { + match: defaultSetInvocation, + result: { + code: 0, + stderr: "", + stdout: JSON.stringify({ + devices: { "com.apple.CoreSimulator.SimRuntime.iOS-18-4": "not a list" }, + }), + }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.listComponents()).resolves.toEqual([ + { + foreignDevices: 0, + receipt: { build: "22E238", image: "IMG-A" }, + variant: "22E238", + version: "18.4", + }, + ]); + }); + + it("rejects with DriverCrashError when the default device set answers with something that is not JSON", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([image("18.4", "22E238")]), + { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout: "not json" } }, + ]); + const driver = await createDriver(runner); + + const error = await driver.listComponents().catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DriverCrashError); + expect((error as Error).message).toContain("Could not parse simctl device list"); + }); + + it("rejects with DriverCrashError when the default device set answers JSON with no devices map", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([image("18.4", "22E238")]), + { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout: "{}" } }, + ]); + const driver = await createDriver(runner); + + const error = await driver.listComponents().catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DriverCrashError); + expect((error as Error).message).toBe("Invalid simctl device list JSON"); + }); + + it("rejects when the default device set cannot be read, rather than report no foreign devices", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([image("18.4", "22E238")]), + { + match: defaultSetInvocation, + result: { code: 1, stderr: "CoreSimulator is down", stdout: "" }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.listComponents()).rejects.toBeInstanceOf(DriverCrashError); + }); +}); diff --git a/src/drivers/ios/index-passthrough.test.ts b/src/drivers/ios/index-passthrough.test.ts new file mode 100644 index 000000000..f34583dc9 --- /dev/null +++ b/src/drivers/ios/index-passthrough.test.ts @@ -0,0 +1,297 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it, describe } from "vitest"; +import { PassthroughRefusedError } from "../../core/index.js"; +import { + ScriptedProcessRunner, + SystemClock, + NodeFilesystem, + NodeProcessRunner, +} from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { deviceRoot } from "./test-slim.js"; +import { + UNSCOPED_LEGACY_CALLS, + createDriver, + prepareFull, + scopedRunners, + simctlPassthrough, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + it("scopes a simctl passthrough to the device set the way its own calls are scoped", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, ["install", "booted", "./MyApp.app"])).resolves.toEqual({ + args: ["simctl", "--set", deviceRoot, "install", "booted", "./MyApp.app"], + command: "xcrun", + env: {}, + }); + }); + + it("fills a simctl device verb's missing device with the lease's UDID", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + const udid = "6F1D2A8C-3B4E-4F7A-9C2D-8E5B1A7F3C90"; + const simulator = { address: "", deviceId: udid, driverData: {}, physical: false } as const; + const scoped = ["simctl", "--set", deviceRoot]; + const cases: readonly (readonly [readonly string[], readonly string[]])[] = [ + [ + ["install", "/b/App.app"], + ["install", udid, "/b/App.app"], + ], + [ + ["launch", "--console", "com.x"], + ["launch", "--console", udid, "com.x"], + ], + [ + ["launch", "-a", "arm64", "com.x"], + ["launch", "-a", "arm64", udid, "com.x"], + ], + [ + ["spawn", "-s", "launchctl", "list"], + ["spawn", "-s", udid, "launchctl", "list"], + ], + [ + ["io", "screenshot", "/tmp/s.png"], + ["io", udid, "screenshot", "/tmp/s.png"], + ], + [ + ["install", udid, "/b/App.app"], + ["install", udid, "/b/App.app"], + ], + [ + ["install", udid.toLowerCase(), "/b/App.app"], + ["install", udid.toLowerCase(), "/b/App.app"], + ], + [ + ["list", "devices"], + ["list", "devices"], + ], + [ + ["runtime", "list"], + ["runtime", "list"], + ], + ]; + + for (const [args, expected] of cases) { + await expect( + driver.passthrough("simctl", simulator, args, { hasTerminal: false, platform: "ios" }), + ).resolves.toEqual({ args: [...scoped, ...expected], command: "xcrun", env: {} }); + } + }); + + it("refuses a simctl command that names another device, or that runs on another platform", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + const udid = "6F1D2A8C-3B4E-4F7A-9C2D-8E5B1A7F3C90"; + const other = "0A1B2C3D-0000-4000-8000-123456789ABC"; + const simulator = { address: "", deviceId: udid, driverData: {}, physical: false } as const; + const refusal = (command: string, named: string) => + `Refusing \`simctl ${command}\`: it names device ${named}, and this lease holds ${udid}. Leave the device out; Simlock fills it in.`; + const cases: readonly (readonly [readonly string[], string])[] = [ + [["install", other, "/b/App.app"], refusal(`install ${other} /b/App.app`, other)], + [["install", "booted", "/b/App.app"], refusal("install booted /b/App.app", "booted")], + [ + ["launch", "--console", "booted", "com.x"], + refusal("launch --console booted com.x", "booted"), + ], + ]; + + for (const [args, message] of cases) { + await expect( + driver.passthrough("simctl", simulator, args, { hasTerminal: false, platform: "ios" }), + ).rejects.toMatchObject({ message, tool: "simctl" }); + } + await expect( + driver.passthrough("simctl", simulator, ["install", "/b/App.app"], { + hasTerminal: false, + platform: "android", + }), + ).rejects.toMatchObject({ + message: + "Refusing `simctl`: it only serves an iOS simulator, and this lease is on the android platform.", + tool: "simctl", + }); + }); + + it.each([["create"], ["erase"], ["delete"]])( + "refuses to proxy %s, naming the command that reclaims a device properly", + async (verb) => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow( + PassthroughRefusedError, + ); + await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow(/simlock release/); + await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow(/simlock cleanup/); + }, + ); + + it("finds the refused verb past a leading flag rather than only in first position", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, ["--verbose", "delete", "ABCD"])).rejects.toThrow( + PassthroughRefusedError, + ); + }); + + it("proxies a refused word that is an argument rather than the subcommand", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect((await simctlPassthrough(driver, ["spawn", "booted", "log", "erase"])).args).toContain( + "erase", + ); + }); + + it.each([ + [["--profiles", "/tmp", "erase", "all"]], + [["--set", "/tmp", "delete", "all"]], + [["--set=/tmp", "create", "sim"]], + ])( + "refuses a caller-supplied scoping flag rather than reading its value as the subcommand: %j", + async (args) => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + // why: The bug this pins was not the flag itself: the flag's separated value was found as + // the subcommand, so the refused verb behind it was never seen, and the wrapper + // answered with `--set ` prepended -- Simlock handing over the very path + // that contains every other agent's devices. + await expect(simctlPassthrough(driver, args)).rejects.toThrow(PassthroughRefusedError); + await expect(simctlPassthrough(driver, args)).rejects.toThrow( + /supplies the device set itself/, + ); + }, + ); + + it("refuses any unrecognized flag ahead of the subcommand, rather than skipping over it as presumed harmless", async () => { + // why: Round 4, F6: structurally the same escape round 3 closed on the Android side + // (`--reply-fd 9 -H attacker.example`). Before this fix, an unrecognized `-...` token was + // treated as a no-value flag and the scan continued to the next argument -- so a + // caller-supplied global's *value* was read as the subcommand, and the real + // `--set`/`--profiles` behind it was never seen by `#assertProxyable` at all: this exact + // line used to stop the scan at `/tmp` (mistaken for the subcommand), so the trailing + // `--set /evil` was never scanned for refusal and rode through to `xcrun simctl` unchecked. + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect( + simctlPassthrough(driver, ["-x", "/tmp", "--set", "/evil", "boot", "UDID"]), + ).rejects.toThrow(PassthroughRefusedError); + }); + + it("refuses to shut down every device in the set at once", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, ["shutdown", "all"])).rejects.toThrow( + PassthroughRefusedError, + ); + await expect(simctlPassthrough(driver, ["shutdown", "all"])).rejects.toThrow(/simlock release/); + }); + + it("still proxies shutting a single device down by udid", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect((await simctlPassthrough(driver, ["shutdown", "ABCD"])).args).toEqual([ + "simctl", + "--set", + deviceRoot, + "shutdown", + "ABCD", + ]); + }); + + it("refuses to delete a runtime shared with Xcode and other tools", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( + PassthroughRefusedError, + ); + await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( + /shared with Xcode and other tools/, + ); + await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( + /through Xcode/, + ); + }); + + it("does not say in the runtime delete refusal that iOS runtimes cannot be downloaded", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + let message = ""; + try { + await simctlPassthrough(driver, ["runtime", "delete", "26.5"]); + } catch (error) { + message = (error as Error).message; + } + + expect(message).toMatch(/shared with Xcode/); + expect(message).not.toMatch(/download|install/i); + }); + + it("points the runtime delete refusal at simlock component remove", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( + /use `simlock component remove ios `/, + ); + }); + + it("still proxies the runtime operations that only read", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect((await simctlPassthrough(driver, ["runtime", "list"])).args).toEqual([ + "simctl", + "--set", + deviceRoot, + "runtime", + "list", + ]); + }); + + it.skipIf(process.env.SIMLOCK_LIVE_IOS !== "1")( + "runs a provision-to-destroy smoke test against simctl", + async () => { + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new SystemClock(), + driverConfig: {}, + filesystem: new NodeFilesystem(), + idGenerator: { generate: () => `test-${process.pid}` }, + instanceId: `live-${process.pid}`, + processRunner: new NodeProcessRunner(), + simlockHome: join(tmpdir(), `simlock-live-ios-${process.pid}`), + }); + let device: Awaited> | undefined; + + try { + const liveSpec = await driver.resolveSpec({ model: "iPhone 17 Pro", platform: "ios" }); + device = await driver.provision(liveSpec); + await driver.makeReady(device, prepareFull); + } finally { + if (device !== undefined) { + await driver.destroy(device); + } + } + }, + 150_000, + ); +}); diff --git a/src/drivers/ios/index-provenance-root.test.ts b/src/drivers/ios/index-provenance-root.test.ts new file mode 100644 index 000000000..2248912ae --- /dev/null +++ b/src/drivers/ios/index-provenance-root.test.ts @@ -0,0 +1,358 @@ +import { readFileSync } from "node:fs"; +import { afterEach, expect, it, describe } from "vitest"; +import { OWNED_ROOT_MARKER_FILE, OwnedRootError } from "../../core/index.js"; +import { ScriptedProcessRunner, MemoryFilesystem, FakeClock } from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { deviceRoot, driverData, dataPath, plantManagedDevice, instanceId } from "./test-slim.js"; +import { + PHYSICAL_DEVICE, + UNSCOPED_LEGACY_CALLS, + createDriver, + deviceListResponse, + legacyUdid, + listDevicesInvocation, + listFixture, + listInvocation, + scopedRunners, + scriptedListRunner, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +const listDevicesFixture = readFileSync( + new URL("./fixtures/simctl-list-devices.json", import.meta.url), + "utf8", +); + +const durableMarkPath = `${deviceRoot}/${driverData.udid}/simlock-mark.json`; + +const erasableMarkPath = `${dataPath}/simlock-mark.json`; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + it("shells out to simctl exactly once per listCatalog call", async () => { + const runner = scriptedListRunner(); + const driver = await createDriver(runner); + + await driver.listCatalog(); + + expect(runner.calls).toEqual([{ ...listInvocation, options: { timeoutMs: 30_000 } }]); + }); + + it("reports every device in its set, whatever the device is named", async () => { + const runner = new ScriptedProcessRunner([ + { match: listDevicesInvocation, result: { code: 0, stderr: "", stdout: listDevicesFixture } }, + ]); + const driver = await createDriver(runner); + + const reality = await driver.listManaged(); + + // why: The fixture's last device carries no `simlock-` prefix. Membership in the set is the + // ownership proof (safety rule 8), so it is Simlock's like every other entry here. + expect( + reality.devices.map((device) => ({ deviceId: device.deviceId, runState: device.runState })), + ).toEqual([ + { deviceId: "00000000-0000-0000-0000-000000000101", runState: "running" }, + { deviceId: "00000000-0000-0000-0000-000000000102", runState: "stopped" }, + { deviceId: "00000000-0000-0000-0000-000000000103", runState: "transitioning" }, + { deviceId: "00000000-0000-0000-0000-000000000104", runState: "transitioning" }, + { deviceId: "00000000-0000-0000-0000-000000000105", runState: "running" }, + ]); + }); + + it("populates processes from booted managed devices only", async () => { + const runner = new ScriptedProcessRunner([ + { match: listDevicesInvocation, result: { code: 0, stderr: "", stdout: listDevicesFixture } }, + ]); + const driver = await createDriver(runner); + + const reality = await driver.listManaged(); + + expect(reality.processes.map((device) => device.deviceId)).toEqual([ + "00000000-0000-0000-0000-000000000101", + "00000000-0000-0000-0000-000000000105", + ]); + }); + + it("leaves neither provenance mark when the erasable half cannot be written", async () => { + const filesystem = new MemoryFilesystem(); + const runner = new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: listFixture }, + }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); + // why: The device directory exists, its data container does not, and `writeFileAtomic` + // creates no parents -- the erasable write cannot land. + await filesystem.mkdirp(`${deviceRoot}/${driverData.udid}`); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + await expect(driver.provision(spec)).rejects.toThrow( + `ENOENT: no such file or directory, open '${dataPath}'`, + ); + + // why: A durable mark standing alone is exactly what `Doctor` reads as a foreign erase, so + // a half-written pair would have `doctor` accusing the user of erasing the device + // Simlock itself just erased. Neither half is what "never marked" looks like. + expect(await filesystem.exists(durableMarkPath)).toBe(false); + expect(await filesystem.exists(erasableMarkPath)).toBe(false); + }); + + it("reports matching provenance tokens for a healthy managed device", async () => { + const filesystem = new MemoryFilesystem(); + const runner = new ScriptedProcessRunner([ + { + match: listDevicesInvocation, + result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + await plantManagedDevice(filesystem); + await filesystem.writeFileAtomic(durableMarkPath, JSON.stringify({ token: "tok-1" })); + await filesystem.writeFileAtomic(erasableMarkPath, JSON.stringify({ token: "tok-1" })); + + const reality = await driver.listManaged(); + + expect(reality.devices).toEqual([ + expect.objectContaining({ + mark: { durable: "tok-1", erasable: "tok-1", erasableReadable: true }, + }), + ]); + }); + + it("reports erasable undefined when the data-container mark is gone (foreign erase)", async () => { + const filesystem = new MemoryFilesystem(); + const runner = new ScriptedProcessRunner([ + { + match: listDevicesInvocation, + result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + await plantManagedDevice(filesystem); + await filesystem.writeFileAtomic(durableMarkPath, JSON.stringify({ token: "tok-1" })); + + const reality = await driver.listManaged(); + + expect(reality.devices).toEqual([ + expect.objectContaining({ + mark: { durable: "tok-1", erasable: undefined, erasableReadable: true }, + }), + ]); + }); + + it("reports mark undefined when both provenance regions are absent (pre-upgrade device)", async () => { + const filesystem = new MemoryFilesystem(); + const runner = new ScriptedProcessRunner([ + { + match: listDevicesInvocation, + result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + await plantManagedDevice(filesystem); + + const reality = await driver.listManaged(); + + expect(reality.devices[0]?.mark).toBeUndefined(); + }); + + it("reads a corrupt mark file as undefined without throwing", async () => { + const filesystem = new MemoryFilesystem(); + const runner = new ScriptedProcessRunner([ + { + match: listDevicesInvocation, + result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + await plantManagedDevice(filesystem); + await filesystem.writeFileAtomic(durableMarkPath, "not json"); + await filesystem.writeFileAtomic(erasableMarkPath, JSON.stringify({ notAToken: true })); + + const reality = await driver.listManaged(); + + expect(reality.devices[0]?.mark).toBeUndefined(); + }); + + it("creates and marks its device root under SIMLOCK_HOME when none is configured", async () => { + const filesystem = new MemoryFilesystem(); + + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + driverConfig: {}, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + simlockHome: "/home/.simlock", + }); + + expect(driver.deviceRoot).toBe("/home/.simlock/devices/ios"); + await expect( + filesystem.exists(`/home/.simlock/devices/ios/${OWNED_ROOT_MARKER_FILE}`), + ).resolves.toBe(true); + }); + + it("does not start when its device root belongs to another Simlock instance", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(deviceRoot); + await filesystem.writeFileAtomic( + `${deviceRoot}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ + instanceId: "someone-else", + owner: "simlock", + platform: "ios", + schemaVersion: 1, + }), + ); + + const failure = await createDriver(new ScriptedProcessRunner([]), new FakeClock(), filesystem) + .then(() => undefined) + .catch((error: unknown) => error); + + // why: Fails closed: no fallback to the machine's default device set (safety rule 9). + expect(failure).toBeInstanceOf(OwnedRootError); + expect(failure).toMatchObject({ platform: "ios", reason: "wrong-instance" }); + }); + + it("refuses the platform, not the daemon, when the configured device root is not a path", async () => { + const failure = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + driverConfig: { deviceRoot: true }, + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + simlockHome: "/home/.simlock", + }) + .then(() => undefined) + .catch((error: unknown) => error); + + // why: An `OwnedRootError` costs iOS alone; anything else takes the daemon down with it, + // and with it every way of finding out why -- `"deviceRoot": true` and + // `"deviceRoot": "devices/ios"` must not differ by that much. + expect(failure).toBeInstanceOf(OwnedRootError); + expect(failure).toMatchObject({ platform: "ios", reason: "not-absolute" }); + expect(failure).toMatchObject({ message: expect.stringContaining("true") }); + }); + + it("re-proves an intact root by re-running the validation its start was judged by", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(driver.revalidateRoot()).resolves.toBeUndefined(); + }); + + it("refuses to re-prove a root that has become a symlink since startup", async () => { + const filesystem = new MemoryFilesystem(); + const driver = await createDriver(new ScriptedProcessRunner([]), new FakeClock(), filesystem); + // why: The case the re-proof exists for: ownership was proven at startup and the path has + // been swapped since, so `listManaged` now answers from somewhere else entirely. + filesystem.defineSymlink(deviceRoot, "/Users/someone/Library/Developer/CoreSimulator/Devices"); + + const failure = await driver + .revalidateRoot() + .then(() => undefined) + .catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(OwnedRootError); + expect(failure).toMatchObject({ platform: "ios", reason: "symlink" }); + }); + + it("finds a pre-root device in the machine's default set, without --set", async () => { + const runner = new ScriptedProcessRunner([ + { + match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, + result: { + code: 0, + stderr: "", + stdout: JSON.stringify({ + devices: { + "com.apple.CoreSimulator.SimRuntime.iOS-26-5": [ + { + dataPath: `/Library/Devices/${legacyUdid}/data`, + name: "simlock-old", + state: "Shutdown", + udid: legacyUdid, + }, + ], + }, + }), + }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.findLegacy(legacyUdid)).resolves.toMatchObject({ + device: { deviceId: legacyUdid }, + path: `/Library/Devices/${legacyUdid}`, + }); + }); + + it("reports no pre-root device when the default set does not hold the udid", async () => { + const runner = new ScriptedProcessRunner([ + { + match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, + result: { code: 0, stderr: "", stdout: JSON.stringify({ devices: {} }) }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.findLegacy(legacyUdid)).resolves.toBeUndefined(); + }); + + it("destroys a pre-root device through the unscoped path it actually lives on", async () => { + const runner = new ScriptedProcessRunner([ + { match: { args: ["simctl", "shutdown", legacyUdid], command: "xcrun" } }, + { match: { args: ["simctl", "delete", legacyUdid], command: "xcrun" } }, + ]); + const driver = await createDriver(runner); + + await driver.destroyLegacy({ + address: legacyUdid, + deviceId: legacyUdid, + driverData: { ...driverData, udid: legacyUdid }, + }); + + expect(runner.calls.map((call) => call.args)).toEqual([ + ["simctl", "shutdown", legacyUdid], + ["simctl", "delete", legacyUdid], + ]); + }); + + it("answers uninstall for the reclaim strategy of a physical device", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "standard" })).toBe("uninstall"); + expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "full" })).toBe("uninstall"); + }); +}); diff --git a/src/drivers/ios/index-remove-component.test.ts b/src/drivers/ios/index-remove-component.test.ts new file mode 100644 index 000000000..1aa5cf583 --- /dev/null +++ b/src/drivers/ios/index-remove-component.test.ts @@ -0,0 +1,852 @@ +import { afterEach, expect, describe, it, vi } from "vitest"; +import { ComponentInUseError, ComponentNotOwnedError, DriverCrashError } from "../../core/index.js"; +import { MemoryFilesystem, ScriptedProcessRunner, FakeClock } from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { memorySourceDatabase, plantSource } from "./testing.js"; +import { + deviceRoot, + simctl, + driverData, + waitForCalls, + flushMicrotasks221, + settledValue, +} from "./test-slim.js"; +import { + IOS_RUNTIME_ASSET_ROOT, + RecordingLogger, + SizedFilesystem, + UNSCOPED_LEGACY_CALLS, + assetInfoPlist, + createDriver, + defaultSetInvocation, + defaultSetListed, + image, + imagesListed, + listFixture, + listInvocation, + scopedRunners, + scriptedListRunner, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver listComponents()", () => { + describe("removeComponent()", () => { + const target = image("26.4", "23E244", { sizeBytes: 7_000_000_000 }); + const other = image("18.4", "22E238"); + const receipt = { build: "23E244", image: "IMG-26.4-23E244" }; + const deleteInvocation = simctl("runtime", "delete", "IMG-26.4-23E244"); + const deleted = { match: deleteInvocation, result: { code: 0, stderr: "", stdout: "" } }; + const signal = () => new AbortController().signal; + + async function assetStore( + builds: readonly (readonly [string, string])[], + ): Promise { + return addAssets(new MemoryFilesystem(), builds); + } + + async function addAssets( + filesystem: MemoryFilesystem, + builds: readonly (readonly [string, string])[], + ): Promise { + for (const [index, [version, build]] of builds.entries()) { + const bundle = `${IOS_RUNTIME_ASSET_ROOT}/bundle-${String(index)}.asset`; + await filesystem.mkdirp(bundle); + await filesystem.writeFileAtomic(`${bundle}/Info.plist`, assetInfoPlist(version, build)); + } + return filesystem; + } + + it("deletes the image the receipt names with simctl runtime delete and no --keep-asset, and reports no residue when it had no unused default-set device and its download is gone", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target, other]), + defaultSetListed({ [other.runtime]: { unused: 4 } }), + deleted, + imagesListed([other]), + imagesListed([other]), + defaultSetListed({ [other.runtime]: { unused: 4 } }), + ]); + const filesystem = await assetStore([ + ["18.4", "22E238"], + ["26.4", "23E200"], + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + await expect(driver.removeComponent(receipt, { signal: signal() })).resolves.toEqual({ + sizeBytes: 7_000_000_000, + }); + expect(runner.calls[2]?.args).toEqual(deleteInvocation.args); + }); + + it("deletes the seeds made from the removed runtime before the runtime, logs one it cannot delete and goes on, and keeps the seeds of another runtime", async () => { + const [goneSeed, stuckSeed, keptSeed] = [1, 2, 3].map( + (n) => `00000000-0000-0000-0000-0000000002${String(n)}0`, + ) as [string, string, string]; + const database = await memorySourceDatabase(); + const removed = { + deviceTypeId: driverData.deviceTypeId, + runtimeId: target.runtime, + }; + plantSource(database, { ...removed, udid: goneSeed }); + plantSource(database, { + ...removed, + deviceTypeId: "com.apple.CoreSimulator.SimDeviceType.iPad", + udid: stuckSeed, + }); + plantSource(database, { + deviceTypeId: driverData.deviceTypeId, + runtimeId: other.runtime, + udid: keptSeed, + }); + const runner = new ScriptedProcessRunner([ + imagesListed([target, other]), + defaultSetListed({}), + { match: sourcesSimctl("shutdown", goneSeed) }, + { match: sourcesSimctl("delete", goneSeed) }, + { match: sourcesSimctl("shutdown", stuckSeed) }, + { + match: sourcesSimctl("delete", stuckSeed), + result: { code: 1, stderr: "busy", stdout: "" }, + }, + deleted, + imagesListed([other]), + imagesListed([other]), + defaultSetListed({}), + ]); + const logger = new RecordingLogger(); + const driver = await createDriver( + runner, + new FakeClock(), + await assetStore([]), + database, + logger, + ); + + await driver.removeComponent(receipt, { signal: signal() }); + + expect(runner.calls.map((call) => [call.args[3], call.args[4]])).toEqual( + expect.arrayContaining([ + ["shutdown", goneSeed], + ["delete", goneSeed], + ["delete", stuckSeed], + ["runtime", "delete"], + ]), + ); + expect(runner.calls.some((call) => call.args.includes(keptSeed))).toBe(false); + expect( + runner.calls.findIndex((call) => call.args[3] === "runtime" && call.args[4] === "delete"), + ).toBeGreaterThan(runner.calls.findIndex((call) => call.args[4] === stuckSeed)); + expect(logger.warnings).toEqual([ + expect.stringContaining( + `could not remove seed ${stuckSeed} of removed runtime ${target.runtime}:`, + ), + ]); + expect(database.prepare("SELECT udid FROM ios_source_data ORDER BY udid").all()).toEqual([ + { udid: stuckSeed }, + { udid: keptSeed }, + ]); + }); + + it("starts the five-minute deletion budget after the seeds are deleted, so a seed held by a late clone does not use it up", async () => { + const live = image("26.5", "23F77"); + const runtimeDelete = simctl("runtime", "delete", live.id); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, + imagesListed([live]), + defaultSetListed({}), + { match: sourcesSimctl("shutdown", seedUdid) }, + { match: sourcesSimctl("delete", seedUdid) }, + { match: runtimeDelete, result: { code: 0, stderr: "", stdout: "" } }, + imagesListed([live], "Deleting"), + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]); + const clock = new FakeClock(); + const driver = await createDriver( + runner, + clock, + await assetStore([]), + await seededDatabase(), + ); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + const provision = driver.provision(spec).catch((error: unknown) => error); + await waitForCalls(runner, 2); + clock.advance(30_000); + await provision; + + const removal = driver + .removeComponent({ build: live.build, image: live.id }, { signal: signal() }) + .catch((caught: unknown) => caught); + await flushMicrotasks221(); + clock.advance(300_000); + const outcome = promiseState(removal); + for (let step = 0; step < 10 && outcome.state === "pending"; step += 1) { + await flushMicrotasks221(); + clock.advance(1_000); + } + + expect(await removal).toEqual({ sizeBytes: 1_000 }); + expect(runner.calls.map((call) => call.args)).toContainEqual(runtimeDelete.args); + }); + + it("reports residue naming Xcode's Settings -> Platforms when the build's download is still in the asset store", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]); + const filesystem = await assetStore([["26.4", "23E244"]]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toContain("26.4 (23E244)"); + expect(removal.residue).toContain(IOS_RUNTIME_ASSET_ROOT); + expect(removal.residue).toContain("Xcode's Settings -> Platforms"); + await expect(filesystem.exists(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`)).resolves.toBe( + true, + ); + }); + + it("states the measured size of the download in residue, 8.5 GiB for an asset of 8,917,036 KiB", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]); + // why: The iOS 18.6 download measured on a real machine. + const filesystem = new SizedFilesystem( + new Map([[`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024]]), + ); + await addAssets(filesystem, [["26.4", "23E244"]]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toContain("its download (8.5 GiB) is still in"); + }); + + it("states the same measured size in the doctor advisory as in the residue of the removal", async () => { + const filesystem = new SizedFilesystem( + new Map([[`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024]]), + ); + await addAssets(filesystem, [["26.4", "23E244"]]); + const remover = await createDriver( + new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]), + new FakeClock(), + filesystem, + ); + const residue = (await remover.removeComponent(receipt, { signal: signal() })).residue; + const doctor = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + const advisory = (await doctor.advisories()).find( + (candidate) => candidate.code === "runtime-cache-unreclaimable", + ); + + const sizeIn = (text: string | undefined) => /its download \(([^)]+)\)/.exec(text ?? "")?.[1]; + expect(sizeIn(residue)).toBe("8.5 GiB"); + expect(sizeIn(advisory?.message)).toBe(sizeIn(residue)); + }); + + it("states the total size of two downloads the removed build leaves, as downloads of one runtime", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]); + const filesystem = new SizedFilesystem( + new Map([ + [`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024], + [`${IOS_RUNTIME_ASSET_ROOT}/bundle-1.asset`, 8_916_744 * 1024], + ]), + ); + await addAssets(filesystem, [ + ["26.4", "23E244"], + ["26.4", "23E244"], + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toContain( + "iOS 26.4 (23E244) is no longer installed, but its downloads (17.0 GiB in all) are " + + "still in", + ); + expect(removal.residue).toContain("does not reclaim them"); + }); + + it("states no size in residue when the download's size cannot be read", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({}), + ]); + const filesystem = await assetStore([["26.4", "23E244"]]); + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset/AssetData`); + filesystem.defineFailure(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset/AssetData`, "EACCES"); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toContain("its download is still in"); + expect(removal.residue).not.toContain("GiB"); + }); + + it("removes the runtime when its only default-set devices were never used (#241)", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({ [target.runtime]: { unused: 11 } }), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({ [target.runtime]: { unused: 11 } }), + ]); + const driver = await createDriver(runner, new FakeClock(), await assetStore([])); + + await expect(driver.removeComponent(receipt, { signal: signal() })).resolves.toMatchObject({ + sizeBytes: 7_000_000_000, + }); + expect(runner.calls[2]?.args).toEqual(deleteInvocation.args); + }); + + it.each([ + [ + 3, + "3 never-used simulators in the default device set are now unavailable; Simlock does " + + "not delete simulators there -- `xcrun simctl delete unavailable` clears them", + ], + [ + 1, + "1 never-used simulator in the default device set is now unavailable; Simlock does " + + "not delete simulators there -- `xcrun simctl delete unavailable` clears them", + ], + ])( + "reports %s unused default-set devices as unavailable in residue, naming xcrun simctl delete unavailable, in the singular for one (#241)", + async (unused, residue) => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({ [target.runtime]: { unused } }), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({ [target.runtime]: { unused } }), + ]); + const driver = await createDriver(runner, new FakeClock(), await assetStore([])); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toBe(residue); + }, + ); + + it("carries both the unavailable devices and the download left in the asset store in residue (#241)", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({ [target.runtime]: { unused: 2 } }), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({ [target.runtime]: { unused: 2 } }), + ]); + const driver = await createDriver( + runner, + new FakeClock(), + await assetStore([["26.4", "23E244"]]), + ); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal.residue).toMatch( + /^2 never-used simulators .*`xcrun simctl delete unavailable` clears them\. iOS 26\.4 \(23E244\) is no longer installed, but its download .*Xcode's Settings -> Platforms to get the space back$/, + ); + }); + + it("reports no unavailable simulators when another installed build of the same runtime stays behind (#241)", async () => { + // why: Two builds of iOS 26.4 share one runtime identifier, so its simulators keep working + // on the build that stays. + const sibling = image("26.4", "23E200", { id: "IMG-26.4-23E200" }); + const runner = new ScriptedProcessRunner([ + imagesListed([target, sibling]), + defaultSetListed({ [target.runtime]: { unused: 3 } }), + deleted, + imagesListed([sibling]), + imagesListed([sibling]), + defaultSetListed({ [target.runtime]: { unused: 3 } }), + ]); + const driver = await createDriver(runner, new FakeClock(), await assetStore([])); + + const removal = await driver.removeComponent(receipt, { signal: signal() }); + + expect(removal).toEqual({ sizeBytes: 7_000_000_000 }); + }); + + it("runs no simctl command that writes to the default device set: its only unscoped call is the device list (#241)", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({ [target.runtime]: { unused: 11 } }), + deleted, + imagesListed([]), + imagesListed([]), + defaultSetListed({ [target.runtime]: { unused: 11 } }), + ]); + const driver = await createDriver(runner, new FakeClock(), await assetStore([])); + + await driver.removeComponent(receipt, { signal: signal() }); + + // why: Every call that names no `--set` addresses the default set; each must be the read. + const unscoped = runner.calls.filter( + (call) => call.command === "xcrun" && !call.args.includes("--set"), + ); + expect(unscoped.map((call) => call.args)).toEqual([ + defaultSetInvocation.args, + defaultSetInvocation.args, + ]); + expect(runner.calls).toHaveLength(6); + }); + + it("reports the same foreign-device count in listComponents and in removeComponent's refusal for the same default set (#241)", async () => { + const set = defaultSetListed({ [target.runtime]: { unused: 9, used: 2 } }); + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + set, + imagesListed([target]), + set, + ]); + const driver = await createDriver(runner); + + const [listed] = await driver.listComponents(); + const error = await driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught); + + expect(listed?.foreignDevices).toBe(2); + expect(error).toBeInstanceOf(ComponentInUseError); + expect(error).toMatchObject({ foreignDevices: listed?.foreignDevices }); + }); + + it("fails with simctl's own error when runtime delete exits non-zero", async () => { + // why: No listing after the failure: the scripted runner fails any call it was not given. + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + { + match: deleteInvocation, + result: { code: 1, stderr: "Unable to delete runtime: in use", stdout: "" }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.removeComponent(receipt, { signal: signal() })).rejects.toThrow( + "Unable to delete runtime: in use", + ); + expect(runner.calls).toHaveLength(3); + }); + + it("refuses with ComponentInUseError when one default-set device of the runtime has lastUsedAt, and deletes nothing", async () => { + // why: The scripted runner fails any call it was not given, so a delete would fail this test. + // Ten never-used devices beside the used one: only the used one counts. + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({ [target.runtime]: { unused: 10, used: 1 } }), + ]); + const driver = await createDriver(runner); + + const error = await driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ComponentInUseError); + expect(error).toMatchObject({ foreignDevices: 1 }); + expect(runner.calls).toHaveLength(2); + }); + + it("refuses with ComponentNotOwnedError when no listed image has the receipt, and deletes nothing", async () => { + const runner = new ScriptedProcessRunner([imagesListed([target]), defaultSetListed({})]); + const driver = await createDriver(runner); + + await expect( + driver.removeComponent({ build: "23E244", image: "IMG-OTHER" }, { signal: signal() }), + ).rejects.toBeInstanceOf(ComponentNotOwnedError); + expect(runner.calls).toHaveLength(2); + }); + + it("fails when the image is still listed after simctl answered", async () => { + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + deleted, + imagesListed([target]), + imagesListed([target]), + defaultSetListed({}), + ]); + const driver = await createDriver(runner); + + // why: The clock never moves, so a removal that waited would never settle. + const error = await settledValue( + driver.removeComponent(receipt, { signal: signal() }).catch((caught: unknown) => caught), + ); + + expect(error).toBeInstanceOf(DriverCrashError); + expect(error).toMatchObject({ message: "ios 26.4 is still installed after it was removed" }); + expect(runner.calls).toHaveLength(6); + }); + + it("answers the removal with its size and residue when simctl runtime list shows the image as Deleting after simctl runtime delete returns and drops it seconds later (#259)", async () => { + // why: As seen on Xcode 26.4.1: `simctl runtime delete` exits while `simctl runtime list` + // still lists the image, in state `Deleting`, for about eight seconds. The runner answers + // by the fake clock rather than by call order, so it does not dictate how a fix waits. + const clock = new FakeClock(); + const deletingForMs = 8_000; + const runner = new ClockedSimctl259(clock, deletingForMs, { + defaultSet: defaultSetListed({ [target.runtime]: { unused: 2 } }), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], "Deleting"), + }); + const driver = await createDriver(runner, clock, await assetStore([])); + + let settled = false; + const removal = driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught) + .finally(() => { + settled = true; + }); + // why: Lets time pass in one-second steps, well inside the five-minute removal budget. + for (let second = 0; second < 30 && !settled; second += 1) { + await flushMicrotasks221(); + clock.advance(1_000); + } + + expect(await settledValue(removal)).toEqual({ + residue: + "2 never-used simulators in the default device set are now unavailable; Simlock does " + + "not delete simulators there -- `xcrun simctl delete unavailable` clears them", + sizeBytes: 7_000_000_000, + }); + expect(runner.deletes).toBe(1); + }); + + it("fails with DriverCrashError five minutes after simctl runtime delete started, not after it answered, when simctl runtime list keeps showing the image as Deleting (#259)", async () => { + // why: The delete itself takes two minutes, so a budget restarted when it answers would end + // two minutes later than one measured from its start. + const clock = new FakeClock(); + const runner = new ClockedSimctl259( + clock, + Number.POSITIVE_INFINITY, + { + defaultSet: defaultSetListed({}), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], "Deleting"), + }, + // why: Half a second off the one-second polls, so the last pause must be cut short. + 120_500, + ); + const driver = await createDriver(runner, clock, await assetStore([])); + + let settledAt: number | undefined; + const removal = driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught) + .finally(() => { + settledAt = clock.now(); + }); + // why: Half-second steps land on the deadline itself, so the time it fails at is exact. + for (let step = 0; step < 1_200 && settledAt === undefined; step += 1) { + await flushMicrotasks221(); + clock.advance(500); + } + + const error = await settledValue(removal); + expect(error).toBeInstanceOf(DriverCrashError); + expect(error).toMatchObject({ + message: "ios 26.4 was still being deleted 300000ms after its removal started", + }); + expect(settledAt).toBe((runner.deleteStartedAt ?? Number.NaN) + 300_000); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("fails with DriverCrashError as soon as the signal fires while simctl runtime list shows the image as Deleting, and lists no more (#259)", async () => { + const clock = new FakeClock(); + const runner = new ClockedSimctl259(clock, Number.POSITIVE_INFINITY, { + defaultSet: defaultSetListed({}), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], "Deleting"), + }); + const driver = await createDriver(runner, clock, await assetStore([])); + const controller = new AbortController(); + + const removal = driver + .removeComponent(receipt, { signal: controller.signal }) + .catch((caught: unknown) => caught); + await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(1)); + await flushMicrotasks221(); + // why: The clock never moves: only the signal can end the wait. + controller.abort(); + + const error = await settledValue(removal); + expect(error).toBeInstanceOf(DriverCrashError); + expect(error).toMatchObject({ + message: "The removal of ios 26.4 was ended while it was still being deleted", + }); + expect(runner.listsAfterDelete).toBe(1); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("lists the images again every second while simctl runtime list shows the image as Deleting, and leaves no abort listener behind (#259)", async () => { + const clock = new FakeClock(); + const runner = new ClockedSimctl259(clock, 3_500, { + defaultSet: defaultSetListed({}), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], "Deleting"), + }); + const driver = await createDriver(runner, clock, await assetStore([])); + const controller = new AbortController(); + const added = vi.spyOn(controller.signal, "addEventListener"); + const removed = vi.spyOn(controller.signal, "removeEventListener"); + + const removal = driver.removeComponent(receipt, { signal: controller.signal }); + await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(1)); + for (const expected of [2, 3, 4]) { + await flushMicrotasks221(); + clock.advance(999); + await flushMicrotasks221(); + expect(runner.listsAfterDelete).toBe(expected - 1); + clock.advance(1); + await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(expected)); + } + // why: Gone at 3.5 s: the listing at 4 s ends the wait. + await flushMicrotasks221(); + clock.advance(1_000); + await expect(removal).resolves.toEqual({ sizeBytes: 7_000_000_000 }); + expect(added.mock.calls.length).toBeGreaterThan(1); + expect(removed.mock.calls.map(([, listener]) => listener)).toEqual( + expect.arrayContaining(added.mock.calls.map(([, listener]) => listener)), + ); + }); + + it.each(["deleting", "DELETING"])( + "waits while simctl runtime list shows the image in state %s, the state compared ignoring case (#259)", + async (state) => { + const clock = new FakeClock(); + const runner = new ClockedSimctl259(clock, 2_000, { + defaultSet: defaultSetListed({}), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], state), + }); + const driver = await createDriver(runner, clock, await assetStore([])); + + let settled = false; + const removal = driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught) + .finally(() => { + settled = true; + }); + for (let second = 0; second < 30 && !settled; second += 1) { + await flushMicrotasks221(); + clock.advance(1_000); + } + + expect(await settledValue(removal)).toEqual({ sizeBytes: 7_000_000_000 }); + }, + ); + + it("fails with DriverCrashError when the signal fires while simctl runtime list is answering that the image is Deleting, and lists no more (#259)", async () => { + const clock = new FakeClock(); + const controller = new AbortController(); + const runner = new ClockedSimctl259( + clock, + Number.POSITIVE_INFINITY, + { + defaultSet: defaultSetListed({}), + deleted, + gone: imagesListed([]), + ready: imagesListed([target]), + deleting: imagesListed([target], "Deleting"), + }, + 0, + () => { + controller.abort(); + }, + ); + const driver = await createDriver(runner, clock, await assetStore([])); + + // why: The clock never moves: only the signal, already fired when the pause starts, ends it. + const error = await settledValue( + driver + .removeComponent(receipt, { signal: controller.signal }) + .catch((caught: unknown) => caught), + ); + + expect(error).toBeInstanceOf(DriverCrashError); + expect(error).toMatchObject({ + message: "The removal of ios 26.4 was ended while it was still being deleted", + }); + expect(runner.listsAfterDelete).toBe(1); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("ends simctl runtime delete after five minutes, SIGTERM then SIGKILL, and answers only once it has exited", async () => { + // why: Ignores SIGTERM: only the SIGKILL ten seconds later ends it. + const stubborn = { hangs: true, ignoresSigterm: true, match: deleteInvocation }; + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + stubborn, + ]); + const driver = await createDriver(runner, clock); + let settled = false; + const removal = driver + .removeComponent(receipt, { signal: signal() }) + .catch((caught: unknown) => caught) + .finally(() => { + settled = true; + }); + await vi.waitFor(() => expect(runner.calls).toHaveLength(3)); + + clock.advance(5 * 60_000); + await flushMicrotasks221(); + expect(settled).toBe(false); + clock.advance(10_000); + + expect(await settledValue(removal)).toMatchObject({ + message: expect.stringMatching(/timed out after 300000ms/), + }); + }); + + it("ends simctl runtime delete when its signal fires, and starts none when it already has", async () => { + const abort = new AbortController(); + const runner = new ScriptedProcessRunner([ + imagesListed([target]), + defaultSetListed({}), + { hangs: true, match: deleteInvocation }, + ]); + const driver = await createDriver(runner); + const aborted = driver + .removeComponent(receipt, { signal: abort.signal }) + .catch((caught: unknown) => caught); + await vi.waitFor(() => expect(runner.calls).toHaveLength(3)); + abort.abort(); + expect(await settledValue(aborted)).toMatchObject({ + message: expect.stringMatching(/was ended before it finished/), + }); + + // why: The scripted runner fails any call it was not given, so a delete would fail this. + const lateRunner = new ScriptedProcessRunner([imagesListed([target]), defaultSetListed({})]); + const lateDriver = await createDriver(lateRunner); + await expect( + lateDriver.removeComponent(receipt, { signal: AbortSignal.abort() }), + ).rejects.toThrow(/was ended before it finished/); + expect(lateRunner.calls).toHaveLength(2); + }); + }); +}); + +type ScriptedStep259 = ConstructorParameters[0][number]; + +class ClockedSimctl259 extends ScriptedProcessRunner { + deletes = 0; + deleteStartedAt: number | undefined; + listsAfterDelete = 0; + #deletedAt: number | undefined; + + constructor( + private readonly clock: FakeClock, + private readonly deletingForMs: number, + private readonly steps: { + readonly ready: ScriptedStep259; + readonly deleting: ScriptedStep259; + readonly gone: ScriptedStep259; + readonly deleted: ScriptedStep259; + readonly defaultSet: ScriptedStep259; + }, + private readonly deleteTakesMs = 0, + private readonly onListAfterDelete: () => void = () => undefined, + ) { + super([]); + } + + override spawn(...call: Parameters) { + const [command, args] = call; + const argv = args.join(" "); + let step: ScriptedStep259; + if (argv.endsWith("runtime delete IMG-26.4-23E244")) { + this.deletes += 1; + this.deleteStartedAt = this.clock.now(); + this.clock.advance(this.deleteTakesMs); + this.#deletedAt = this.clock.now(); + step = this.steps.deleted; + } else if (argv.endsWith("runtime list -j")) { + if (this.#deletedAt !== undefined) { + this.listsAfterDelete += 1; + this.onListAfterDelete(); + } + step = + this.#deletedAt === undefined + ? this.steps.ready + : this.clock.now() - this.#deletedAt < this.deletingForMs + ? this.steps.deleting + : this.steps.gone; + } else if (argv === "simctl list -j devices") { + step = this.steps.defaultSet; + } else { + throw new Error(`Unexpected process invocation: ${command} ${argv}`); + } + const handle = new ScriptedProcessRunner([step]).spawn(...call); + this.calls.push({ args: [...args], command, options: call[2] ?? {} }); + return handle; + } +} diff --git a/src/drivers/ios/index-save-source-lifecycle.test.ts b/src/drivers/ios/index-save-source-lifecycle.test.ts new file mode 100644 index 000000000..1beb0023e --- /dev/null +++ b/src/drivers/ios/index-save-source-lifecycle.test.ts @@ -0,0 +1,575 @@ +import { afterEach, expect, describe, it } from "vitest"; +import { BootTimeoutError, DriverCrashError } from "../../core/index.js"; +import { MemoryFilesystem, ScriptedProcessRunner, FakeClock } from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { + deviceRoot, + driverData, + simctl, + plantManagedDevice, + overridesFolder, + overridesPath, + overridesPlist, + FaultyFilesystem, + instanceId, + overridesRoot, + flushMicrotasks221, + simctlArgs, + waitForCalls, +} from "./test-slim.js"; +import { + RecordingLogger, + UNSCOPED_LEGACY_CALLS, + createDriver, + deviceListResponse, + launchServicesFiles, + legacyUdid, + listDevicesInvocation, + listFixture, + listInvocation, + plantLaunchServices, + prepareFull, + scopedRunners, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + describe("saveSource", () => { + const cloneUdid = "00000000-0000-0000-0000-0000000000bb"; + const baselineName = "simlock-baseline-device-1"; + const sourcesSet = `${deviceRoot}/sources`; + const device = { + address: driverData.udid, + deviceId: driverData.udid, + driverData, + physical: false, + } as const; + const cloneInvocation = simctl("clone", driverData.udid, baselineName, sourcesSet); + + it("keeps a shut-down copy of the device in the sources set as a ready baseline source with its disabled.plist, and a failing clone keeps no row", async () => { + const database = await memorySourceDatabase(); + const rowsAtShutdown: unknown[] = []; + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(overridesFolder(driverData.udid)); + await filesystem.writeFileAtomic( + overridesPath(driverData.udid), + overridesPlist({ "a.b": true }), + ); + await filesystem.mkdirp(`${sourcesSet}/${cloneUdid}/data`); + await filesystem.writeFileAtomic(`${sourcesSet}/${cloneUdid}/data/state`, "12345"); + await plantLaunchServices(filesystem, `${sourcesSet}/${cloneUdid}`); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: simctl("shutdown", driverData.udid), + onSpawn: () => { + rowsAtShutdown.push( + ...database.prepare("SELECT kind, state, udid FROM ios_source_data").all(), + ); + }, + }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${cloneUdid}\n` } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + await expect(driver.saveSource(device)).resolves.toEqual({ + sizeBytes: 5, + sourceId: "device-1", + }); + + expect(rowsAtShutdown, "the source's row is making before the device shuts down").toEqual([ + { kind: "baseline", state: "making", udid: null }, + ]); + expect(runner.calls).toEqual([ + { ...listInvocation, options: { timeoutMs: 30_000 } }, + { ...simctl("shutdown", driverData.udid), options: { timeoutMs: 30_000 } }, + { ...cloneInvocation, options: { timeoutMs: 330_000 } }, + ]); + expect(await filesystem.readFile(overridesPath(cloneUdid))).toBe( + overridesPlist({ "a.b": true }), + ); + expect(await launchServicesFiles(filesystem, `${sourcesSet}/${cloneUdid}`)).toEqual([ + "com.apple.LaunchServices.unregisteredtypes-501.db", + ]); + await expect(driver.listSources()).resolves.toEqual([ + { + id: "device-1", + kind: "baseline", + model: "iPhone 16", + osVersion: "26.5", + sizeBytes: 5, + }, + ]); + + const failingRunner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: simctl("shutdown", driverData.udid) }, + { + match: cloneInvocation, + result: { code: 1, stderr: "No space left on device", stdout: "" }, + }, + ]); + const emptyDatabase = await memorySourceDatabase(); + const failing = await createDriver(failingRunner, new FakeClock(), filesystem, emptyDatabase); + await expect(failing.saveSource(device)).rejects.toMatchObject({ + message: expect.stringContaining("No space left on device"), + }); + expect(emptyDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); + + const stuckRunner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: simctl("shutdown", driverData.udid), + result: { code: 1, stderr: "Unable to shut down device", stdout: "" }, + }, + ]); + const stuckDatabase = await memorySourceDatabase(); + const stuck = await createDriver(stuckRunner, new FakeClock(), filesystem, stuckDatabase); + await expect(stuck.saveSource(device)).rejects.toMatchObject({ + message: expect.stringContaining("Unable to shut down device"), + }); + expect(stuckRunner.calls.map((call) => call.args[1])).not.toContain("clone"); + expect(stuckDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); + + const lateRunner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: simctl("shutdown", driverData.udid) }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${cloneUdid}\n` } }, + { match: sourcesSimctl("shutdown", cloneUdid) }, + { match: sourcesSimctl("delete", cloneUdid) }, + ]); + const lateDatabase = await memorySourceDatabase(); + lateDatabase.exec( + "CREATE TRIGGER refuse_update BEFORE UPDATE ON ios_source_data BEGIN SELECT RAISE(ABORT, 'source row locked'); END", + ); + const late = await createDriver(lateRunner, new FakeClock(), filesystem, lateDatabase); + await expect(late.saveSource(device)).rejects.toMatchObject({ + message: expect.stringContaining("source row locked"), + }); + expect( + lateRunner.calls.slice(3).map((call) => call.args.at(-2)), + "the clone made before the failure is shut down and deleted from the sources set", + ).toEqual(["shutdown", "delete"]); + expect(lateDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); + }); + }); + + it.each([ + { + name: "copies the seed's disabled.plist to the new simulator", + seedHas: true, + failing: undefined, + }, + { name: "copies nothing when the seed has none", seedHas: false, failing: undefined }, + { + name: "logs a warning and goes on when the copy fails", + seedHas: true, + failing: "copyFile" as const, + }, + ])("a provision $name", async ({ seedHas, failing }) => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + if (seedHas) { + await filesystem.mkdirp(overridesFolder(seedUdid)); + await filesystem.writeFileAtomic(overridesPath(seedUdid), overridesPlist({ "a.b": true })); + } + await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); + filesystem.failing = failing; + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + ]); + const logger = new RecordingLogger(); + const driver = await IosSimctlDriver.create({ + database: await seededDatabase(), + clock: new FakeClock(), + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + launchdOverridesRoot: overridesRoot, + logger, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + scopedRunners.push(runner); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + await expect(driver.provision(spec)).resolves.toMatchObject({ deviceId: driverData.udid }); + + expect( + await launchServicesFiles(filesystem, `${deviceRoot}/${driverData.udid}`), + "a copy of a seed loses the LaunchServices database that names the seed's folder", + ).toEqual(["com.apple.LaunchServices.unregisteredtypes-501.db"]); + const copied = failing === undefined && seedHas; + expect(await filesystem.exists(overridesPath(driverData.udid))).toBe(copied); + if (copied) { + expect(await filesystem.readFile(overridesPath(driverData.udid))).toBe( + overridesPlist({ "a.b": true }), + ); + } + expect(logger.warnings).toEqual( + failing === undefined + ? [] + : [ + `could not copy the launchd overrides of seed ${seedUdid} to ${driverData.udid}: disk full`, + ], + ); + }); + + it("keeps no copy of a slim spec's mode in driver data", async () => { + const runner = new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: listFixture }, + }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + { + match: listDevicesInvocation, + result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, + }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); + await driver.resolveSpec({ + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + await expect(driver.provision({ ...spec, mode: "slim" })).resolves.toEqual({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + }); + }); + + it("surfaces simctl failures with stderr", async () => { + const runner = new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: listFixture }, + }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 1, stderr: "Invalid runtime", stdout: "" }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), undefined, await seededDatabase()); + await driver.resolveSpec({ + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + await expect(driver.provision(spec)).rejects.toEqual( + expect.objectContaining({ + message: expect.stringContaining("Invalid runtime"), + name: "DriverCrashError", + }), + ); + }); + + it("boots then checks bootstatus and installer readiness with bounded commands", async () => { + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", driverData.udid) }, + { match: simctl("bootstatus", driverData.udid, "-b") }, + { match: simctl("listapps", driverData.udid) }, + ]); + const driver = await createDriver(runner); + + await expect( + driver.makeReady( + { address: driverData.udid, deviceId: driverData.udid, driverData }, + prepareFull, + ), + ).resolves.toEqual({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + // why: Slim mode is off, so this boot slimmed nothing: the device is full. + mode: "full", + }); + expect(runner.calls).toEqual([ + { ...simctl("boot", driverData.udid), options: { timeoutMs: 300_000 } }, + { ...simctl("bootstatus", driverData.udid, "-b"), options: { timeoutMs: 300_000 } }, + { ...simctl("listapps", driverData.udid), options: { timeoutMs: 30_000 } }, + ]); + }); + + it.each([false, true])( + "keeps makeReady pending until an installer probe succeeds after a failure (hung: %s)", + async (hangs) => { + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", driverData.udid) }, + { match: simctl("bootstatus", driverData.udid, "-b") }, + { + match: simctl("listapps", driverData.udid), + result: { code: 2, stderr: "installer starting", stdout: "" }, + hangs, + }, + { match: simctl("listapps", driverData.udid) }, + ]); + const driver = await createDriver(runner, clock); + const ready = driver.makeReady( + { address: driverData.udid, deviceId: driverData.udid, driverData }, + prepareFull, + ); + const state = promiseState(ready); + await flushMicrotasks221(); + expect(state.state).toBe("pending"); + expect(runner.calls.at(-1)?.args).toEqual(simctlArgs("listapps", driverData.udid)); + if (hangs) { + clock.advance(30_000); + await flushMicrotasks221(); + expect(state.state).toBe("pending"); + } + clock.advance(1_000); + await flushMicrotasks221(); + expect(state.state).toBe("fulfilled"); + expect((await ready).deviceId).toBe(driverData.udid); + }, + ); + + it.each([false, true])( + "shares the bootstatus deadline with a %s-hanging installer probe and shuts down on expiry", + async (hangs) => { + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", driverData.udid) }, + { + match: simctl("bootstatus", driverData.udid, "-b"), + onSpawn: () => clock.advance(299_500), + }, + { + match: simctl("listapps", driverData.udid), + hangs, + result: { code: 2, stderr: "installer starting", stdout: "" }, + }, + { match: simctl("shutdown", driverData.udid) }, + ]); + const driver = await createDriver(runner, clock); + const ready = driver.makeReady( + { address: driverData.udid, deviceId: driverData.udid, driverData }, + prepareFull, + ); + const state = promiseState(ready); + await flushMicrotasks221(); + expect(state.state).toBe("pending"); + expect(runner.calls.at(-1)).toEqual({ + ...simctl("listapps", driverData.udid), + options: { timeoutMs: 500 }, + }); + clock.advance(499); + await flushMicrotasks221(); + expect(state.state).toBe("pending"); + clock.advance(1); + await flushMicrotasks221(); + expect(state.state).toBe("rejected"); + await expect(ready).rejects.toBeInstanceOf(BootTimeoutError); + expect(runner.calls.at(-1)?.args).toEqual(simctlArgs("shutdown", driverData.udid)); + }, + ); + + it.each(["bootstatus", "listapps"])( + "rejects readiness when a successful %s consumes the remaining budget", + async (slowCommand) => { + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", driverData.udid) }, + { + match: simctl("bootstatus", driverData.udid, "-b"), + onSpawn: () => { + if (slowCommand === "bootstatus") clock.advance(300_000); + }, + }, + ...(slowCommand === "listapps" + ? [{ match: simctl("listapps", driverData.udid), onSpawn: () => clock.advance(300_000) }] + : []), + { match: simctl("shutdown", driverData.udid) }, + ]); + const driver = await createDriver(runner, clock); + await expect( + driver.makeReady( + { address: driverData.udid, deviceId: driverData.udid, driverData }, + prepareFull, + ), + ).rejects.toBeInstanceOf(BootTimeoutError); + expect(runner.calls.map((call) => call.args)).toEqual([ + simctlArgs("boot", driverData.udid), + simctlArgs("bootstatus", driverData.udid, "-b"), + ...(slowCommand === "listapps" ? [simctlArgs("listapps", driverData.udid)] : []), + simctlArgs("shutdown", driverData.udid), + ]); + }, + ); + + it("times out bootstatus and issues a best-effort shutdown", async () => { + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", driverData.udid) }, + { hangs: true, match: simctl("bootstatus", driverData.udid, "-b") }, + { match: simctl("shutdown", driverData.udid) }, + ]); + const driver = await createDriver(runner, clock); + const ready = driver.makeReady( + { address: driverData.udid, deviceId: driverData.udid, driverData }, + prepareFull, + ); + // why: Handled up front so a failure below reports itself rather than surfacing as an + // unhandled rejection in whichever test happens to run next. + void ready.catch(() => undefined); + + await waitForCalls(runner, 2); + clock.advance(300_000); + + await expect(ready).rejects.toBeInstanceOf(BootTimeoutError); + expect(runner.calls.map((call) => call.args)).toEqual([ + simctlArgs("boot", driverData.udid), + simctlArgs("bootstatus", driverData.udid, "-b"), + simctlArgs("shutdown", driverData.udid), + ]); + }); + + it("throws virtual devices are not reclaimed for a simulator, from reclaim and reclaimStrategy", async () => { + const runner = new ScriptedProcessRunner([]); + const driver = await createDriver(runner); + const simulator = { address: driverData.udid, deviceId: driverData.udid, driverData }; + + await expect(driver.reclaim(simulator, { clean: "full" })).rejects.toThrow( + "virtual devices are not reclaimed", + ); + expect(() => driver.reclaimStrategy(simulator, { clean: "standard" })).toThrow( + "virtual devices are not reclaimed", + ); + expect(runner.calls).toEqual([]); + }); + + it("shuts down before deleting and uses benchmark estimates", async () => { + const runner = new ScriptedProcessRunner([ + { match: simctl("shutdown", driverData.udid) }, + { match: simctl("delete", driverData.udid) }, + ]); + const driver = await createDriver(runner); + + await driver.destroy({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + }); + + expect(runner.calls.map((call) => call.args)).toEqual([ + simctlArgs("shutdown", driverData.udid), + simctlArgs("delete", driverData.udid), + ]); + expect(driver.estimate({ operation: "provision" }, spec)).toBe(500); + expect(driver.estimate({ operation: "boot" }, spec)).toBe(60_000); + }); + + it.each([ + { name: "removal succeeds", method: "destroy" as const, failing: undefined, deleteCode: 0 }, + { + name: "rm throws, which is logged and does not fail the destroy", + method: "destroy" as const, + failing: "rm" as const, + deleteCode: 0, + }, + { + name: "simctl delete fails, so the folder is kept and the error thrown", + method: "destroy" as const, + failing: undefined, + deleteCode: 1, + }, + { + name: "destroyLegacy removes it too", + method: "destroyLegacy" as const, + failing: undefined, + deleteCode: 0, + }, + ])( + "removes the overrides folder of a destroyed simulator after simctl delete: $name", + async ({ method, failing, deleteCode }) => { + const udid = method === "destroy" ? driverData.udid : legacyUdid; + const args = (verb: string) => + method === "destroy" ? simctlArgs(verb, udid) : ["simctl", verb, udid]; + const runner = new ScriptedProcessRunner([ + { match: { args: args("shutdown"), command: "xcrun" } }, + { + match: { args: args("delete"), command: "xcrun" }, + result: { code: deleteCode, stderr: "no", stdout: "" }, + }, + ]); + const filesystem = new FaultyFilesystem(); + await filesystem.mkdirp(overridesFolder(udid)); + await filesystem.writeFileAtomic(overridesPath(udid), "x"); + await filesystem.mkdirp(overridesFolder("other-udid")); + filesystem.failing = failing; + const logger = new RecordingLogger(); + scopedRunners.push(runner); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + launchdOverridesRoot: overridesRoot, + logger, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + const device = { address: udid, deviceId: udid, driverData: { ...driverData, udid } }; + + const destroyed = driver[method](device); + + if (deleteCode === 0) { + await expect(destroyed).resolves.toBeUndefined(); + } else { + await expect(destroyed).rejects.toBeInstanceOf(DriverCrashError); + } + const removed = deleteCode === 0 && failing === undefined; + expect(await filesystem.exists(overridesFolder(udid))).toBe(!removed); + expect(await filesystem.exists(overridesFolder("other-udid"))).toBe(true); + expect(logger.warnings).toEqual( + failing === undefined + ? [] + : [`could not remove the launchd overrides folder of ${udid}: disk full`], + ); + }, + ); +}); diff --git a/src/drivers/ios/index-screen-tool-versions.test.ts b/src/drivers/ios/index-screen-tool-versions.test.ts new file mode 100644 index 000000000..0f4f00baf --- /dev/null +++ b/src/drivers/ios/index-screen-tool-versions.test.ts @@ -0,0 +1,170 @@ +import { afterEach, expect, describe, it } from "vitest"; +import { + MemoryFilesystem, + FakeBytePipeLauncher, + FakeClock, + ScriptedProcessRunner, +} from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { deviceRoot, instanceId } from "./test-slim.js"; +import { UNSCOPED_LEGACY_CALLS, createDriver, scopedRunners } from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver screen capture commands", () => { + const physicalDevice = { + address: "UDID-P", + deviceId: "UDID-P", + driverData: {}, + enrolledApps: [], + physical: true, + } as const; + const settings = { bitrateKbps: 1_500, fps: 30, maxDimension: 720 }; + + // why: the daemon hands the driver its packaged programs; `drivers.ios.deviceCaptureCommand` + // replaces only the physical iPhone's program, and an empty string counts as unset exactly as + // `screenCaptureCommand` does, so one mistaken `""` does not turn iPhone streams off. + it.each([ + ["the configured program", "/worker/custom-device-capture", "/worker/custom-device-capture"], + ["the daemon's program when unset", undefined, "/dist/bin/simlock-device-capture"], + ["the daemon's program when set empty", "", "/dist/bin/simlock-device-capture"], + ] as const)("a physical iPhone's stream starts %s", async (_claim, configured, expected) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/worker"); + await filesystem.mkdirp("/dist/bin"); + await filesystem.writeFileAtomic("/worker/custom-device-capture", ""); + await filesystem.writeFileAtomic("/dist/bin/simlock-device-capture", ""); + const launcher = new FakeBytePipeLauncher(); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: new FakeClock(), + driverConfig: configured === undefined ? {} : { deviceCaptureCommand: configured }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: new ScriptedProcessRunner([]), + screenCapture: { + baseEnvironment: { PATH: "/usr/bin" }, + command: "/dist/bin/simlock-screen-capture", + deviceCommand: "/dist/bin/simlock-device-capture", + launcher, + }, + simlockHome: "/home/.simlock", + }); + + const capture = driver.screenCapture; + if (capture === undefined) throw new Error("the driver offers no screen capture"); + await capture.open(physicalDevice, settings, new AbortController().signal); + expect(launcher.starts.map((start) => start.command)).toEqual([expected]); + }); +}); + +describe("IosSimctlDriver defaultModels", () => { + it("lists the built-in iOS preference list per class, newest first, with phone starting at iPhone 17", async () => { + const driver = await createDriver(new ScriptedProcessRunner([])); + + expect(driver.defaultModels).toEqual({ + phone: ["iPhone 17", "iPhone 16", "iPhone 15", "iPhone 14", "iPhone 13"], + tablet: [ + "iPad Pro 11-inch (M5)", + "iPad Pro 11-inch (M4)", + "iPad Air 11-inch (M3)", + "iPad (A16)", + "iPad (10th generation)", + ], + tv: ["Apple TV 4K (3rd generation)", "Apple TV"], + vision: ["Apple Vision Pro"], + watch: [ + "Apple Watch Series 11 (46mm)", + "Apple Watch Series 10 (46mm)", + "Apple Watch Series 9 (45mm)", + ], + }); + }); +}); + +describe("IosSimctlDriver toolVersions()", () => { + it("reports the Xcode version and build that xcodebuild -version prints", async () => { + const runner = new ScriptedProcessRunner([ + { + match: { args: ["-version"], command: "xcodebuild" }, + result: { code: 0, stderr: "", stdout: "Xcode 16.4\nBuild version 16F6\n" }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.toolVersions()).resolves.toEqual([ + { build: "16F6", name: "xcode", version: "16.4" }, + ]); + expect(runner.calls[0]?.options.timeoutMs).toBeGreaterThan(0); + }); + + it("reports the Xcode version without a build when xcodebuild prints none", async () => { + const runner = new ScriptedProcessRunner([ + { + match: { args: ["-version"], command: "xcodebuild" }, + result: { code: 0, stderr: "", stdout: "Xcode 16.4\n" }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.toolVersions()).resolves.toEqual([{ name: "xcode", version: "16.4" }]); + }); + + it("reports no Xcode entry when only the command line tools are installed", async () => { + const runner = new ScriptedProcessRunner([ + { + match: { args: ["-version"], command: "xcodebuild" }, + result: { + code: 1, + stderr: + "xcode-select: error: tool 'xcodebuild' requires Xcode, but active developer directory '/Library/Developer/CommandLineTools' is a command line tools instance", + stdout: "", + }, + }, + ]); + const driver = await createDriver(runner); + + await expect(driver.toolVersions()).resolves.toEqual([]); + }); + + it.each([ + // why: Output that would parse, so only the exit code can be what turns it away. + ["exits non-zero", { code: 1, stderr: "error", stdout: "Xcode 16.4\nBuild version 16F6\n" }], + ["is killed by its timeout", { code: null, stderr: "", stdout: "" }], + ["prints no Xcode version", { code: 0, stderr: "", stdout: "something else\n" }], + ])("rejects when xcodebuild %s", async (_case, result) => { + const runner = new ScriptedProcessRunner([ + { match: { args: ["-version"], command: "xcodebuild" }, result }, + ]); + const driver = await createDriver(runner); + + await expect(driver.toolVersions()).rejects.toThrow(/xcodebuild/); + }); + + it("rejects when xcodebuild cannot be started", async () => { + // why: No scripted invocation: the runner refuses to start it, as a spawn failure would. + const driver = await createDriver(new ScriptedProcessRunner([])); + + await expect(driver.toolVersions()).rejects.toThrow(/xcodebuild/); + }); +}); diff --git a/src/drivers/ios/index-seed-provision.test.ts b/src/drivers/ios/index-seed-provision.test.ts new file mode 100644 index 000000000..b1443b98e --- /dev/null +++ b/src/drivers/ios/index-seed-provision.test.ts @@ -0,0 +1,460 @@ +import { afterEach, expect, it, vi, describe } from "vitest"; +import { DriverCrashError } from "../../core/index.js"; +import { + ScriptedProcessRunner, + MemoryFilesystem, + FakeClock, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ProcessHandle, +} from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { + deviceRoot, + driverData, + plantManagedDevice, + waitForCalls, + simctl, + instanceId, +} from "./test-slim.js"; +import { + UNSCOPED_LEGACY_CALLS, + createDriver, + listFixture, + listInvocation, + scopedRunners, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + it("a provision that makes a new iOS seed boots the seed within ios.full.bootTimeoutMs", async () => { + for (const { full, waitsMs } of [ + { full: undefined, waitsMs: 300_000 }, + { full: 400_000, waitsMs: 400_000 }, + ]) { + const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + { match: sourcesSimctl("boot", seedUdid) }, + { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, + { match: sourcesSimctl("listapps", seedUdid) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` } }, + ]); + const filesystem = new MemoryFilesystem(); + const driver = await createDriver( + runner, + new FakeClock(), + filesystem, + undefined, + undefined, + full, + ); + await plantManagedDevice(filesystem); + + await driver.provision(spec); + + expect( + runner.calls.find((call) => call.args.includes("bootstatus"))?.options, + `key ${String(full)}`, + ).toEqual({ timeoutMs: waitsMs }); + } + }); + + it("provisions a copy of the kind's seed, making the seed first when there is none", async () => { + const copyUdid = "00000000-0000-0000-0000-000000000002"; + const database = await memorySourceDatabase(); + const rowsWhenCreated: unknown[] = []; + const seedCreate = sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ); + const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: seedCreate, + onSpawn: () => { + rowsWhenCreated.push( + ...database.prepare("SELECT state, udid FROM ios_source_data").all(), + ); + }, + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + { match: sourcesSimctl("boot", seedUdid) }, + { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, + { match: sourcesSimctl("listapps", seedUdid) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` } }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, + ]); + const filesystem = new ModeRecordingFilesystem(); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); + + await expect(driver.provision(spec)).resolves.toEqual({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + }); + expect( + rowsWhenCreated, + "the seed's row is making, with no simulator yet, when create runs", + ).toEqual([{ state: "making", udid: null }]); + expect( + filesystem.mkdirpModes.get(`${deviceRoot}/sources`), + "under deviceSet simlock the sources set is asked for private to the user", + ).toBe(0o700); + expect(runner.calls).toEqual([ + { ...listInvocation, options: { timeoutMs: 30_000 } }, + { ...seedCreate, options: { timeoutMs: 330_000 } }, + { ...sourcesSimctl("boot", seedUdid), options: { timeoutMs: 300_000 } }, + { ...sourcesSimctl("bootstatus", seedUdid, "-b"), options: { timeoutMs: 300_000 } }, + { ...sourcesSimctl("listapps", seedUdid), options: { timeoutMs: 30_000 } }, + { ...sourcesSimctl("shutdown", seedUdid), options: { timeoutMs: 30_000 } }, + { ...cloneInvocation, options: { timeoutMs: 330_000 } }, + ]); + + await expect(driver.provision(spec)).resolves.toMatchObject({ deviceId: copyUdid }); + expect(runner.calls.slice(7)).toEqual([ + { ...cloneInvocation, options: { timeoutMs: 330_000 } }, + ]); + }); + + it("a clone that fails makes the provision throw and keeps the seed and its row", async () => { + const database = await seededDatabase(); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 1, stderr: "No space left on device", stdout: "" }, + }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + const error = await driver.provision(spec).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DriverCrashError); + expect((error as Error).message).toContain("No space left on device"); + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "clone"]); + expect(database.prepare("SELECT udid, state FROM ios_source_data").all()).toEqual([ + { state: "ready", udid: seedUdid }, + ]); + }); + + it("two provisions at once make one seed, and the clones of one seed run one at a time", async () => { + const copyUdid = "00000000-0000-0000-0000-000000000002"; + const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + { match: sourcesSimctl("boot", seedUdid) }, + { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, + { match: sourcesSimctl("listapps", seedUdid) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { hangs: true, match: cloneInvocation }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, + ]); + const filesystem = new MemoryFilesystem(); + const clock = new FakeClock(); + const driver = await createDriver(runner, clock, filesystem); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); + const clones = (): number => runner.calls.filter((call) => call.args[3] === "clone").length; + + const first = driver.provision(spec); + const second = driver.provision(spec); + const firstSettled = first.catch((caught: unknown) => caught); + await waitForCalls(runner, 8); + + expect(runner.calls.filter((call) => call.args[3] === "create")).toHaveLength(1); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + expect(clones()).toBe(1); + + clock.advance(30_000); + + expect(await firstSettled).toBeInstanceOf(DriverCrashError); + await expect(second).resolves.toMatchObject({ deviceId: copyUdid }); + expect(clones()).toBe(2); + }); + + it.each([ + { + call: "a provision's clone", + act: (driver: IosSimctlDriver) => driver.provision(spec), + seeded: true, + steps: [ + { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, + ], + }, + { + call: "a seed's create", + act: (driver: IosSimctlDriver) => driver.provision(spec), + seeded: false, + steps: [ + { + hangs: true, + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + }, + ], + }, + { + call: "a baseline's clone", + act: (driver: IosSimctlDriver) => + driver.saveSource({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + physical: false, + }), + seeded: false, + steps: [ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: simctl("shutdown", driverData.udid) }, + { + hangs: true, + match: simctl( + "clone", + driverData.udid, + "simlock-baseline-device-1", + `${deviceRoot}/sources`, + ), + }, + ], + }, + ])("$call waits ios.simulator.createTimeoutMs", async ({ act, seeded, steps }) => { + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ...steps, + ]); + scopedRunners.push(runner); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const clock = new FakeClock(); + const config = { simulator: { audio: true, createTimeoutMs: 120_000, headless: true } }; + const driver = await IosSimctlDriver.create({ + clock, + config, + database: seeded ? await seededDatabase() : await memorySourceDatabase(), + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + const outcome = act(driver); + const settled = outcome.catch((error: unknown) => error); + const state = promiseState(outcome); + await waitForCalls(runner, steps.length + 1); + clock.advance(119_999); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + + expect(state.state).toBe("pending"); + clock.advance(1); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + expect(state.state).toBe("rejected"); + expect(await settled).toMatchObject({ + message: expect.stringMatching(/timed out after 120000ms$/), + }); + }); + + const coreSimulatorRoot = "/CoreSim"; + + it.each([ + { + call: "a provision's clone under deviceSet default", + act: (driver: IosSimctlDriver) => driver.provision(spec), + deleteArgs: (udid: string) => ["simctl", "delete", udid], + hung: [ + "simctl", + "--set", + "/home/.simlock/devices/ios-sources", + "clone", + seedUdid, + driverData.name, + `${coreSimulatorRoot}/Devices`, + ], + isDefault: true, + seeded: true, + steps: [], + }, + { + call: "a seed's create", + act: (driver: IosSimctlDriver) => driver.provision(spec), + deleteArgs: (udid: string) => sourcesSimctl("delete", udid).args, + hung: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ).args, + isDefault: false, + seeded: false, + steps: [], + }, + { + call: "a baseline's clone", + act: (driver: IosSimctlDriver) => + driver.saveSource({ + address: driverData.udid, + deviceId: driverData.udid, + driverData, + physical: false, + }), + deleteArgs: (udid: string) => sourcesSimctl("delete", udid).args, + hung: simctl("clone", driverData.udid, "simlock-baseline-device-1", `${deviceRoot}/sources`) + .args, + isDefault: false, + seeded: false, + steps: [ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: simctl("shutdown", driverData.udid) }, + ], + }, + ])( + "a create or clone that answers after its limit deletes the simulator it names in its own set ($call)", + async ({ act, deleteArgs, hung, isDefault, seeded, steps }) => { + const devicesDir = `${coreSimulatorRoot}/Devices`; + const lateUdid = "00000000-0000-0000-0000-0000000000c1"; + const list = isDefault + ? { args: ["simctl", "list", "-j", "devicetypes", "runtimes"], command: "xcrun" } + : listInvocation; + const scripted = new ScriptedProcessRunner([ + { match: list, result: { code: 0, stderr: "", stdout: listFixture } }, + ...steps, + { match: { args: deleteArgs(lateUdid), command: "xcrun" } }, + ]); + if (!isDefault) scopedRunners.push(scripted); + let answer: ((result: ProcessResult) => void) | undefined; + const runner: ProcessRunner = { + run: scripted.run.bind(scripted), + spawnStreaming: scripted.spawnStreaming.bind(scripted), + spawn( + command: string, + args: readonly string[], + options?: ProcessRunOptions, + ): ProcessHandle { + if (args.join(" ") !== hung.join(" ")) { + return scripted.spawn(command, args, options); + } + scripted.calls.push({ args: [...args], command, options: options ?? {} }); + // why: `simctl clone` asks CoreSimulator's service for the copy; a signal ends only the + // `simctl` process, so the copy still finishes, and it prints the UDID only if it is alive. + const result = new Promise((resolve) => { + answer = resolve; + }); + const empty = (async function* () {})(); + return { + kill() { + answer?.({ code: null, stderr: "", stdout: "" }); + }, + pid: 99, + stderr: empty, + stdout: empty, + unref() {}, + wait: () => result, + }; + }, + }; + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const clock = new FakeClock(); + const driver = await IosSimctlDriver.create({ + database: seeded ? await seededDatabase() : await memorySourceDatabase(), + clock, + ...(isDefault ? { coreSimulatorRoot } : {}), + driverConfig: isDefault ? { deviceSet: "default" } : { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + const call = act(driver).catch((error: unknown) => error); + await vi.waitFor(() => expect(answer).toBeDefined()); + clock.advance(30_000); + const failure = await call; + await filesystem.mkdirp(`${devicesDir}/${lateUdid}/data`); + answer?.({ code: 0, stderr: "", stdout: `${lateUdid}\n` }); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + + expect(failure).toBeInstanceOf(DriverCrashError); + expect((failure as Error).message).toMatch(/timed out after 30000ms$/); + expect( + scripted.calls.filter((entry) => entry.args.includes("delete")).map((entry) => entry.args), + ).toEqual([deleteArgs(lateUdid)]); + }, + ); +}); + +/** why: Records the mode each `mkdirp` was asked for, because `MemoryFilesystem` creates every directory owner-only by default and so cannot show a caller that left the mode out. */ +class ModeRecordingFilesystem extends MemoryFilesystem { + readonly mkdirpModes = new Map(); + + override async mkdirp( + path: string, + options?: Parameters[1], + ): Promise { + this.mkdirpModes.set(path, options?.mode); + await super.mkdirp(path, options); + } +} diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts new file mode 100644 index 000000000..f640a09b0 --- /dev/null +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -0,0 +1,644 @@ +import { afterEach, expect, it, describe } from "vitest"; +import { DriverCrashError, BootTimeoutError } from "../../core/index.js"; +import { ScriptedProcessRunner, FakeClock, MemoryFilesystem } from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { plantSource, memorySourceDatabase } from "./testing.js"; +import { + deviceRoot, + driverData, + waitForCalls, + FaultyFilesystem, + plantManagedDevice, + simctl, +} from "./test-slim.js"; +import { + RecordingLogger, + UNSCOPED_LEGACY_CALLS, + createDriver, + launchServicesFiles, + listFixture, + listInvocation, + plantLaunchServices, + scopedRunners, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +interface Catalog { + devicetypes: { identifier: string }[]; + runtimes: { buildversion?: string; identifier: string }[]; +} + +describe("IosSimctlDriver", () => { + it("a create command still silent after the second limit is killed and logged with the name it was asked to create", async () => { + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, + ]); + const logger = new RecordingLogger(); + const clock = new FakeClock(); + const driver = await createDriver(runner, clock, undefined, await seededDatabase(), logger); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + const provision = driver.provision(spec).catch((error: unknown) => error); + await waitForCalls(runner, 2); + const clone = promiseState(runner.handles[1]!.wait()); + clock.advance(30_000); + await provision; + clock.advance(299_999); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + expect(clone.state).toBe("pending"); + expect(logger.warnings).toEqual([]); + + clock.advance(1); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + + expect(clone.state).toBe("fulfilled"); + expect(logger.warnings).toEqual([ + `simctl clone ${seedUdid} ${driverData.name} ${deviceRoot} gave no simulator after its limit`, + ]); + }); + + it("a seed is not deleted, even by a deletion queued during the clone, while a clone that answered late is still running", async () => { + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { match: sourcesSimctl("delete", seedUdid) }, + ]); + const clock = new FakeClock(); + const driver = await createDriver(runner, clock, undefined, await seededDatabase()); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + const [listed] = await driver.listSources(); + + const provision = driver.provision(spec).catch((error: unknown) => error); + await waitForCalls(runner, 3); + const destroyed = driver.destroySource(listed?.id ?? ""); + const destroy = promiseState(destroyed); + clock.advance(30_000); + await provision; + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + + expect(destroy.state).toBe("pending"); + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list", "clone"]); + + clock.advance(300_000); + await destroyed; + + expect(runner.calls.map((call) => call.args[3])).toEqual([ + "list", + "list", + "clone", + "shutdown", + "delete", + ]); + }); + + it("provision deletes the clone and throws when the LaunchServices database cannot be dropped", async () => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); + filesystem.failing = "rm"; + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + { match: simctl("delete", driverData.udid) }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + await expect(driver.provision(spec)).rejects.toThrow("disk full"); + + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "clone", "delete"]); + }); + + it.each([ + { name: "a named source is copied instead of the seed", id: "planted-named", cloned: true }, + { name: "an unknown source is refused", id: "nope", cloned: false }, + { name: "a stale source is refused", id: "planted-old", cloned: false }, + ])("provision with a source: $name", async ({ id, cloned }) => { + const namedUdid = "00000000-0000-0000-0000-0000000000bb"; + const oldUdid = "00000000-0000-0000-0000-0000000000cc"; + const database = await seededDatabase(); + const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; + const planted = plantSource(database, { ...kind, kind: "baseline", udid: namedUdid }); + database + .prepare("UPDATE ios_source_data SET source_id = 'planted-named' WHERE udid = ?") + .run(namedUdid); + plantSource(planted, { ...kind, kind: "baseline", runtimeBuild: "23F77", udid: oldUdid }); + database + .prepare("UPDATE ios_source_data SET source_id = 'planted-old' WHERE udid = ?") + .run(oldUdid); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); + const catalog = JSON.parse(listFixture) as { + runtimes: { buildversion?: string; identifier: string }[]; + }; + for (const runtime of catalog.runtimes) { + if (runtime.identifier === driverData.runtimeId) runtime.buildversion = "23F79"; + } + const listed = JSON.stringify(catalog); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listed } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listed } }, + { + match: sourcesSimctl("clone", namedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + const provisioned = driver.provision(spec, { source: id }); + + if (cloned) { + await expect(provisioned).resolves.toMatchObject({ deviceId: driverData.udid }); + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list", "clone"]); + expect( + await launchServicesFiles(filesystem, `${deviceRoot}/${driverData.udid}`), + "the copy of a source loses the LaunchServices database that names the source's folder", + ).toEqual(["com.apple.LaunchServices.unregisteredtypes-501.db"]); + } else { + await expect(provisioned).rejects.toBeInstanceOf(DriverCrashError); + expect(runner.calls.map((call) => call.args[3])).not.toContain("clone"); + } + }); + + it.each([ + { name: "300 s old is kept", now: 300_000, destroyed: false }, + { name: "300.001 s old is destroyed", now: 300_001, destroyed: true }, + ])( + "the next seed make of a kind destroys this process's own making row of that kind older than 300 s: a row $name", + async ({ now, destroyed }) => { + const stuckUdid = "00000000-0000-0000-0000-0000000000dd"; + const database = plantSource(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + maker: "device-1", + runtimeId: driverData.runtimeId, + state: "making", + udid: stuckUdid, + }); + const copyUdid = "00000000-0000-0000-0000-000000000002"; + const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ...(destroyed + ? [ + { match: sourcesSimctl("shutdown", stuckUdid) }, + { match: sourcesSimctl("delete", stuckUdid) }, + ] + : []), + { + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + { match: sourcesSimctl("boot", seedUdid) }, + { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, + { match: sourcesSimctl("listapps", seedUdid) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); + const driver = await createDriver(runner, new FakeClock(now), filesystem, database); + + await driver.provision(spec); + + expect( + runner.calls.filter((call) => call.args[4] === stuckUdid).map((call) => call.args[3]), + ).toEqual(destroyed ? ["shutdown", "delete"] : []); + expect( + database.prepare("SELECT udid FROM ios_source_data WHERE state = 'making'").all(), + ).toEqual(destroyed ? [] : [{ udid: stuckUdid }]); + }, + ); + + it.each([ + { + name: "simctl create fails", + message: "Invalid runtime", + script: [{ code: 1, op: "create" }], + tail: [], + }, + { + name: "the seed's boot fails", + message: `seed ${seedUdid} did not boot: `, + script: [{ code: 1, op: "bootstatus" }], + tail: ["shutdown", "delete"], + }, + { + name: "the seed's shutdown fails", + message: "Unable to shutdown", + script: [{ code: 1, op: "shutdown" }], + tail: ["shutdown", "delete"], + }, + ] as const)( + "a seed make that fails because $name leaves no seed and no row, and the provision throws a DriverCrashError, never a BootTimeoutError", + async ({ message, script, tail }) => { + const failing = script[0]; + const steps = ["create", "boot", "bootstatus", "listapps", "shutdown"]; + const upTo = steps.indexOf(failing.op); + const results = new Map([ + ["create", { code: 0, stderr: "", stdout: `${seedUdid}\n` }], + ]); + const database = await memorySourceDatabase(); + const entries = steps.slice(0, upTo + 1).map((op) => { + const match = sourcesSimctl( + ...(op === "create" + ? ["create", "simlock-seed-device-1", driverData.deviceTypeId, driverData.runtimeId] + : op === "bootstatus" + ? [op, seedUdid, "-b"] + : [op, seedUdid]), + ); + if (op !== failing.op) + return { match, ...(results.has(op) ? { result: results.get(op)! } : {}) }; + return "hangs" in failing + ? { hangs: true, match } + : { match, result: { code: failing.code, stderr: message, stdout: "" } }; + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ...entries, + ...tail.map((op) => ({ match: sourcesSimctl(op, seedUdid) })), + ]); + const clock = new FakeClock(); + const driver = await createDriver(runner, clock, undefined, database); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + const provisioned = driver.provision(spec); + void provisioned.catch(() => undefined); + if ("hangs" in failing) { + await waitForCalls(runner, entries.length + 1); + clock.advance(300_000); + } + const error = await provisioned.catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(DriverCrashError); + expect(error).not.toBeInstanceOf(BootTimeoutError); + expect((error as Error).message).toContain(message); + expect(runner.calls.map((call) => call.args[3])).toEqual([ + "list", + ...steps.slice(0, upTo + 1), + ...tail, + ]); + expect(runner.calls.slice(1).map((call) => call.args[2])).toEqual( + runner.calls.slice(1).map(() => `${deviceRoot}/sources`), + ); + expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ n: 0 }); + }, + ); + + it.each([ + { name: "bootstatus does not finish", hangs: "bootstatus" }, + { name: "the installer never answers listapps", hangs: "listapps" }, + ] as const)( + "a seed whose boot runs past ios.full.bootTimeoutMs because $name makes provision throw BootTimeoutError, as an Android seed's does, and leaves no seed and no row", + async ({ hangs }) => { + const bootSteps = ["boot", "bootstatus", "listapps"]; + const ran = bootSteps.slice(0, bootSteps.indexOf(hangs) + 1); + const database = await memorySourceDatabase(); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + ...ran.map((op) => { + const match = sourcesSimctl( + ...(op === "bootstatus" ? [op, seedUdid, "-b"] : [op, seedUdid]), + ); + return op === hangs ? { hangs: true, match } : { match }; + }), + ...["shutdown", "shutdown", "delete"].map((op) => ({ match: sourcesSimctl(op, seedUdid) })), + ]); + const clock = new FakeClock(); + const driver = await createDriver(runner, clock, undefined, database, undefined, 2_000); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + const provisioned = driver.provision(spec); + void provisioned.catch(() => undefined); + await waitForCalls(runner, ran.length + 2); + clock.advance(2_000); + const error = await provisioned.catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(BootTimeoutError); + expect((error as Error).message).toBe("Timed out waiting for device to boot: device-1"); + expect(runner.calls.map((call) => call.args[3])).toEqual([ + "list", + "create", + ...ran, + "shutdown", + "shutdown", + "delete", + ]); + expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ n: 0 }); + }, + ); + + it.each([ + { name: "its runtime changed build", seedBuild: "23F77", runtimeBuild: "23F79", remade: true }, + { name: "its runtime is unchanged", seedBuild: "23F79", runtimeBuild: "23F79", remade: false }, + { name: "it recorded no build", seedBuild: undefined, runtimeBuild: "23F79", remade: false }, + { + name: "its runtime lists no build now", + seedBuild: "23F77", + runtimeBuild: undefined, + remade: false, + }, + ])( + "makes the seed again before copying it only when it is stale: $name -> remade $remade", + async ({ seedBuild, runtimeBuild, remade }) => { + const staleUdid = "00000000-0000-0000-0000-0000000000bb"; + const catalog = JSON.parse(listFixture) as { + runtimes: { buildversion?: string; identifier: string }[]; + }; + for (const entry of catalog.runtimes) { + if (entry.identifier === driverData.runtimeId && runtimeBuild !== undefined) { + entry.buildversion = runtimeBuild; + } + } + const planted = staleUdid; + const cloneOf = (udid: string) => sourcesSimctl("clone", udid, driverData.name, deviceRoot); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: JSON.stringify(catalog) } }, + ...(remade + ? [ + { match: sourcesSimctl("shutdown", staleUdid) }, + { match: sourcesSimctl("delete", staleUdid) }, + { + match: sourcesSimctl( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, + }, + { match: sourcesSimctl("boot", seedUdid) }, + { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, + { match: sourcesSimctl("listapps", seedUdid) }, + { match: sourcesSimctl("shutdown", seedUdid) }, + ] + : []), + { + match: cloneOf(remade ? seedUdid : staleUdid), + result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, + }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const driver = await createDriver( + runner, + new FakeClock(), + filesystem, + await seededDatabase({ + udid: planted, + ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), + }), + ); + await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); + + await driver.provision(spec); + + expect(runner.calls.map((call) => call.args[3]).filter((verb) => verb !== "list")).toEqual( + remade + ? ["shutdown", "delete", "create", "boot", "bootstatus", "listapps", "shutdown", "clone"] + : ["clone"], + ); + }, + ); + + it.each([ + { name: "healthy", edit: () => undefined, reason: undefined }, + { + name: "its runtime is gone", + edit: (catalog: Catalog) => { + catalog.runtimes = catalog.runtimes.filter( + (entry) => entry.identifier !== driverData.runtimeId, + ); + }, + reason: `runtime ${driverData.runtimeId} is not installed`, + }, + { + name: "its runtime changed build", + edit: (catalog: Catalog) => { + for (const entry of catalog.runtimes) { + if (entry.identifier === driverData.runtimeId) entry.buildversion = "23F79"; + } + }, + reason: `runtime ${driverData.runtimeId} changed from build 23F77 to 23F79`, + }, + { + name: "its device type is gone", + edit: (catalog: Catalog) => { + catalog.devicetypes = catalog.devicetypes.filter( + (entry) => entry.identifier !== driverData.deviceTypeId, + ); + }, + reason: `device type ${driverData.deviceTypeId} is not available`, + }, + ])( + "lists a ready seed as a source with its model, version and size, and its staleReason when $name, which clears once it is back", + async ({ edit, reason }) => { + const catalogOf = (change: (catalog: Catalog) => void) => { + const catalog = JSON.parse(listFixture) as Catalog; + for (const entry of catalog.runtimes) { + if (entry.identifier === driverData.runtimeId) entry.buildversion = "23F77"; + } + change(catalog); + return JSON.stringify(catalog); + }; + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: catalogOf(edit) } }, + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: catalogOf(() => undefined) }, + }, + ]); + const database = plantSource(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + runtimeBuild: "23F77", + runtimeId: driverData.runtimeId, + sizeBytes: 1_073_741_824, + udid: seedUdid, + }); + const driver = await createDriver(runner, new FakeClock(), undefined, database); + + expect(await driver.listSources()).toMatchObject([ + { + kind: "seed", + model: "iPhone 16", + osVersion: "26.5", + sizeBytes: 1_073_741_824, + ...(reason === undefined ? {} : { staleReason: reason }), + }, + ]); + expect((await driver.listSources())[0]).not.toHaveProperty("staleReason"); + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list"]); + }, + ); + + it("the first listSources of a process deletes another process's making rows and their simulators, keeps a row whose delete failed, and does not sweep again", async () => { + const [gone, stuck, ready] = [1, 2, 3].map( + (n) => `00000000-0000-0000-0000-0000000001${String(n)}0`, + ) as [string, string, string]; + const database = await memorySourceDatabase(); + const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; + plantSource(database, { ...kind, maker: "earlier-process", state: "making", udid: gone }); + plantSource(database, { ...kind, maker: "earlier-process", state: "making" }); + plantSource(database, { ...kind, maker: "earlier-process", state: "making", udid: stuck }); + plantSource(database, { ...kind, udid: ready }); + const runner = new ScriptedProcessRunner([ + { match: sourcesSimctl("shutdown", gone) }, + { match: sourcesSimctl("delete", gone) }, + { match: sourcesSimctl("shutdown", stuck) }, + { match: sourcesSimctl("delete", stuck), result: { code: 1, stderr: "busy", stdout: "" } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ]); + const logger = new RecordingLogger(); + const driver = await createDriver(runner, new FakeClock(), undefined, database, logger); + + expect(await driver.listSources()).toHaveLength(1); + await driver.listSources(); + + expect(runner.calls.map((call) => [call.args[3], call.args[4]])).toEqual([ + ["shutdown", gone], + ["delete", gone], + ["shutdown", stuck], + ["delete", stuck], + ["list", "-j"], + ["list", "-j"], + ]); + expect(logger.warnings).toEqual([ + expect.stringContaining(`could not remove leftover source ${stuck}:`), + ]); + expect(database.prepare("SELECT udid, state FROM ios_source_data ORDER BY udid").all()).toEqual( + [ + { state: "making", udid: stuck }, + { state: "ready", udid: ready }, + ], + ); + }); + + it("listSources reports a simulator in the sources set that no row names as an orphan source, and destroySource deletes it", async () => { + const orphanUdid = "00000000-0000-0000-0000-0000000000ee"; + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(`${deviceRoot}/sources/${seedUdid}`); + await filesystem.mkdirp(`${deviceRoot}/sources/${orphanUdid}`); + const database = await seededDatabase(); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: sourcesSimctl("shutdown", orphanUdid) }, + { match: sourcesSimctl("delete", orphanUdid) }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + const sources = await driver.listSources(); + + expect(sources.filter((source) => source.orphan !== true)).toHaveLength(1); + expect(sources.filter((source) => source.orphan === true)).toEqual([ + { id: orphanUdid, kind: "seed", model: "unknown", orphan: true, osVersion: "unknown" }, + ]); + await driver.destroySource(orphanUdid); + expect(runner.calls.slice(1).map((call) => [call.args[3], call.args[4]])).toEqual([ + ["shutdown", orphanUdid], + ["delete", orphanUdid], + ]); + await expect( + driver.destroySource("00000000-0000-0000-0000-0000000000f0"), + ).rejects.toBeInstanceOf(DriverCrashError); + }); + + it.each([ + { + name: "deletes its simulator in the sources set and its row", + deleteCode: 0, + stderr: "", + rows: 0, + }, + { + name: "keeps the row and throws when the delete fails", + deleteCode: 1, + stderr: "busy", + rows: 1, + }, + { + name: "drops the row of a seed simctl no longer knows (Invalid device)", + deleteCode: 148, + stderr: `Invalid device: ${seedUdid}\n`, + rows: 0, + }, + ])("destroySource $name", async ({ deleteCode, stderr, rows }) => { + const database = plantSource(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + runtimeId: driverData.runtimeId, + udid: seedUdid, + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: sourcesSimctl("shutdown", seedUdid) }, + { + match: sourcesSimctl("delete", seedUdid), + result: { code: deleteCode, stderr, stdout: "" }, + }, + ]); + const driver = await createDriver(runner, new FakeClock(), undefined, database); + const [listed] = await driver.listSources(); + + const destroyed = driver.destroySource(listed?.id ?? ""); + + if (rows === 0) await expect(destroyed).resolves.toBeUndefined(); + else await expect(destroyed).rejects.toBeInstanceOf(DriverCrashError); + expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "shutdown", "delete"]); + expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ + n: rows, + }); + }); + + it("destroySource refuses an id that names no source, without running simctl", async () => { + const runner = new ScriptedProcessRunner([]); + const driver = await createDriver(runner); + + await expect(driver.destroySource("nope")).rejects.toBeInstanceOf(DriverCrashError); + expect(runner.calls).toEqual([]); + }); +}); diff --git a/src/drivers/ios/index-simulator-audio.test.ts b/src/drivers/ios/index-simulator-audio.test.ts new file mode 100644 index 000000000..8e8f70f00 --- /dev/null +++ b/src/drivers/ios/index-simulator-audio.test.ts @@ -0,0 +1,167 @@ +import { afterEach, expect, describe, it } from "vitest"; +import { + type ProcessResult, + ScriptedProcessRunner, + FakeClock, + MemoryFilesystem, +} from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { type SilenceFailedFact, IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { + deviceRoot, + driverData, + simctlArgs, + simctl, + instanceId, + overridesRoot, + waitForCalls, + flushMicrotasks221, +} from "./test-slim.js"; +import { UNSCOPED_LEGACY_CALLS, scopedRunners, wantedReadback } from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver simulator audio", () => { + const volumeState = "com.apple.springboard.volumestate"; + const udid = driverData.udid; + const slimData = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-5" }; + + const silence = [ + simctlArgs("spawn", udid, "notifyutil", "-s", volumeState, "0"), + simctlArgs("spawn", udid, "notifyutil", "-p", volumeState), + ]; + const bootCalls = [ + simctlArgs("boot", udid), + simctlArgs("bootstatus", udid, "-b"), + simctlArgs("listapps", udid), + ]; + const readbackCall = simctlArgs("spawn", udid, "launchctl", "print-disabled", "system"); + + it.each([ + { + name: "prepare", + options: { mode: "full", purpose: "prepare" } as const, + expected: [...bootCalls, ...silence], + }, + { + name: "recover", + options: { mode: "full", purpose: "recover" } as const, + expected: [...bootCalls, ...silence], + }, + { + name: "a slim prepare, silenced before the readback", + data: slimData, + options: { mode: "slim", purpose: "prepare" } as const, + expected: [...bootCalls, ...silence, readbackCall], + }, + { + name: "the setting true", + audio: true, + options: { mode: "full", purpose: "prepare" } as const, + expected: bootCalls, + }, + { + name: "a volume command that fails", + volume: { code: 1, stderr: "boom", stdout: "" }, + options: { mode: "full", purpose: "prepare" } as const, + expected: [...bootCalls, silence[0]], + failure: "boom", + }, + { + name: "a volume command that times out", + volume: "hangs" as const, + options: { mode: "full", purpose: "prepare" } as const, + expected: [...bootCalls, silence[0]], + failure: "timed out", + }, + ])( + "sets a simulator's volume to 0 before handing it on when ios.simulator.audio is false: $name", + async (c) => { + const data = c.data ?? driverData; + const silenceCall = (index: number, result?: ProcessResult) => ({ + match: { args: silence[index]!, command: "xcrun" }, + ...(result === undefined ? {} : { result }), + }); + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", udid) }, + { match: simctl("bootstatus", udid, "-b") }, + { match: simctl("listapps", udid) }, + ...(c.audio === true + ? [] + : c.volume === "hangs" + ? [{ ...silenceCall(0), hangs: true }] + : c.volume !== undefined + ? [silenceCall(0, c.volume)] + : [silenceCall(0), silenceCall(1)]), + ...(c.options.mode === "slim" + ? [ + { + match: simctl("spawn", udid, "launchctl", "print-disabled", "system"), + result: { code: 0, stderr: "", stdout: wantedReadback() }, + }, + ] + : []), + ]); + scopedRunners.push(runner); + const clock = new FakeClock(); + const failures: SilenceFailedFact[] = []; + const driver = await IosSimctlDriver.create({ + clock, + database: await memorySourceDatabase(), + driverConfig: { deviceRoot }, + filesystem: new MemoryFilesystem(), + idGenerator: { generate: () => "device-1" }, + instanceId, + launchdOverridesRoot: overridesRoot, + onSilenceFailed: (fact) => failures.push(fact), + processRunner: runner, + simlockHome: "/home/.simlock", + config: { + simulator: { audio: c.audio ?? false, createTimeoutMs: 30_000, headless: true }, + slim: { bootTimeoutMs: 300_000, categories: ["widgets"] }, + }, + }); + + const ready = driver.makeReady( + { address: udid, deviceId: udid, driverData: data }, + c.options, + ); + if (c.volume === "hangs") { + const state = promiseState(ready); + await waitForCalls(runner, c.expected.length); + clock.advance(29_999); + await flushMicrotasks221(); + expect(state.state).toBe("pending"); + clock.advance(1); + await flushMicrotasks221(); + expect(state.state).toBe("fulfilled"); + } + const device = await ready; + + expect(device.address).toBe(udid); + expect(runner.calls.map((call) => call.args)).toEqual(c.expected); + expect(failures).toEqual( + c.failure === undefined ? [] : [{ deviceId: udid, detail: c.failure }], + ); + }, + ); +}); diff --git a/src/drivers/ios/index-slim-mode.test.ts b/src/drivers/ios/index-slim-mode.test.ts new file mode 100644 index 000000000..ef0ad4a9a --- /dev/null +++ b/src/drivers/ios/index-slim-mode.test.ts @@ -0,0 +1,601 @@ +import { it, expect, vi, describe } from "vitest"; +import { BootTimeoutError } from "../../core/index.js"; +import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { + iosRuntimeVersionFromId, + supportsPersistentSlim, + type SlimSkippedFact, + IosSimctlDriver, +} from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; +import { + driverData, + FaultyFilesystem, + plantManagedDevice, + createSlimDriver, + prepareSlim, + parseOverrides, + overridesPath, + simctlArgs, + prepareSlimWith, + overridesFolder, + slimBootCalls, + simctl, + deviceRoot, + instanceId, + overridesRoot, + waitForCalls, + flushMicrotasks221, + settledValue, + overridesPlist, + printDisabled, +} from "./test-slim.js"; +import { + neverDisabled, + prepareFull, + slimOptions, + spec, + wantedOverrides, + wantedReadback, + widgetsLabelList, +} from "./test-index.js"; + +describe("IosSimctlDriver", () => { + describe("slim mode", () => { + const widgetsLabels = widgetsLabelList(); + + const widgetsSignature = "v1:0dd8c925463fb678"; + + const slim18_5 = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-5" }; + + const managedCount = Object.keys(wantedOverrides()).length; + + const trueLabel = widgetsLabels[0] ?? ""; + + const falseLabel = "com.apple.siriactionsd"; + + // why: the label written first is not the first in sorted order, so naming the sorted first + // differs from naming the first one the readback check meets. + const firstWritten = Object.keys(wantedOverrides())[0] ?? ""; + + const firstSorted = Object.keys(wantedOverrides()).sort()[0] ?? ""; + + function flipped(labels: readonly string[]): Record { + return Object.fromEntries( + labels.map((label) => [label, wantedOverrides()[label] === true ? "enabled" : "disabled"]), + ); + } + + it("parses simctl runtime ids into [major, minor], including edge cases", () => { + expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18-5")).toEqual([ + 18, 5, + ]); + expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-26-0")).toEqual([ + 26, 0, + ]); + expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18")).toEqual([18, 0]); + expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18-5-1")).toEqual([ + 18, 5, + ]); + expect(iosRuntimeVersionFromId("")).toBeUndefined(); + expect(iosRuntimeVersionFromId("garbage")).toBeUndefined(); + }); + + it("parses a device-specific runtime id (iOS-27-2-iPhone19-4) to the version it names (#1064)", () => { + // why: Xcode 27.2 installs `com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4` next to + // `...iOS-27-2`; the suffix after the version names a device family, not a version part. + expect( + iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4"), + ).toEqual([27, 2]); + }); + + it("gates persistent slim on iOS 18.5+ without falling into string comparison", () => { + // why: A naive string/lexicographic compare would put "9.0" ahead of "18.5" -- the gate must + // compare the parsed integers instead. + expect(supportsPersistentSlim([18, 5])).toBe(true); + expect(supportsPersistentSlim([26, 0])).toBe(true); + expect(supportsPersistentSlim([9, 0])).toBe(false); + expect(supportsPersistentSlim([18, 4])).toBe(false); + expect(supportsPersistentSlim(undefined)).toBe(false); + }); + + it.each([ + { name: "no file yet", existing: undefined, expected: wantedOverrides(), rewritten: true }, + { + name: "an unknown category beside a known one, which is reported", + existing: undefined, + expected: wantedOverrides(), + rewritten: true, + configured: ["widgets", "nonsense"], + unknown: ["nonsense"], + }, + { + name: "a file with the runtime's own entries, which are kept", + existing: { "com.apple.runtime.own": true, "com.example.other": false }, + expected: { + ...wantedOverrides(), + "com.apple.runtime.own": true, + "com.example.other": false, + }, + rewritten: true, + }, + { + name: "a file already as wanted, which is not rewritten", + existing: wantedOverrides(), + expected: wantedOverrides(), + rewritten: false, + }, + { + name: "a file slimmed under wider categories, whose dropped labels are now off the list", + existing: Object.fromEntries( + Object.entries(wantedOverrides()).map(([label]) => [ + label, + !neverDisabled.includes(label), + ]), + ), + expected: wantedOverrides(), + rewritten: true, + }, + ])( + "writes the launchd overrides before the one boot and grants slim after the readback: $name", + async ({ existing, expected, rewritten, configured, unknown }) => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + await plantOverrides(filesystem, existing); + const clock = new FakeClock(); + const { runner, fileAtBoot } = runnerReadingFileAtBoot(filesystem, clock); + const onSlimmed = vi.fn(); + const onSlimSkipped = vi.fn(); + const driver = await createSlimDriver( + runner, + filesystem, + { bootTimeoutMs: 300_000, categories: configured ?? ["widgets"] }, + onSlimmed, + onSlimSkipped, + false, + clock, + ); + + const result = await driver.makeReady( + { address: slim18_5.udid, deviceId: slim18_5.udid, driverData: slim18_5 }, + prepareSlim, + ); + + expect(result).toEqual({ + address: slim18_5.udid, + deviceId: slim18_5.udid, + driverData: slim18_5, + mode: "slim", + }); + expect(parseOverrides(fileAtBoot() ?? "")).toEqual(expected); + expect(filesystem.writes).toEqual(rewritten ? [overridesPath(slim18_5.udid)] : []); + expect(filesystem.modes).toEqual(rewritten ? { file: 0o644, folder: 0o700 } : {}); + expect(runner.calls.map((call) => call.args)).toEqual([ + simctlArgs("boot", slim18_5.udid), + simctlArgs("bootstatus", slim18_5.udid, "-b"), + simctlArgs("listapps", slim18_5.udid), + simctlArgs("spawn", slim18_5.udid, "launchctl", "print-disabled", "system"), + ]); + expect(runner.calls[1]?.options).toEqual({ timeoutMs: 260_000 }); + expect(runner.calls[3]?.options).toEqual({ timeoutMs: 120_000 }); + expect(onSlimmed).toHaveBeenCalledTimes(1); + expect(onSlimmed).toHaveBeenCalledWith({ + address: slim18_5.udid, + categories: ["widgets"], + deviceId: slim18_5.udid, + durationMs: 40_000, + labelCount: widgetsLabels.length, + signature: widgetsSignature, + unknownLabels: unknown ?? [], + }); + expect(onSlimSkipped).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { + name: "a true label reads enabled", + readback: () => readbackWith({ [trueLabel]: "enabled" }), + mode: "full", + detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, + }, + { + name: "a true label is missing", + readback: () => readbackWith({ [trueLabel]: undefined }), + mode: "full", + detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, + }, + { + name: "a false label reads disabled", + readback: () => readbackWith({ [falseLabel]: "disabled" }), + mode: "full", + detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${falseLabel}`, + }, + { + name: "two labels disagree, the first by sort order is named", + readback: () => readbackWith(flipped([firstWritten, firstSorted])), + mode: "full", + detail: `2 of ${managedCount} managed labels did not read back as written at boot; first: ${firstSorted}`, + }, + { + name: "a true label reads true (older launchd)", + readback: () => readbackWith({ [trueLabel]: "true" }), + mode: "slim", + detail: undefined, + }, + { + name: "a false label is missing", + readback: () => readbackWith({ [falseLabel]: undefined }), + mode: "slim", + detail: undefined, + }, + { + name: "the readback exits non-zero", + readback: () => ({ code: 1, stderr: "launchctl: boom\n" }), + mode: "slim", + detail: "could not read back the launchd overrides: exit 1: launchctl: boom", + }, + { + name: "the readback times out", + readback: () => "hangs" as const, + mode: "slim", + detail: "could not read back the launchd overrides: timed out after 120000ms", + }, + ])( + "grants the mode the readback shows and tells why when it disagrees or fails: $name", + async ({ readback, mode, detail }) => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + const script = readback(); + + const { onSlimSkipped, onSlimmed, result } = await prepareSlimWith( + filesystem, + typeof script === "string" && script !== "hangs" ? { stdout: script } : script, + slimOptions(), + ); + + expect(result.mode).toBe(mode); + if (detail === undefined) { + expect(onSlimmed).toHaveBeenCalledTimes(1); + expect(onSlimSkipped).not.toHaveBeenCalled(); + } else { + expect(onSlimmed).not.toHaveBeenCalled(); + expect(onSlimSkipped.mock.calls).toEqual([ + [{ deviceId: slim18_5.udid, detail, reason: "apply-failed" }], + ]); + } + }, + ); + + it.each([ + { + name: "writeFileAtomic throws and the readback is as wanted", + failing: "writeFileAtomic" as const, + existing: undefined, + readback: { stdout: readbackWith({}) }, + mode: "slim", + detail: "could not write the launchd overrides file: disk full", + }, + { + name: "mkdirp throws and the readback disagrees", + failing: "mkdirp" as const, + existing: undefined, + readback: { stdout: readbackWith({ [trueLabel]: "enabled" }) }, + mode: "full", + detail: `could not write the launchd overrides file: disk full; 1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, + }, + { + name: "the write throws and the readback fails", + failing: "writeFileAtomic" as const, + existing: undefined, + readback: { code: 1, stderr: "boom" }, + mode: "slim", + detail: + "could not write the launchd overrides file: disk full; could not read back the launchd overrides: exit 1: boom", + }, + { + name: "the existing file is not a plist dict, is left unchanged, and the readback decides", + failing: undefined, + existing: "not a plist", + readback: { stdout: readbackWith({}) }, + mode: "slim", + detail: + "could not write the launchd overrides file: the existing file is not a property list dictionary", + }, + ])( + "boots anyway when the overrides file cannot be written and says so: $name", + async ({ failing, existing, readback, mode, detail }) => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + if (existing !== undefined) { + await filesystem.mkdirp(overridesFolder(slim18_5.udid)); + await filesystem.writeFileAtomic(overridesPath(slim18_5.udid), existing); + } + filesystem.failing = failing; + + const { onSlimSkipped, onSlimmed, result, runner } = await prepareSlimWith( + filesystem, + readback, + slimOptions(), + ); + + expect(result.mode).toBe(mode); + expect(runner.calls.map((call) => call.args[3])).toEqual([ + "boot", + "bootstatus", + "listapps", + "spawn", + ]); + expect(onSlimmed).not.toHaveBeenCalled(); + expect(onSlimSkipped.mock.calls).toEqual([ + [{ deviceId: slim18_5.udid, detail, reason: "apply-failed" }], + ]); + if (existing !== undefined) { + expect(await filesystem.readFile(overridesPath(slim18_5.udid))).toBe(existing); + } + }, + ); + + it("ignores a full flag an older daemon stamped into driver data: the spec mode decides", async () => { + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const stamped = { ...slim18_5, full: true }; + const runner = new ScriptedProcessRunner(slimBootCalls(stamped.udid, wantedReadback())); + const driver = await createSlimDriver(runner, filesystem, slimOptions()); + + const result = await driver.makeReady( + { address: stamped.udid, deviceId: stamped.udid, driverData: stamped }, + prepareSlim, + ); + + expect(result.mode).toBe("slim"); + }); + + it.each([ + { + name: "full mode", + runtimeId: slim18_5.runtimeId, + options: { mode: "full", purpose: "prepare" } as const, + categories: ["widgets"], + skipped: undefined, + }, + { + name: "a purpose: 'recover' boot, which only reboots", + runtimeId: slim18_5.runtimeId, + options: { mode: "slim", purpose: "recover" } as const, + categories: ["widgets"], + skipped: undefined, + }, + { + name: "runtime 18.4", + runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-4", + options: prepareSlim, + categories: ["widgets"], + skipped: "runtime-too-old", + }, + { + name: "a runtime id that does not parse", + runtimeId: "not-a-runtime-id", + options: prepareSlim, + categories: ["widgets"], + skipped: "unknown-runtime", + }, + { + name: "configured categories none of which is known", + runtimeId: slim18_5.runtimeId, + options: prepareSlim, + categories: ["nonsense"], + skipped: "apply-failed", + }, + ])( + "touches no overrides file and runs no readback for a device that must not be slimmed: $name", + async ({ runtimeId, options, categories, skipped }) => { + const filesystem = new FaultyFilesystem(); + await plantManagedDevice(filesystem); + filesystem.writes.length = 0; + const data = { ...driverData, runtimeId }; + const runner = new ScriptedProcessRunner([ + { match: simctl("boot", data.udid) }, + { match: simctl("bootstatus", data.udid, "-b") }, + { match: simctl("listapps", data.udid) }, + ]); + const onSlimmed = vi.fn(); + const onSlimSkipped = vi.fn(); + const driver = await createSlimDriver( + runner, + filesystem, + { bootTimeoutMs: 300_000, categories }, + onSlimmed, + onSlimSkipped, + ); + + const result = await driver.makeReady( + { address: data.udid, deviceId: data.udid, driverData: data }, + options, + ); + + expect(result).toEqual({ + address: data.udid, + deviceId: data.udid, + driverData: data, + mode: "full", + }); + expect(runner.calls.map((call) => call.args)).toEqual([ + simctlArgs("boot", data.udid), + simctlArgs("bootstatus", data.udid, "-b"), + simctlArgs("listapps", data.udid), + ]); + expect(filesystem.writes).toEqual([]); + expect(await filesystem.exists(overridesFolder(data.udid))).toBe(false); + expect(onSlimmed).not.toHaveBeenCalled(); + expect(onSlimSkipped.mock.calls.map(([fact]) => (fact as SlimSkippedFact).reason)).toEqual( + skipped === undefined ? [] : [skipped], + ); + }, + ); + + it("times out the slim boot's bootstatus using slim.bootTimeoutMs, not the default 120s", async () => { + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: { command: "xcrun", args: simctlArgs("boot", slim18_5.udid) } }, + { + hangs: true, + match: { command: "xcrun", args: simctlArgs("bootstatus", slim18_5.udid, "-b") }, + }, + { match: { command: "xcrun", args: simctlArgs("shutdown", slim18_5.udid) } }, + ]); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + driverConfig: { deviceRoot }, + instanceId, + simlockHome: "/home/.simlock", + clock, + filesystem, + idGenerator: { generate: () => "device-1" }, + launchdOverridesRoot: overridesRoot, + processRunner: runner, + config: { slim: slimOptions() }, + }); + + const ready = driver.makeReady( + { address: slim18_5.udid, deviceId: slim18_5.udid, driverData: slim18_5 }, + prepareSlim, + ); + + await waitForCalls(runner, 2); + const readyState = promiseState(ready); + + // why: Just past the default 120s: a boot timed against it would have given up by now. + clock.advance(120_001); + await flushMicrotasks221(); + expect(readyState.state).toBe("pending"); + + clock.advance(300_000 - 120_001); + expect(await settledValue(ready)).toBeInstanceOf(BootTimeoutError); + expect(runner.calls[1]?.options).toEqual({ timeoutMs: 300_000 }); + }); + + it("an operator's full iOS boot waits ios.full.bootTimeoutMs, then fails with BootTimeoutError", async () => { + const slim18_4 = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }; + const cases = [ + { data: driverData, full: undefined, options: prepareFull, waitsMs: 300_000 }, + { data: driverData, full: 400_000, options: prepareFull, waitsMs: 400_000 }, + { data: slim18_4, full: 400_000, options: prepareSlim, waitsMs: 400_000 }, + { data: driverData, full: 400_000, options: recoverFull, waitsMs: 400_000 }, + { data: driverData, full: 400_000, options: recoverSlim, waitsMs: 400_000 }, + { data: slim18_5, full: 400_000, options: prepareSlim, waitsMs: 250_000 }, + ]; + for (const { data, full, options, waitsMs } of cases) { + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const clock = new FakeClock(); + const runner = new ScriptedProcessRunner([ + { match: { command: "xcrun", args: simctlArgs("boot", data.udid) } }, + { + hangs: true, + match: { command: "xcrun", args: simctlArgs("bootstatus", data.udid, "-b") }, + }, + { match: { command: "xcrun", args: simctlArgs("shutdown", data.udid) } }, + ]); + const driver = await IosSimctlDriver.create({ + database: await memorySourceDatabase(), + driverConfig: { deviceRoot }, + instanceId, + simlockHome: "/home/.simlock", + clock, + filesystem, + idGenerator: { generate: () => "device-1" }, + launchdOverridesRoot: overridesRoot, + processRunner: runner, + config: { + slim: slimOptions(250_000), + ...(full === undefined ? {} : { full: { bootTimeoutMs: full } }), + }, + }); + + const ready = driver.makeReady( + { address: data.udid, deviceId: data.udid, driverData: data }, + options, + ); + await waitForCalls(runner, 2); + const readyState = promiseState(ready); + + clock.advance(waitsMs - 1); + await flushMicrotasks221(); + expect(readyState.state, `${JSON.stringify({ full, options })} still waiting`).toBe( + "pending", + ); + + clock.advance(1); + expect(await settledValue(ready)).toBeInstanceOf(BootTimeoutError); + } + }); + + it.each([ + { name: "slim spec", spec: { ...spec, mode: "slim" } as const }, + { name: "full spec", spec }, + ])( + "quotes the cold-boot cost for a boot of a $name, since a slim boot is one boot", + async (c) => { + const driver = await createSlimDriver( + new ScriptedProcessRunner([]), + new MemoryFilesystem(), + slimOptions(), + ); + + expect(driver.estimate({ operation: "boot" }, c.spec)).toBe(60_000); + }, + ); + }); +}); + +/** why: the state a case starts from, with the writes and modes the planting made forgotten. */ +async function plantOverrides( + filesystem: FaultyFilesystem, + existing: Readonly> | undefined, +): Promise { + if (existing !== undefined) { + await filesystem.mkdirp(overridesFolder(driverData.udid)); + await filesystem.writeFileAtomic(overridesPath(driverData.udid), overridesPlist(existing)); + } + filesystem.writes.length = 0; + delete filesystem.modes.file; + delete filesystem.modes.folder; +} + +/** why: the boot is where launchd reads the file, so the file is captured as the boot starts. */ +function runnerReadingFileAtBoot(filesystem: FaultyFilesystem, clock: FakeClock) { + let contents: string | undefined; + const calls = slimBootCalls(driverData.udid, wantedReadback()); + const runner = new ScriptedProcessRunner([ + { + ...calls[0]!, + onSpawn: async () => { + clock.advance(40_000); + const path = overridesPath(driverData.udid); + contents = (await filesystem.exists(path)) ? await filesystem.readFile(path) : undefined; + }, + }, + ...calls.slice(1), + ]); + return { fileAtBoot: () => contents, runner }; +} + +/** why: The readback of a boot that honoured every label, with the given labels changed or dropped (`undefined`). */ +function readbackWith(changes: Readonly>): string { + const entries: Record = Object.fromEntries( + Object.entries(wantedOverrides()).map(([label, off]) => [label, off ? "disabled" : "enabled"]), + ); + for (const [label, state] of Object.entries(changes)) { + if (state === undefined) delete entries[label]; + else entries[label] = state; + } + return printDisabled(entries); +} + +const recoverFull = { mode: "full", purpose: "recover" } as const; + +const recoverSlim = { mode: "slim", purpose: "recover" } as const; diff --git a/src/drivers/ios/index-slim-resolve-advisories.test.ts b/src/drivers/ios/index-slim-resolve-advisories.test.ts new file mode 100644 index 000000000..856ea6ffb --- /dev/null +++ b/src/drivers/ios/index-slim-resolve-advisories.test.ts @@ -0,0 +1,457 @@ +import { afterEach, expect, describe, it } from "vitest"; +import { MemoryFilesystem, ScriptedProcessRunner, FakeClock } from "../../ports/index.js"; +import { deviceRoot, createSlimDriver } from "./test-slim.js"; +import { + IOS_RUNTIME_ASSET_ROOT, + SizedFilesystem, + UNSCOPED_LEGACY_CALLS, + assetInfoPlist, + createDriver, + listFixture, + listInvocation, + scopedRunners, + scriptedListRunner, + slimOptions, +} from "./test-index.js"; + +/** + * why: The one global invariant, asserted over everything every test in this file recorded + * rather than per call site: scoping is what makes ownership provable, so a spawn that + * reached `xcrun` without `simctl --set ` in front of it would address the machine's + * default device set (safety rule 8). Per-test argv equality proves today's calls are + * scoped; only this proves the next one added is, whether or not it goes through + * `#invokeSimctl`. + */ +afterEach(() => { + for (const call of scopedRunners.splice(0).flatMap((runner) => runner.calls)) { + // why: Not every spawn addresses a device: `xcodebuild -downloadPlatform` installs a runtime + // and has no device set to point at. The invariant is about the calls that *do* -- a + // `simctl` that reached the machine's default set is the leak this guards. + if (call.command !== "xcrun" || call.args[0] !== "simctl") continue; + if (UNSCOPED_LEGACY_CALLS.has(call.args.join(" "))) continue; + expect(call.args.slice(0, 2)).toEqual(["simctl", "--set"]); + expect([deviceRoot, `${deviceRoot}/sources`]).toContain(call.args[2]); + } +}); + +describe("IosSimctlDriver", () => { + describe("slim mode", () => { + describe("resolveSpec and the device mode", () => { + const iphone16 = { model: "iPhone 16", platform: "ios" } as const; + + it("returns a full spec for a slim request on a runtime below 18.5", async () => { + const driver = await createSlimDriver( + scriptedListRunner(), + new MemoryFilesystem(), + slimOptions(), + ); + + await expect( + driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "18.4" }), + ).resolves.toEqual({ ...iphone16, osVersion: "18.4", physical: false }); + }); + + it("gives two requests with different modes for one model and runtime different specs", async () => { + const list = { + match: listInvocation, + result: { code: 0, stderr: "", stdout: listFixture }, + }; + const driver = await createSlimDriver( + new ScriptedProcessRunner([list, list]), + new MemoryFilesystem(), + slimOptions(), + ); + + const slim = await driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "26.5" }); + const full = await driver.resolveSpec({ ...iphone16, mode: "full", osVersion: "26.5" }); + + expect(slim).toEqual({ ...iphone16, mode: "slim", osVersion: "26.5", physical: false }); + expect(full).toEqual({ ...iphone16, osVersion: "26.5", physical: false }); + }); + + it("returns a full spec for a slim request when slim is not configured", async () => { + const driver = await createDriver(scriptedListRunner()); + + await expect( + driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "26.5" }), + ).resolves.toEqual({ ...iphone16, osVersion: "26.5", physical: false }); + }); + }); + + describe("advisories()", () => { + it("reports slim-runtime-unsupported when the default mode is slim and an installed runtime predates 18.5", async () => { + const driver = await createSlimDriver( + scriptedListRunner(), + new MemoryFilesystem(), + slimOptions(), + undefined, + undefined, + true, + ); + + await expect(driver.advisories()).resolves.toEqual([ + { + code: "slim-runtime-unsupported", + message: expect.stringContaining("18.4"), + }, + ]); + }); + + it("reports nothing when every installed runtime is 18.5+", async () => { + const onlyNewFixture = JSON.stringify({ + devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, + runtimes: ( + JSON.parse(listFixture) as { runtimes: { version: string }[] } + ).runtimes.filter((runtime) => runtime.version !== "18.4"), + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: onlyNewFixture } }, + ]); + const driver = await createSlimDriver( + runner, + new MemoryFilesystem(), + slimOptions(), + undefined, + undefined, + true, + ); + + await expect(driver.advisories()).resolves.toEqual([]); + }); + + it("reports nothing when slim is not configured", async () => { + const driver = await createDriver(scriptedListRunner()); + + await expect(driver.advisories()).resolves.toEqual([]); + }); + + it("reports an installed runtime with an unparseable identifier as unsupported, even though its marketing version looks modern (finding #5, issue #87 review)", async () => { + // why: The marketing `version` string here ("26.5") would pass the persistent-slim floor if + // `advisories()` still parsed it directly -- but this fixture's `identifier` doesn't + // match `iosRuntimeVersionFromId`'s pattern, and `planSlimBoot` (the boot-time decision) + // only ever reads the identifier. `advisories()` must reuse that same call so the two + // can never disagree about whether this runtime actually gets slimmed. + const devicetypes = (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes; + const weirdIdentifierFixture = JSON.stringify({ + devicetypes, + runtimes: [ + { + identifier: "com.apple.CoreSimulator.SimRuntime.iOS-not-a-version", + isAvailable: true, + name: "iOS 26.5", + supportedDeviceTypes: [], + version: "26.5", + }, + ], + }); + const runner = new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: weirdIdentifierFixture }, + }, + ]); + const driver = await createSlimDriver( + runner, + new MemoryFilesystem(), + slimOptions(), + undefined, + undefined, + true, + ); + + await expect(driver.advisories()).resolves.toEqual([ + { + code: "slim-runtime-unsupported", + message: expect.stringContaining("26.5"), + }, + ]); + }); + + it("reports nothing for an iOS 27.2 device-specific runtime installed beside the general one (#1064)", async () => { + const devicetypes = (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes; + const deviceSpecificFixture = JSON.stringify({ + devicetypes, + runtimes: [ + { + buildversion: "24B91016s", + identifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4", + isAvailable: true, + name: "iOS 27.2", + supportedDeviceTypes: [], + version: "27.2", + }, + { + buildversion: "24B8117", + identifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-2", + isAvailable: true, + name: "iOS 27.2", + supportedDeviceTypes: [], + version: "27.2", + }, + ], + }); + const runner = new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: deviceSpecificFixture }, + }, + ]); + const driver = await createSlimDriver( + runner, + new MemoryFilesystem(), + slimOptions(), + undefined, + undefined, + true, + ); + + await expect(driver.advisories()).resolves.toEqual([]); + }); + + it("reports nothing when the default mode is full, even with a runtime below 18.5 installed", async () => { + const driver = await createSlimDriver( + scriptedListRunner(), + new MemoryFilesystem(), + slimOptions(), + undefined, + undefined, + false, + ); + + await expect(driver.advisories()).resolves.toEqual([]); + }); + + it("reports downloaded runtime assets whose runtime is not in the catalog, and only those (#79)", async () => { + // why: `simctl runtime delete` only unregisters a runtime from CoreSimulator: its ~8.5 GiB + // asset bundle stays in mobileassetd's store, tagged `NeverCollected`, on the same + // volume the download preflight measures. So the bytes stay spent and CoreSimulator can + // re-register the runtime from them later. The catalog here installs 18.4 and 26.5 while + // the store still holds two builds nobody deleted the assets for. + const filesystem = new MemoryFilesystem(); + for (const [bundle, simulatorVersion, build] of [ + ["a1.asset", "18.4", "22E238"], + ["b2.asset", "26.5", "23F79"], + ["c3.asset", "18.6", "22G86"], + ["d4.asset", "26.3", "23D60"], + ] as const) { + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, + assetInfoPlist(simulatorVersion, build), + ); + } + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + // why: The advisory code and message shape are a proposal (see the triage report), so this + // pins only what any fix must do: name both orphan builds, name neither installed one. + const advisories = await driver.advisories(); + const text = advisories.map((advisory) => advisory.message).join("\n"); + expect(advisories.map((advisory) => advisory.code)).toContain( + "runtime-cache-unreclaimable", + ); + expect(text).toContain("18.6"); + expect(text).toContain("26.3"); + expect(text).not.toContain("18.4"); + expect(text).not.toContain("26.5"); + }); + + it("tells two downloads of one marketing version apart by build, and reports only the uninstalled one (#79)", async () => { + // why: simctl reports `buildversion`, and a version match alone would call this bundle + // installed: 18.4 is in the catalog, but not from this download. Deleting the runtime + // and re-downloading the same version leaves exactly this -- a stale build nobody can + // reclaim, sitting behind an installed runtime of the same name. + const catalog = JSON.parse(listFixture) as { + devicetypes: unknown; + runtimes: { version: string }[]; + }; + const withBuilds = JSON.stringify({ + devicetypes: catalog.devicetypes, + runtimes: catalog.runtimes.map((runtime) => ({ + ...runtime, + buildversion: runtime.version === "18.4" ? "22E238" : "23F79", + })), + }); + const filesystem = new MemoryFilesystem(); + for (const [bundle, build] of [ + ["a1.asset", "22E238"], + ["stale.asset", "22E247"], + ] as const) { + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, + assetInfoPlist("18.4", build), + ); + } + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: withBuilds } }, + ]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + const advisories = await driver.advisories(); + + expect(advisories).toEqual([ + { + code: "runtime-cache-unreclaimable", + message: expect.stringContaining("22E247"), + }, + ]); + expect(advisories[0]?.message).not.toContain("22E238"); + }); + + it("names orphaned runtimes oldest first, not in string order (#79)", async () => { + // why: A store that has been collecting for months is read by a human deciding what to + // reclaim; "iOS 9.3, iOS 18.6" is that list, and a plain string sort inverts it. + const filesystem = new MemoryFilesystem(); + for (const [bundle, simulatorVersion, build] of [ + ["new.asset", "18.6", "22G86"], + ["old.asset", "9.3", "13E233"], + ] as const) { + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, + assetInfoPlist(simulatorVersion, build), + ); + } + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + const message = (await driver.advisories())[0]?.message ?? ""; + + expect(message.indexOf("9.3")).toBeLessThan(message.indexOf("18.6")); + }); + + it("states no size in the advisory when a leftover download's size cannot be read", async () => { + const bundle = `${IOS_RUNTIME_ASSET_ROOT}/c3.asset`; + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(`${bundle}/AssetData`); + await filesystem.writeFileAtomic(`${bundle}/Info.plist`, assetInfoPlist("18.6", "22G86")); + filesystem.defineFailure(`${bundle}/AssetData`, "EACCES"); + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + const advisories = await driver.advisories(); + + expect(advisories).toEqual([ + { + code: "runtime-cache-unreclaimable", + message: expect.stringContaining( + "iOS 18.6 (22G86) is no longer installed, but its download is still in", + ), + }, + ]); + expect(advisories[0]?.message).not.toContain("GiB"); + }); + + it("counts two downloads of one build as downloads of one runtime, with their total size", async () => { + const filesystem = new SizedFilesystem( + new Map([ + [`${IOS_RUNTIME_ASSET_ROOT}/c3.asset`, 8_917_036 * 1024], + [`${IOS_RUNTIME_ASSET_ROOT}/c4.asset`, 8_917_036 * 1024], + ]), + ); + for (const bundle of ["c3.asset", "c4.asset"]) { + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, + assetInfoPlist("18.6", "22G86"), + ); + } + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + const message = (await driver.advisories())[0]?.message ?? ""; + + expect(message).toContain( + "iOS 18.6 (22G86) is no longer installed, but its downloads (17.0 GiB in all) are " + + "still in", + ); + }); + + it("states the measured sizes of several leftover downloads as one total", async () => { + // why: The sizes of the iOS 18.6 and 18.4 downloads measured on a real machine; this catalog + // installs 18.4, so the second bundle is labelled 18.3 to be reported. + const filesystem = new SizedFilesystem( + new Map([ + [`${IOS_RUNTIME_ASSET_ROOT}/c3.asset`, 8_917_036 * 1024], + [`${IOS_RUNTIME_ASSET_ROOT}/e5.asset`, 8_916_744 * 1024], + ]), + ); + for (const [bundle, simulatorVersion, build] of [ + ["c3.asset", "18.6", "22G86"], + ["e5.asset", "18.3", "22D8075"], + ] as const) { + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, + assetInfoPlist(simulatorVersion, build), + ); + } + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + const advisories = await driver.advisories(); + + expect(advisories).toEqual([ + { + code: "runtime-cache-unreclaimable", + message: expect.stringContaining( + "iOS 18.3 (22D8075), 18.6 (22G86) are no longer installed, but their downloads " + + "(17.0 GiB in all) are still in", + ), + }, + ]); + }); + + it("reports nothing when every downloaded asset belongs to an installed runtime (#79)", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/a1.asset`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/a1.asset/Info.plist`, + assetInfoPlist("18.4", "22E238"), + ); + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + await expect(driver.advisories()).resolves.toEqual([]); + }); + + it("stays quiet, and asks simctl nothing, when the asset store cannot be read (#79)", async () => { + // why: The store is macOS's, not Simlock's: absent on a machine that never downloaded a + // runtime, and readable only to whoever the OS says. Neither is a `doctor` failure -- + // and with nothing to report, the `simctl list` that would classify the bundles is + // never worth running, which is why the runner below scripts no invocation at all. + const filesystem = new MemoryFilesystem(); + filesystem.defineFailure(IOS_RUNTIME_ASSET_ROOT, "EACCES"); + const runner = new ScriptedProcessRunner([]); + const driver = await createDriver(runner, new FakeClock(), filesystem); + + await expect(driver.advisories()).resolves.toEqual([]); + expect(runner.calls).toEqual([]); + }); + + it("skips an asset bundle it cannot identify by build rather than naming it (#79)", async () => { + // why: The build is what decides whether a bundle's runtime is installed, so a bundle + // without one cannot be classified -- and guessing would send the operator to delete + // a runtime they are still using. `missing-build.asset` below names iOS 19.9 and + // nothing else; the store around it is still reported. + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/missing-build.asset`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/missing-build.asset/Info.plist`, + assetInfoPlist("19.9", "22H1").replace("Build22H1", ""), + ); + await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/orphan.asset`); + await filesystem.writeFileAtomic( + `${IOS_RUNTIME_ASSET_ROOT}/orphan.asset/Info.plist`, + assetInfoPlist("18.6", "22G86"), + ); + const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); + + const advisories = await driver.advisories(); + + expect(advisories).toEqual([ + { + code: "runtime-cache-unreclaimable", + message: expect.stringContaining("18.6"), + }, + ]); + expect(advisories[0]?.message).not.toContain("19.9"); + }); + }); + }); +}); diff --git a/src/drivers/ios/index.test.ts b/src/drivers/ios/index.test.ts index b8a6e6ae3..e23b9ed68 100644 --- a/src/drivers/ios/index.test.ts +++ b/src/drivers/ios/index.test.ts @@ -1,86 +1,18 @@ -import { readFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { - BootTimeoutError, - ComponentInUseError, - ComponentNotOwnedError, - DriverCrashError, - OwnedRootError, - OWNED_ROOT_MARKER_FILE, - PassthroughRefusedError, - RuntimeMissingError, - UnknownModelError, -} from "../../core/index.js"; -import { - FakeBytePipeLauncher, - FakeClock, - MemoryFilesystem, - NodeFilesystem, - NodeProcessRunner, - ScriptedProcessRunner, - SystemClock, - type Filesystem, - type Logger, - type ProcessHandle, - type ProcessResult, - type ProcessRunner, - type ProcessRunOptions, - type SqliteDatabase, -} from "../../ports/index.js"; -import { promiseState } from "../../test-support/promise-state.js"; +import { afterEach, expect, it, describe } from "vitest"; +import { UnknownModelError, DriverCrashError, RuntimeMissingError } from "../../core/index.js"; +import { ScriptedProcessRunner, MemoryFilesystem } from "../../ports/index.js"; +import { SLIM_CATEGORIES } from "./index.js"; +import { deviceRoot, createSlimDriver } from "./test-slim.js"; import { - IosSimctlDriver, - iosRuntimeVersionFromId, - SLIM_CATEGORIES, - supportsPersistentSlim, - type SilenceFailedFact, - type SlimSkippedFact, -} from "./index.js"; -import { memorySourceDatabase, plantReadySeed, plantSource } from "./testing.js"; -import { - FaultyFilesystem, - createSlimDriver, - dataPath, - deviceRoot, - driverData, - instanceId, - overridesFolder, - overridesPath, - overridesPlist, - overridesRoot, - parseOverrides, - plantManagedDevice, - prepareSlim, - deviceHubEdit, - flushMicrotasks221, - prepareSlimWith, - printDisabled, - settledValue, - simctl, - simctlArgs, - slimBootCalls, - waitForCalls, -} from "./test-slim.js"; - -const listFixture = readFileSync(new URL("./fixtures/simctl-list.json", import.meta.url), "utf8"); -const listDevicesFixture = readFileSync( - new URL("./fixtures/simctl-list-devices.json", import.meta.url), - "utf8", -); -const listInvocation = simctl("list", "-j", "devicetypes", "runtimes"); -const listDevicesInvocation = simctl("list", "-j", "devices"); + UNSCOPED_LEGACY_CALLS, + createDriver, + listInvocation, + pairingFixture, + scopedRunners, + scriptedListRunner, + spec, +} from "./test-index.js"; -// why: iPhone 16 pairs with both installed runtimes; iPhone Xs pairs only with the older one (iOS 26 -// dropped it, mirroring real Xcode 27) and its range caps out at iOS 18.6; iPhone 7 caps out at -// iOS 15.0, below the 16.0 auto-download floor. -const pairingFixture = readFileSync( - new URL("./fixtures/simctl-list-pairing.json", import.meta.url), - "utf8", -); const modelRuntimesFixture = JSON.stringify({ devicetypes: [ deviceType("iPhone-16", "iPhone 16"), @@ -125,48 +57,6 @@ function runtime(version: string, build: string, isAvailable: boolean, supports: }; } -// why: Same catalog as `listFixture`, plus an iOS 18.6 runtime that has just finished downloading -- -// stands in for the re-scanned catalog `resolveSpec` reads after a successful `xcodebuild` call. -const listFixtureAfterDownload = JSON.stringify({ - devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, - runtimes: [ - ...(JSON.parse(listFixture) as { runtimes: unknown[] }).runtimes, - { - identifier: "com.apple.CoreSimulator.SimRuntime.iOS-18-6", - isAvailable: true, - name: "iOS 18.6", - supportedDeviceTypes: [ - { identifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", name: "iPhone 16" }, - ], - version: "18.6", - }, - ], -}); -const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; -const durableMarkPath = `${deviceRoot}/${driverData.udid}/simlock-mark.json`; -const erasableMarkPath = `${dataPath}/simlock-mark.json`; - -/** why: Every simctl call the driver makes is scoped to its device set; so is every expectation. */ - -const scopedRunners: ScriptedProcessRunner[] = []; - -const legacyUdid = "00000000-0000-0000-0000-0000000000ff"; - -/** - * why: The complete argv of every call the pre-root path is allowed to make without `--set`, - * spelled out rather than pattern-matched: `findLegacy` / `destroyLegacy` deliberately - * address the machine's default set (ADR 0001, Migration), and `listComponents` reads it with - * the same `list` to count foreign devices (ADR 0010 §8). Every *other* call must still fail - * the invariant below if its scoping ever goes missing. - */ -const UNSCOPED_LEGACY_CALLS = new Set( - [ - ["simctl", "list", "-j", "devices"], - ["simctl", "shutdown", legacyUdid], - ["simctl", "delete", legacyUdid], - ].map((argv) => argv.join(" ")), -); - /** * why: The one global invariant, asserted over everything every test in this file recorded * rather than per call site: scoping is what makes ownership provable, so a spawn that @@ -187,51 +77,6 @@ afterEach(() => { } }); -const seedUdid = "00000000-0000-0000-0000-0000000000aa"; - -interface Catalog { - devicetypes: { identifier: string }[]; - runtimes: { buildversion?: string; identifier: string }[]; -} - -/** why: A seed lives in the sources set, a sibling of the lease device set, and its calls are scoped to that. */ -function sourcesSimctl(...args: readonly string[]): { - readonly command: string; - readonly args: string[]; -} { - return { args: ["simctl", "--set", `${deviceRoot}/sources`, ...args], command: "xcrun" }; -} - -/** why: A database that already holds the seed of the kind `spec` names, so a test of the copy step starts at the clone. */ -async function seededDatabase( - seed: { readonly runtimeBuild?: string; readonly udid?: string } = {}, -): Promise { - return plantReadySeed(await memorySourceDatabase(), { - deviceTypeId: driverData.deviceTypeId, - runtimeId: driverData.runtimeId, - udid: seedUdid, - ...seed, - }); -} - -function deviceListResponse(state: string): string { - return JSON.stringify({ - devices: { - "com.apple.CoreSimulator.SimRuntime.iOS-26-5": [ - { name: driverData.name, udid: driverData.udid, state, dataPath }, - ], - }, - }); -} - -const PHYSICAL_DEVICE = { - address: "00008110-001A2B3C4D5E", - deviceId: "00008110-001A2B3C4D5E", - driverData: {}, - enrolledApps: ["com.example.kept"], - physical: true, -} as const; - describe("IosSimctlDriver", () => { it("resolves an exact model and requested installed runtime", async () => { const runner = scriptedListRunner(); @@ -473,5718 +318,186 @@ describe("IosSimctlDriver", () => { }); }); - describe("component install", () => { - const runtimeListInvocation = simctl("runtime", "list", "-j"); - const imagesBefore = [ - { build: "22E238", id: "IMG-184", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }, - { build: "23F77", id: "IMG-265", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, - ]; - const imagesAfter = [ - ...imagesBefore, - { build: "22G86", id: "IMG-186", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-6" }, - ]; - - function listed(stdout: string) { - return { match: listInvocation, result: { code: 0, stderr: "", stdout } }; - } - - function imagesListed(images: readonly { build: string; id: string; runtime: string }[]) { - const stdout = JSON.stringify( - Object.fromEntries( - images.map((image) => [ - image.id, - { - build: image.build, - identifier: image.id, - runtimeIdentifier: image.runtime, - state: "Ready", - }, - ]), - ), - ); - return { match: runtimeListInvocation, result: { code: 0, stderr: "", stdout } }; - } - - function xcodebuild(args: readonly string[], extra: object = {}) { - return { match: { args: [...args], command: "xcodebuild" }, ...extra }; - } - - function install( - driver: IosSimctlDriver, - component: string, - options: { onProgress?: (progress: unknown) => void; signal?: AbortSignal } = {}, - ) { - return driver.installComponent(component, { - onProgress: options.onProgress ?? (() => undefined), - signal: options.signal ?? new AbortController().signal, - }); - } - - it("installs an exact version with -buildVersion and no timeout of its own, verifies it in a fresh read, and reports it installed with its image and build", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"]), - listed(listFixtureAfterDownload), - imagesListed(imagesAfter), - ]); - const driver = await createDriver(runner); - - await expect(install(driver, "18.6")).resolves.toEqual({ - outcome: "installed", - receipt: { build: "22G86", image: "IMG-186" }, - version: "18.6", - }); - const xcodebuildCall = runner.calls.find((call) => call.command === "xcodebuild"); - expect(xcodebuildCall?.options).toEqual({ lineEnd: "carriage-return-too" }); - }); - - it("reports already-installed when the runtime the installer leaves was there before it ran", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS"]), - listed(listFixture), - imagesListed(imagesBefore), - ]); - const driver = await createDriver(runner); - - await expect(install(driver, "latest")).resolves.toEqual({ - outcome: "already-installed", - receipt: { build: "23F77", image: "IMG-265" }, - version: "26.5", - }); - }); - - it("installs a bare major with -buildVersion and reports the newest runtime of it", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18"]), - listed(listFixtureAfterDownload), - imagesListed(imagesAfter), - ]); - const driver = await createDriver(runner); + it("lists resolvable models and installed runtimes, defaulting to the newest", async () => { + const driver = await createDriver(scriptedListRunner()); - await expect(install(driver, "18")).resolves.toMatchObject({ - outcome: "installed", - version: "18.6", - }); + // why: Strict: a catalog key an iOS driver does not fill (`images`, `customModels`) must be + // absent, not present as undefined. + await expect(driver.listCatalog()).resolves.toStrictEqual({ + defaultRuntime: "26.5", + modelAliases: {}, + modelClasses: {}, + modelRuntimes: { + "iPhone 15 Pro": ["18.4", "26.5"], + "iPhone 16": ["18.4", "26.5"], + "iPhone 17 Pro": ["18.4", "26.5"], + }, + models: ["iPhone 17 Pro", "iPhone 16", "iPhone 15 Pro"], + runtimes: ["18.4", "26.5"], + slimCategories: expect.any(Array), }); + }); - it("refuses a version below the download floor without running anything", async () => { - const runner = new ScriptedProcessRunner([]); - const driver = await createDriver(runner); - - const result = await install(driver, "13.0").catch((error: unknown) => error); - - expect(result).toBeInstanceOf(RuntimeMissingError); - expect(result).toMatchObject({ - downloadable: false, - message: expect.stringContaining("16.0"), - }); - expect(runner.calls).toEqual([]); + it("lists every slim category with none configured, and only the configured ones otherwise", async () => { + const unset = await createDriver(scriptedListRunner()); + const configured = await createSlimDriver(scriptedListRunner(), new MemoryFilesystem(), { + bootTimeoutMs: 1000, + categories: ["store"], }); - it("fails when xcodebuild exits 0 but the runtime is still not installed", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"]), - listed(listFixture), - imagesListed(imagesBefore), - ]); - const driver = await createDriver(runner); - - const result = await install(driver, "18.6").catch((error: unknown) => error); - - expect(result).toBeInstanceOf(DriverCrashError); - expect((result as Error).message).toContain("iOS 18.6 is still not installed"); - }); + expect((await unset.listCatalog()).slimCategories).toEqual( + SLIM_CATEGORIES.map((category) => category.name), + ); + expect((await configured.listCatalog()).slimCategories).toEqual(["store"]); + }); - it("fails with xcodebuild's own output when it exits non-zero", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { - result: { code: 1, stderr: "no network", stdout: "" }, - }), + describe("model classes", () => { + function classesRunner(deviceTypes: unknown[]): ScriptedProcessRunner { + return new ScriptedProcessRunner([ + { + match: listInvocation, + result: { + code: 0, + stderr: "", + stdout: JSON.stringify({ + devicetypes: deviceTypes, + runtimes: [runtime("26.0", "23A339", true, [])], + }), + }, + }, ]); - const driver = await createDriver(runner); + } - const result = await install(driver, "18.6").catch((error: unknown) => error); + it("classes a device type by its product family: iPhone is phone, iPad is tablet, Apple Watch is watch, Apple TV is tv, Apple Vision is vision", async () => { + const driver = await createDriver( + classesRunner([ + deviceType("iPhone-17", "iPhone 17", 0xffffff, "iPhone"), + deviceType("iPad-Pro", "iPad Pro", 0xffffff, "iPad"), + deviceType("Watch-11", "Apple Watch Series 11 (46mm)", 0xffffff, "Apple Watch"), + deviceType("TV-4K", "Apple TV 4K", 0xffffff, "Apple TV"), + deviceType("Vision-Pro", "Apple Vision Pro", 0xffffff, "Apple Vision"), + ]), + ); - expect(result).toBeInstanceOf(DriverCrashError); - expect((result as Error).message).toContain("no network"); + expect((await driver.listCatalog()).modelClasses).toEqual({ + "Apple TV 4K": "tv", + "Apple Vision Pro": "vision", + "Apple Watch Series 11 (46mm)": "watch", + "iPad Pro": "tablet", + "iPhone 17": "phone", + }); }); - it("fails with a non-downloadable RuntimeMissingError when Apple no longer offers the version", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { - result: { code: 70, stderr: "iOS 18.6 is not available for download.", stdout: "" }, - }), - ]); - const driver = await createDriver(runner); + it("keeps a device type with no product family, or one outside those five, in models with no modelClasses entry", async () => { + const driver = await createDriver( + classesRunner([ + deviceType("iPhone-17", "iPhone 17", 0xffffff, "iPhone"), + deviceType("Mystery", "Mystery Device"), + deviceType("HomePod", "HomePod", 0xffffff, "HomePod"), + ]), + ); - const result = await install(driver, "18.6").catch((error: unknown) => error); + const catalog = await driver.listCatalog(); - expect(result).toBeInstanceOf(RuntimeMissingError); - expect(result).toMatchObject({ downloadable: false }); + expect(catalog.models).toEqual(["iPhone 17", "Mystery Device", "HomePod"]); + expect(catalog.modelClasses).toEqual({ "iPhone 17": "phone" }); }); + }); - it("reports each percentage xcodebuild prints as progress, and nothing for a line without one", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { - stdoutLines: [ - "Downloading iOS 18.6 Simulator (22G86): 12.5% (1 GB of 8 GB)", - "Verifying download", - "Downloading iOS 18.6 Simulator (22G86): 100.0% (8 GB of 8 GB)", - ], - }), - listed(listFixtureAfterDownload), - imagesListed(imagesAfter), - ]); - const driver = await createDriver(runner); - const progress: unknown[] = []; - - await install(driver, "18.6", { onProgress: (report) => progress.push(report) }); + describe("model and runtime pairing", () => { + function pairingRunner(calls = 1): ScriptedProcessRunner { + return new ScriptedProcessRunner( + Array.from({ length: calls }, () => ({ + match: listInvocation, + result: { code: 0, stderr: "", stdout: modelRuntimesFixture }, + })), + ); + } - expect(progress).toEqual([ - { percent: 12.5, stage: "downloading" }, - { percent: 100, stage: "downloading" }, - ]); - }); + it("lists for each model exactly the installed runtimes that support its device type and fall in its version range", async () => { + const driver = await createDriver(pairingRunner()); - it("ends xcodebuild when the signal fires, and fails the install", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { hangs: true }), - ]); - const driver = await createDriver(runner); - const controller = new AbortController(); + const catalog = await driver.listCatalog(); - let error: unknown; - void install(driver, "18.6", { signal: controller.signal }).catch((caught: unknown) => { - error = caught; + expect(catalog.runtimes).toEqual(["18.4", "26.0", "26.0"]); + expect(catalog.modelRuntimes).toEqual({ + "iPhone 16": ["18.4", "26.0"], + "iPhone XS": ["18.4"], + "iPad Pro": ["26.0"], + "iPad mini": ["26.0"], + "iPhone 8": [], }); - for (let tick = 0; tick < 50 && runner.handles.length < 3; tick += 1) { - await Promise.resolve(); - } - expect(runner.handles).toHaveLength(3); - controller.abort(); - for (let tick = 0; tick < 50 && error === undefined; tick += 1) { - await Promise.resolve(); - } - - // why: Settled within a few microtasks of the signal: the kill ended it, not a timer. - expect(error).toBeInstanceOf(DriverCrashError); - expect((error as Error).message).toContain("was ended before it finished"); }); - it.each([ - [ - "an exact version", - { physical: false, model: "iPhone 7", osVersion: "13.0", platform: "ios" as const }, - "13.0", - ], - ["no version", { model: "iPhone 7", platform: "ios" as const }, "15"], - ])( - "reports a missing runtime below the download floor as not downloadable, for %s", - async (_case, request, named) => { - const runner = new ScriptedProcessRunner([listed(pairingFixture)]); - const driver = await createDriver(runner); - - const result = await driver.resolveSpec(request).catch((error: unknown) => error); - - expect(result).toBeInstanceOf(RuntimeMissingError); - expect(result).toMatchObject({ - component: undefined, - downloadable: false, - message: expect.stringContaining(`iOS ${named} predates`), - }); - }, - ); - - it("tells the requester to name an exact release when the download of a bare major fails", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18"], { - result: { code: 1, stderr: "no such build", stdout: "" }, - }), - ]); - const driver = await createDriver(runner); - - const result = await install(driver, "18").catch((error: unknown) => error); + it("resolveSpec accepts every listed pair and refuses every unlisted pair of a listed model and an installed runtime", async () => { + const models = ["iPhone 16", "iPhone XS", "iPad Pro", "iPad mini", "iPhone 8"]; + const installed = ["18.4", "26.0"]; + const driver = await createDriver(pairingRunner(1 + models.length * installed.length)); + const catalog = await driver.listCatalog(); + expect(catalog.models).toEqual(models); - expect(result).toBeInstanceOf(DriverCrashError); - expect((result as Error).message).toMatch(/tried 18.*no such build.*pass --os /); + let accepted = 0; + let refused = 0; + for (const model of models) { + for (const osVersion of installed) { + const resolution = driver.resolveSpec({ + model, + osVersion, + platform: "ios", + }); + if (catalog.modelRuntimes[model]?.includes(osVersion) === true) { + await expect(resolution).resolves.toEqual({ + physical: false, + model, + osVersion, + platform: "ios", + }); + accepted += 1; + } else { + // why: Every runtime here is installed, so the refusal is the range or the pairing + // check -- never a missing runtime, and never a download. + await expect(resolution).rejects.toThrow( + /is out of range|is installed but does not support/, + ); + refused += 1; + } + } + } + expect({ accepted, refused }).toEqual({ accepted: 5, refused: 5 }); }); - it("names the image whose build matches the runtime's when two images provide one runtime identifier", async () => { - const twoBuilds = JSON.stringify({ - devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, - runtimes: [ - { - buildversion: "23F79", - identifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-5", - isAvailable: true, - name: "iOS 26.5", - supportedDeviceTypes: [], - version: "26.5", - }, + it("pairs two device types that differ only in letter case through the first, as resolveSpec matches them", async () => { + const fixture = JSON.stringify({ + devicetypes: [ + deviceType("iPhone-16", "iPhone 16"), + deviceType("iphone-16-dup", "iphone 16"), ], + runtimes: [runtime("26.0", "23A339", true, ["iPhone-16"])], }); - const runner = new ScriptedProcessRunner([ - listed(twoBuilds), - imagesListed([ - { build: "23F77", id: "IMG-OLD", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, - { build: "23F79", id: "IMG-NEW", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-26-5" }, - ]), - ]); + const runner = new ScriptedProcessRunner( + Array.from({ length: 2 }, () => ({ + match: listInvocation, + result: { code: 0, stderr: "", stdout: fixture }, + })), + ); const driver = await createDriver(runner); - await expect(driver.findComponent("26.5")).resolves.toEqual({ - receipt: { build: "23F79", image: "IMG-NEW" }, - version: "26.5", - }); - }); + const catalog = await driver.listCatalog(); - it("kills xcodebuild with SIGKILL when it ignores the SIGTERM its signal sent", async () => { - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - xcodebuild(["-downloadPlatform", "iOS", "-buildVersion", "18.6"], { - hangs: true, - ignoresSigterm: true, + expect(catalog.models).toEqual(["iPhone 16", "iphone 16"]); + expect(catalog.modelRuntimes).toEqual({ "iPhone 16": ["26.0"], "iphone 16": ["26.0"] }); + await expect( + driver.resolveSpec({ + model: "iphone 16", + osVersion: "26.0", + platform: "ios", }), - ]); - const driver = await createDriver(runner, clock); - const controller = new AbortController(); - - let error: unknown; - void install(driver, "18.6", { signal: controller.signal }).catch((caught: unknown) => { - error = caught; - }); - for (let tick = 0; tick < 50 && runner.handles.length < 3; tick += 1) { - await Promise.resolve(); - } - controller.abort(); - for (let tick = 0; tick < 50; tick += 1) await Promise.resolve(); - expect(error).toBeUndefined(); - - clock.advance(10_000); - for (let tick = 0; tick < 50 && error === undefined; tick += 1) { - await Promise.resolve(); - } - expect(error).toBeInstanceOf(DriverCrashError); - }); - - it("finds an installed version with its receipt, and nothing for latest, a bare major or a missing version", async () => { - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(imagesBefore), - listed(listFixture), - imagesListed(imagesBefore), - listed(listFixture), - imagesListed(imagesBefore), - listed(listFixture), - imagesListed(imagesBefore), - ]); - const driver = await createDriver(runner); - - await expect(driver.findComponent("26.5")).resolves.toEqual({ - receipt: { build: "23F77", image: "IMG-265" }, - version: "26.5", - }); - await expect(driver.findComponent("latest")).resolves.toBeUndefined(); - await expect(driver.findComponent("26")).resolves.toBeUndefined(); - await expect(driver.findComponent("18.6")).resolves.toBeUndefined(); - expect(runner.calls.map((call) => call.command)).not.toContain("xcodebuild"); - }); - - it("states 8 GiB on the configured CoreSimulator volume as the footprint of one install", async () => { - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot: "/Users/agent/Library/Developer/CoreSimulator", - driverConfig: { deviceRoot }, - filesystem: new MemoryFilesystem(), - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - simlockHome: "/home/.simlock", - }); - - expect(driver.componentFootprint).toEqual({ - bytes: 8 * 1024 ** 3, - path: "/Users/agent/Library/Developer/CoreSimulator", - }); - }); - }); - - it("a provision that makes a new iOS seed boots the seed within ios.full.bootTimeoutMs", async () => { - for (const { full, waitsMs } of [ - { full: undefined, waitsMs: 300_000 }, - { full: 400_000, waitsMs: 400_000 }, - ]) { - const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - { match: sourcesSimctl("boot", seedUdid) }, - { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, - { match: sourcesSimctl("listapps", seedUdid) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` } }, - ]); - const filesystem = new MemoryFilesystem(); - const driver = await createDriver( - runner, - new FakeClock(), - filesystem, - undefined, - undefined, - full, - ); - await plantManagedDevice(filesystem); - - await driver.provision(spec); - - expect( - runner.calls.find((call) => call.args.includes("bootstatus"))?.options, - `key ${String(full)}`, - ).toEqual({ timeoutMs: waitsMs }); - } - }); - - it("provisions a copy of the kind's seed, making the seed first when there is none", async () => { - const copyUdid = "00000000-0000-0000-0000-000000000002"; - const database = await memorySourceDatabase(); - const rowsWhenCreated: unknown[] = []; - const seedCreate = sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ); - const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: seedCreate, - onSpawn: () => { - rowsWhenCreated.push( - ...database.prepare("SELECT state, udid FROM ios_source_data").all(), - ); - }, - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - { match: sourcesSimctl("boot", seedUdid) }, - { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, - { match: sourcesSimctl("listapps", seedUdid) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` } }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, - ]); - const filesystem = new ModeRecordingFilesystem(); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); - await plantManagedDevice(filesystem); - await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); - - await expect(driver.provision(spec)).resolves.toEqual({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - }); - expect( - rowsWhenCreated, - "the seed's row is making, with no simulator yet, when create runs", - ).toEqual([{ state: "making", udid: null }]); - expect( - filesystem.mkdirpModes.get(`${deviceRoot}/sources`), - "under deviceSet simlock the sources set is asked for private to the user", - ).toBe(0o700); - expect(runner.calls).toEqual([ - { ...listInvocation, options: { timeoutMs: 30_000 } }, - { ...seedCreate, options: { timeoutMs: 330_000 } }, - { ...sourcesSimctl("boot", seedUdid), options: { timeoutMs: 300_000 } }, - { ...sourcesSimctl("bootstatus", seedUdid, "-b"), options: { timeoutMs: 300_000 } }, - { ...sourcesSimctl("listapps", seedUdid), options: { timeoutMs: 30_000 } }, - { ...sourcesSimctl("shutdown", seedUdid), options: { timeoutMs: 30_000 } }, - { ...cloneInvocation, options: { timeoutMs: 330_000 } }, - ]); - - await expect(driver.provision(spec)).resolves.toMatchObject({ deviceId: copyUdid }); - expect(runner.calls.slice(7)).toEqual([ - { ...cloneInvocation, options: { timeoutMs: 330_000 } }, - ]); - }); - - it("a clone that fails makes the provision throw and keeps the seed and its row", async () => { - const database = await seededDatabase(); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 1, stderr: "No space left on device", stdout: "" }, - }, - ]); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); - - const error = await driver.provision(spec).catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(DriverCrashError); - expect((error as Error).message).toContain("No space left on device"); - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "clone"]); - expect(database.prepare("SELECT udid, state FROM ios_source_data").all()).toEqual([ - { state: "ready", udid: seedUdid }, - ]); - }); - - it("two provisions at once make one seed, and the clones of one seed run one at a time", async () => { - const copyUdid = "00000000-0000-0000-0000-000000000002"; - const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - { match: sourcesSimctl("boot", seedUdid) }, - { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, - { match: sourcesSimctl("listapps", seedUdid) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { hangs: true, match: cloneInvocation }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, - ]); - const filesystem = new MemoryFilesystem(); - const clock = new FakeClock(); - const driver = await createDriver(runner, clock, filesystem); - await plantManagedDevice(filesystem); - await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); - const clones = (): number => runner.calls.filter((call) => call.args[3] === "clone").length; - - const first = driver.provision(spec); - const second = driver.provision(spec); - const firstSettled = first.catch((caught: unknown) => caught); - await waitForCalls(runner, 8); - - expect(runner.calls.filter((call) => call.args[3] === "create")).toHaveLength(1); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - expect(clones()).toBe(1); - - clock.advance(30_000); - - expect(await firstSettled).toBeInstanceOf(DriverCrashError); - await expect(second).resolves.toMatchObject({ deviceId: copyUdid }); - expect(clones()).toBe(2); - }); - - it.each([ - { - call: "a provision's clone", - act: (driver: IosSimctlDriver) => driver.provision(spec), - seeded: true, - steps: [ - { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, - ], - }, - { - call: "a seed's create", - act: (driver: IosSimctlDriver) => driver.provision(spec), - seeded: false, - steps: [ - { - hangs: true, - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - }, - ], - }, - { - call: "a baseline's clone", - act: (driver: IosSimctlDriver) => - driver.saveSource({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - physical: false, - }), - seeded: false, - steps: [ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: simctl("shutdown", driverData.udid) }, - { - hangs: true, - match: simctl( - "clone", - driverData.udid, - "simlock-baseline-device-1", - `${deviceRoot}/sources`, - ), - }, - ], - }, - ])("$call waits ios.simulator.createTimeoutMs", async ({ act, seeded, steps }) => { - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - ...steps, - ]); - scopedRunners.push(runner); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const clock = new FakeClock(); - const config = { simulator: { audio: true, createTimeoutMs: 120_000, headless: true } }; - const driver = await IosSimctlDriver.create({ - clock, - config, - database: seeded ? await seededDatabase() : await memorySourceDatabase(), - driverConfig: { deviceRoot }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - const outcome = act(driver); - const settled = outcome.catch((error: unknown) => error); - const state = promiseState(outcome); - await waitForCalls(runner, steps.length + 1); - clock.advance(119_999); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - - expect(state.state).toBe("pending"); - clock.advance(1); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - expect(state.state).toBe("rejected"); - expect(await settled).toMatchObject({ - message: expect.stringMatching(/timed out after 120000ms$/), - }); - }); - - const coreSimulatorRoot = "/CoreSim"; - - it.each([ - { - call: "a provision's clone under deviceSet default", - act: (driver: IosSimctlDriver) => driver.provision(spec), - deleteArgs: (udid: string) => ["simctl", "delete", udid], - hung: [ - "simctl", - "--set", - "/home/.simlock/devices/ios-sources", - "clone", - seedUdid, - driverData.name, - `${coreSimulatorRoot}/Devices`, - ], - isDefault: true, - seeded: true, - steps: [], - }, - { - call: "a seed's create", - act: (driver: IosSimctlDriver) => driver.provision(spec), - deleteArgs: (udid: string) => sourcesSimctl("delete", udid).args, - hung: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ).args, - isDefault: false, - seeded: false, - steps: [], - }, - { - call: "a baseline's clone", - act: (driver: IosSimctlDriver) => - driver.saveSource({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - physical: false, - }), - deleteArgs: (udid: string) => sourcesSimctl("delete", udid).args, - hung: simctl("clone", driverData.udid, "simlock-baseline-device-1", `${deviceRoot}/sources`) - .args, - isDefault: false, - seeded: false, - steps: [ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: simctl("shutdown", driverData.udid) }, - ], - }, - ])( - "a create or clone that answers after its limit deletes the simulator it names in its own set ($call)", - async ({ act, deleteArgs, hung, isDefault, seeded, steps }) => { - const devicesDir = `${coreSimulatorRoot}/Devices`; - const lateUdid = "00000000-0000-0000-0000-0000000000c1"; - const list = isDefault - ? { args: ["simctl", "list", "-j", "devicetypes", "runtimes"], command: "xcrun" } - : listInvocation; - const scripted = new ScriptedProcessRunner([ - { match: list, result: { code: 0, stderr: "", stdout: listFixture } }, - ...steps, - { match: { args: deleteArgs(lateUdid), command: "xcrun" } }, - ]); - if (!isDefault) scopedRunners.push(scripted); - let answer: ((result: ProcessResult) => void) | undefined; - const runner: ProcessRunner = { - run: scripted.run.bind(scripted), - spawnStreaming: scripted.spawnStreaming.bind(scripted), - spawn( - command: string, - args: readonly string[], - options?: ProcessRunOptions, - ): ProcessHandle { - if (args.join(" ") !== hung.join(" ")) { - return scripted.spawn(command, args, options); - } - scripted.calls.push({ args: [...args], command, options: options ?? {} }); - // why: `simctl clone` asks CoreSimulator's service for the copy; a signal ends only the - // `simctl` process, so the copy still finishes, and it prints the UDID only if it is alive. - const result = new Promise((resolve) => { - answer = resolve; - }); - const empty = (async function* () {})(); - return { - kill() { - answer?.({ code: null, stderr: "", stdout: "" }); - }, - pid: 99, - stderr: empty, - stdout: empty, - unref() {}, - wait: () => result, - }; - }, - }; - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const clock = new FakeClock(); - const driver = await IosSimctlDriver.create({ - database: seeded ? await seededDatabase() : await memorySourceDatabase(), - clock, - ...(isDefault ? { coreSimulatorRoot } : {}), - driverConfig: isDefault ? { deviceSet: "default" } : { deviceRoot }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - const call = act(driver).catch((error: unknown) => error); - await vi.waitFor(() => expect(answer).toBeDefined()); - clock.advance(30_000); - const failure = await call; - await filesystem.mkdirp(`${devicesDir}/${lateUdid}/data`); - answer?.({ code: 0, stderr: "", stdout: `${lateUdid}\n` }); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - - expect(failure).toBeInstanceOf(DriverCrashError); - expect((failure as Error).message).toMatch(/timed out after 30000ms$/); - expect( - scripted.calls.filter((entry) => entry.args.includes("delete")).map((entry) => entry.args), - ).toEqual([deleteArgs(lateUdid)]); - }, - ); - - it("a create command still silent after the second limit is killed and logged with the name it was asked to create", async () => { - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, - ]); - const logger = new RecordingLogger(); - const clock = new FakeClock(); - const driver = await createDriver(runner, clock, undefined, await seededDatabase(), logger); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - const provision = driver.provision(spec).catch((error: unknown) => error); - await waitForCalls(runner, 2); - const clone = promiseState(runner.handles[1]!.wait()); - clock.advance(30_000); - await provision; - clock.advance(299_999); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - expect(clone.state).toBe("pending"); - expect(logger.warnings).toEqual([]); - - clock.advance(1); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - - expect(clone.state).toBe("fulfilled"); - expect(logger.warnings).toEqual([ - `simctl clone ${seedUdid} ${driverData.name} ${deviceRoot} gave no simulator after its limit`, - ]); - }); - - it("a seed is not deleted, even by a deletion queued during the clone, while a clone that answered late is still running", async () => { - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { match: sourcesSimctl("delete", seedUdid) }, - ]); - const clock = new FakeClock(); - const driver = await createDriver(runner, clock, undefined, await seededDatabase()); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - const [listed] = await driver.listSources(); - - const provision = driver.provision(spec).catch((error: unknown) => error); - await waitForCalls(runner, 3); - const destroyed = driver.destroySource(listed?.id ?? ""); - const destroy = promiseState(destroyed); - clock.advance(30_000); - await provision; - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - - expect(destroy.state).toBe("pending"); - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list", "clone"]); - - clock.advance(300_000); - await destroyed; - - expect(runner.calls.map((call) => call.args[3])).toEqual([ - "list", - "list", - "clone", - "shutdown", - "delete", - ]); - }); - - it("provision deletes the clone and throws when the LaunchServices database cannot be dropped", async () => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); - filesystem.failing = "rm"; - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - { match: simctl("delete", driverData.udid) }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - await expect(driver.provision(spec)).rejects.toThrow("disk full"); - - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "clone", "delete"]); - }); - - it.each([ - { name: "a named source is copied instead of the seed", id: "planted-named", cloned: true }, - { name: "an unknown source is refused", id: "nope", cloned: false }, - { name: "a stale source is refused", id: "planted-old", cloned: false }, - ])("provision with a source: $name", async ({ id, cloned }) => { - const namedUdid = "00000000-0000-0000-0000-0000000000bb"; - const oldUdid = "00000000-0000-0000-0000-0000000000cc"; - const database = await seededDatabase(); - const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; - const planted = plantSource(database, { ...kind, kind: "baseline", udid: namedUdid }); - database - .prepare("UPDATE ios_source_data SET source_id = 'planted-named' WHERE udid = ?") - .run(namedUdid); - plantSource(planted, { ...kind, kind: "baseline", runtimeBuild: "23F77", udid: oldUdid }); - database - .prepare("UPDATE ios_source_data SET source_id = 'planted-old' WHERE udid = ?") - .run(oldUdid); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); - const catalog = JSON.parse(listFixture) as { - runtimes: { buildversion?: string; identifier: string }[]; - }; - for (const runtime of catalog.runtimes) { - if (runtime.identifier === driverData.runtimeId) runtime.buildversion = "23F79"; - } - const listed = JSON.stringify(catalog); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listed } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listed } }, - { - match: sourcesSimctl("clone", namedUdid, driverData.name, deviceRoot), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); - - const provisioned = driver.provision(spec, { source: id }); - - if (cloned) { - await expect(provisioned).resolves.toMatchObject({ deviceId: driverData.udid }); - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list", "clone"]); - expect( - await launchServicesFiles(filesystem, `${deviceRoot}/${driverData.udid}`), - "the copy of a source loses the LaunchServices database that names the source's folder", - ).toEqual(["com.apple.LaunchServices.unregisteredtypes-501.db"]); - } else { - await expect(provisioned).rejects.toBeInstanceOf(DriverCrashError); - expect(runner.calls.map((call) => call.args[3])).not.toContain("clone"); - } - }); - - it.each([ - { name: "300 s old is kept", now: 300_000, destroyed: false }, - { name: "300.001 s old is destroyed", now: 300_001, destroyed: true }, - ])( - "the next seed make of a kind destroys this process's own making row of that kind older than 300 s: a row $name", - async ({ now, destroyed }) => { - const stuckUdid = "00000000-0000-0000-0000-0000000000dd"; - const database = plantSource(await memorySourceDatabase(), { - deviceTypeId: driverData.deviceTypeId, - maker: "device-1", - runtimeId: driverData.runtimeId, - state: "making", - udid: stuckUdid, - }); - const copyUdid = "00000000-0000-0000-0000-000000000002"; - const cloneInvocation = sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - ...(destroyed - ? [ - { match: sourcesSimctl("shutdown", stuckUdid) }, - { match: sourcesSimctl("delete", stuckUdid) }, - ] - : []), - { - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - { match: sourcesSimctl("boot", seedUdid) }, - { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, - { match: sourcesSimctl("listapps", seedUdid) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` } }, - ]); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); - const driver = await createDriver(runner, new FakeClock(now), filesystem, database); - - await driver.provision(spec); - - expect( - runner.calls.filter((call) => call.args[4] === stuckUdid).map((call) => call.args[3]), - ).toEqual(destroyed ? ["shutdown", "delete"] : []); - expect( - database.prepare("SELECT udid FROM ios_source_data WHERE state = 'making'").all(), - ).toEqual(destroyed ? [] : [{ udid: stuckUdid }]); - }, - ); - - it.each([ - { - name: "simctl create fails", - message: "Invalid runtime", - script: [{ code: 1, op: "create" }], - tail: [], - }, - { - name: "the seed's boot fails", - message: `seed ${seedUdid} did not boot: `, - script: [{ code: 1, op: "bootstatus" }], - tail: ["shutdown", "delete"], - }, - { - name: "the seed's shutdown fails", - message: "Unable to shutdown", - script: [{ code: 1, op: "shutdown" }], - tail: ["shutdown", "delete"], - }, - ] as const)( - "a seed make that fails because $name leaves no seed and no row, and the provision throws a DriverCrashError, never a BootTimeoutError", - async ({ message, script, tail }) => { - const failing = script[0]; - const steps = ["create", "boot", "bootstatus", "listapps", "shutdown"]; - const upTo = steps.indexOf(failing.op); - const results = new Map([ - ["create", { code: 0, stderr: "", stdout: `${seedUdid}\n` }], - ]); - const database = await memorySourceDatabase(); - const entries = steps.slice(0, upTo + 1).map((op) => { - const match = sourcesSimctl( - ...(op === "create" - ? ["create", "simlock-seed-device-1", driverData.deviceTypeId, driverData.runtimeId] - : op === "bootstatus" - ? [op, seedUdid, "-b"] - : [op, seedUdid]), - ); - if (op !== failing.op) - return { match, ...(results.has(op) ? { result: results.get(op)! } : {}) }; - return "hangs" in failing - ? { hangs: true, match } - : { match, result: { code: failing.code, stderr: message, stdout: "" } }; - }); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - ...entries, - ...tail.map((op) => ({ match: sourcesSimctl(op, seedUdid) })), - ]); - const clock = new FakeClock(); - const driver = await createDriver(runner, clock, undefined, database); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - const provisioned = driver.provision(spec); - void provisioned.catch(() => undefined); - if ("hangs" in failing) { - await waitForCalls(runner, entries.length + 1); - clock.advance(300_000); - } - const error = await provisioned.catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(DriverCrashError); - expect(error).not.toBeInstanceOf(BootTimeoutError); - expect((error as Error).message).toContain(message); - expect(runner.calls.map((call) => call.args[3])).toEqual([ - "list", - ...steps.slice(0, upTo + 1), - ...tail, - ]); - expect(runner.calls.slice(1).map((call) => call.args[2])).toEqual( - runner.calls.slice(1).map(() => `${deviceRoot}/sources`), - ); - expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ n: 0 }); - }, - ); - - it.each([ - { name: "bootstatus does not finish", hangs: "bootstatus" }, - { name: "the installer never answers listapps", hangs: "listapps" }, - ] as const)( - "a seed whose boot runs past ios.full.bootTimeoutMs because $name makes provision throw BootTimeoutError, as an Android seed's does, and leaves no seed and no row", - async ({ hangs }) => { - const bootSteps = ["boot", "bootstatus", "listapps"]; - const ran = bootSteps.slice(0, bootSteps.indexOf(hangs) + 1); - const database = await memorySourceDatabase(); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - ...ran.map((op) => { - const match = sourcesSimctl( - ...(op === "bootstatus" ? [op, seedUdid, "-b"] : [op, seedUdid]), - ); - return op === hangs ? { hangs: true, match } : { match }; - }), - ...["shutdown", "shutdown", "delete"].map((op) => ({ match: sourcesSimctl(op, seedUdid) })), - ]); - const clock = new FakeClock(); - const driver = await createDriver(runner, clock, undefined, database, undefined, 2_000); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - const provisioned = driver.provision(spec); - void provisioned.catch(() => undefined); - await waitForCalls(runner, ran.length + 2); - clock.advance(2_000); - const error = await provisioned.catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(BootTimeoutError); - expect((error as Error).message).toBe("Timed out waiting for device to boot: device-1"); - expect(runner.calls.map((call) => call.args[3])).toEqual([ - "list", - "create", - ...ran, - "shutdown", - "shutdown", - "delete", - ]); - expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ n: 0 }); - }, - ); - - it.each([ - { name: "its runtime changed build", seedBuild: "23F77", runtimeBuild: "23F79", remade: true }, - { name: "its runtime is unchanged", seedBuild: "23F79", runtimeBuild: "23F79", remade: false }, - { name: "it recorded no build", seedBuild: undefined, runtimeBuild: "23F79", remade: false }, - { - name: "its runtime lists no build now", - seedBuild: "23F77", - runtimeBuild: undefined, - remade: false, - }, - ])( - "makes the seed again before copying it only when it is stale: $name -> remade $remade", - async ({ seedBuild, runtimeBuild, remade }) => { - const staleUdid = "00000000-0000-0000-0000-0000000000bb"; - const catalog = JSON.parse(listFixture) as { - runtimes: { buildversion?: string; identifier: string }[]; - }; - for (const entry of catalog.runtimes) { - if (entry.identifier === driverData.runtimeId && runtimeBuild !== undefined) { - entry.buildversion = runtimeBuild; - } - } - const planted = staleUdid; - const cloneOf = (udid: string) => sourcesSimctl("clone", udid, driverData.name, deviceRoot); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: JSON.stringify(catalog) } }, - ...(remade - ? [ - { match: sourcesSimctl("shutdown", staleUdid) }, - { match: sourcesSimctl("delete", staleUdid) }, - { - match: sourcesSimctl( - "create", - "simlock-seed-device-1", - driverData.deviceTypeId, - driverData.runtimeId, - ), - result: { code: 0, stderr: "", stdout: `${seedUdid}\n` }, - }, - { match: sourcesSimctl("boot", seedUdid) }, - { match: sourcesSimctl("bootstatus", seedUdid, "-b") }, - { match: sourcesSimctl("listapps", seedUdid) }, - { match: sourcesSimctl("shutdown", seedUdid) }, - ] - : []), - { - match: cloneOf(remade ? seedUdid : staleUdid), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - ]); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const driver = await createDriver( - runner, - new FakeClock(), - filesystem, - await seededDatabase({ - udid: planted, - ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), - }), - ); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - await driver.provision(spec); - - expect(runner.calls.map((call) => call.args[3]).filter((verb) => verb !== "list")).toEqual( - remade - ? ["shutdown", "delete", "create", "boot", "bootstatus", "listapps", "shutdown", "clone"] - : ["clone"], - ); - }, - ); - - it.each([ - { name: "healthy", edit: () => undefined, reason: undefined }, - { - name: "its runtime is gone", - edit: (catalog: Catalog) => { - catalog.runtimes = catalog.runtimes.filter( - (entry) => entry.identifier !== driverData.runtimeId, - ); - }, - reason: `runtime ${driverData.runtimeId} is not installed`, - }, - { - name: "its runtime changed build", - edit: (catalog: Catalog) => { - for (const entry of catalog.runtimes) { - if (entry.identifier === driverData.runtimeId) entry.buildversion = "23F79"; - } - }, - reason: `runtime ${driverData.runtimeId} changed from build 23F77 to 23F79`, - }, - { - name: "its device type is gone", - edit: (catalog: Catalog) => { - catalog.devicetypes = catalog.devicetypes.filter( - (entry) => entry.identifier !== driverData.deviceTypeId, - ); - }, - reason: `device type ${driverData.deviceTypeId} is not available`, - }, - ])( - "lists a ready seed as a source with its model, version and size, and its staleReason when $name, which clears once it is back", - async ({ edit, reason }) => { - const catalogOf = (change: (catalog: Catalog) => void) => { - const catalog = JSON.parse(listFixture) as Catalog; - for (const entry of catalog.runtimes) { - if (entry.identifier === driverData.runtimeId) entry.buildversion = "23F77"; - } - change(catalog); - return JSON.stringify(catalog); - }; - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: catalogOf(edit) } }, - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: catalogOf(() => undefined) }, - }, - ]); - const database = plantSource(await memorySourceDatabase(), { - deviceTypeId: driverData.deviceTypeId, - runtimeBuild: "23F77", - runtimeId: driverData.runtimeId, - sizeBytes: 1_073_741_824, - udid: seedUdid, - }); - const driver = await createDriver(runner, new FakeClock(), undefined, database); - - expect(await driver.listSources()).toMatchObject([ - { - kind: "seed", - model: "iPhone 16", - osVersion: "26.5", - sizeBytes: 1_073_741_824, - ...(reason === undefined ? {} : { staleReason: reason }), - }, - ]); - expect((await driver.listSources())[0]).not.toHaveProperty("staleReason"); - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list"]); - }, - ); - - it("the first listSources of a process deletes another process's making rows and their simulators, keeps a row whose delete failed, and does not sweep again", async () => { - const [gone, stuck, ready] = [1, 2, 3].map( - (n) => `00000000-0000-0000-0000-0000000001${String(n)}0`, - ) as [string, string, string]; - const database = await memorySourceDatabase(); - const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; - plantSource(database, { ...kind, maker: "earlier-process", state: "making", udid: gone }); - plantSource(database, { ...kind, maker: "earlier-process", state: "making" }); - plantSource(database, { ...kind, maker: "earlier-process", state: "making", udid: stuck }); - plantSource(database, { ...kind, udid: ready }); - const runner = new ScriptedProcessRunner([ - { match: sourcesSimctl("shutdown", gone) }, - { match: sourcesSimctl("delete", gone) }, - { match: sourcesSimctl("shutdown", stuck) }, - { match: sourcesSimctl("delete", stuck), result: { code: 1, stderr: "busy", stdout: "" } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - ]); - const logger = new RecordingLogger(); - const driver = await createDriver(runner, new FakeClock(), undefined, database, logger); - - expect(await driver.listSources()).toHaveLength(1); - await driver.listSources(); - - expect(runner.calls.map((call) => [call.args[3], call.args[4]])).toEqual([ - ["shutdown", gone], - ["delete", gone], - ["shutdown", stuck], - ["delete", stuck], - ["list", "-j"], - ["list", "-j"], - ]); - expect(logger.warnings).toEqual([ - expect.stringContaining(`could not remove leftover source ${stuck}:`), - ]); - expect(database.prepare("SELECT udid, state FROM ios_source_data ORDER BY udid").all()).toEqual( - [ - { state: "making", udid: stuck }, - { state: "ready", udid: ready }, - ], - ); - }); - - it("listSources reports a simulator in the sources set that no row names as an orphan source, and destroySource deletes it", async () => { - const orphanUdid = "00000000-0000-0000-0000-0000000000ee"; - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - await filesystem.mkdirp(`${deviceRoot}/sources/${seedUdid}`); - await filesystem.mkdirp(`${deviceRoot}/sources/${orphanUdid}`); - const database = await seededDatabase(); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: sourcesSimctl("shutdown", orphanUdid) }, - { match: sourcesSimctl("delete", orphanUdid) }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); - - const sources = await driver.listSources(); - - expect(sources.filter((source) => source.orphan !== true)).toHaveLength(1); - expect(sources.filter((source) => source.orphan === true)).toEqual([ - { id: orphanUdid, kind: "seed", model: "unknown", orphan: true, osVersion: "unknown" }, - ]); - await driver.destroySource(orphanUdid); - expect(runner.calls.slice(1).map((call) => [call.args[3], call.args[4]])).toEqual([ - ["shutdown", orphanUdid], - ["delete", orphanUdid], - ]); - await expect( - driver.destroySource("00000000-0000-0000-0000-0000000000f0"), - ).rejects.toBeInstanceOf(DriverCrashError); - }); - - it.each([ - { - name: "deletes its simulator in the sources set and its row", - deleteCode: 0, - stderr: "", - rows: 0, - }, - { - name: "keeps the row and throws when the delete fails", - deleteCode: 1, - stderr: "busy", - rows: 1, - }, - { - name: "drops the row of a seed simctl no longer knows (Invalid device)", - deleteCode: 148, - stderr: `Invalid device: ${seedUdid}\n`, - rows: 0, - }, - ])("destroySource $name", async ({ deleteCode, stderr, rows }) => { - const database = plantSource(await memorySourceDatabase(), { - deviceTypeId: driverData.deviceTypeId, - runtimeId: driverData.runtimeId, - udid: seedUdid, - }); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: sourcesSimctl("shutdown", seedUdid) }, - { - match: sourcesSimctl("delete", seedUdid), - result: { code: deleteCode, stderr, stdout: "" }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), undefined, database); - const [listed] = await driver.listSources(); - - const destroyed = driver.destroySource(listed?.id ?? ""); - - if (rows === 0) await expect(destroyed).resolves.toBeUndefined(); - else await expect(destroyed).rejects.toBeInstanceOf(DriverCrashError); - expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "shutdown", "delete"]); - expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ - n: rows, - }); - }); - - it("destroySource refuses an id that names no source, without running simctl", async () => { - const runner = new ScriptedProcessRunner([]); - const driver = await createDriver(runner); - - await expect(driver.destroySource("nope")).rejects.toBeInstanceOf(DriverCrashError); - expect(runner.calls).toEqual([]); - }); - - describe("saveSource", () => { - const cloneUdid = "00000000-0000-0000-0000-0000000000bb"; - const baselineName = "simlock-baseline-device-1"; - const sourcesSet = `${deviceRoot}/sources`; - const device = { - address: driverData.udid, - deviceId: driverData.udid, - driverData, - physical: false, - } as const; - const cloneInvocation = simctl("clone", driverData.udid, baselineName, sourcesSet); - - it("keeps a shut-down copy of the device in the sources set as a ready baseline source with its disabled.plist, and a failing clone keeps no row", async () => { - const database = await memorySourceDatabase(); - const rowsAtShutdown: unknown[] = []; - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - await filesystem.mkdirp(overridesFolder(driverData.udid)); - await filesystem.writeFileAtomic( - overridesPath(driverData.udid), - overridesPlist({ "a.b": true }), - ); - await filesystem.mkdirp(`${sourcesSet}/${cloneUdid}/data`); - await filesystem.writeFileAtomic(`${sourcesSet}/${cloneUdid}/data/state`, "12345"); - await plantLaunchServices(filesystem, `${sourcesSet}/${cloneUdid}`); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: simctl("shutdown", driverData.udid), - onSpawn: () => { - rowsAtShutdown.push( - ...database.prepare("SELECT kind, state, udid FROM ios_source_data").all(), - ); - }, - }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${cloneUdid}\n` } }, - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); - - await expect(driver.saveSource(device)).resolves.toEqual({ - sizeBytes: 5, - sourceId: "device-1", - }); - - expect(rowsAtShutdown, "the source's row is making before the device shuts down").toEqual([ - { kind: "baseline", state: "making", udid: null }, - ]); - expect(runner.calls).toEqual([ - { ...listInvocation, options: { timeoutMs: 30_000 } }, - { ...simctl("shutdown", driverData.udid), options: { timeoutMs: 30_000 } }, - { ...cloneInvocation, options: { timeoutMs: 330_000 } }, - ]); - expect(await filesystem.readFile(overridesPath(cloneUdid))).toBe( - overridesPlist({ "a.b": true }), - ); - expect(await launchServicesFiles(filesystem, `${sourcesSet}/${cloneUdid}`)).toEqual([ - "com.apple.LaunchServices.unregisteredtypes-501.db", - ]); - await expect(driver.listSources()).resolves.toEqual([ - { - id: "device-1", - kind: "baseline", - model: "iPhone 16", - osVersion: "26.5", - sizeBytes: 5, - }, - ]); - - const failingRunner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: simctl("shutdown", driverData.udid) }, - { - match: cloneInvocation, - result: { code: 1, stderr: "No space left on device", stdout: "" }, - }, - ]); - const emptyDatabase = await memorySourceDatabase(); - const failing = await createDriver(failingRunner, new FakeClock(), filesystem, emptyDatabase); - await expect(failing.saveSource(device)).rejects.toMatchObject({ - message: expect.stringContaining("No space left on device"), - }); - expect(emptyDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); - - const stuckRunner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: simctl("shutdown", driverData.udid), - result: { code: 1, stderr: "Unable to shut down device", stdout: "" }, - }, - ]); - const stuckDatabase = await memorySourceDatabase(); - const stuck = await createDriver(stuckRunner, new FakeClock(), filesystem, stuckDatabase); - await expect(stuck.saveSource(device)).rejects.toMatchObject({ - message: expect.stringContaining("Unable to shut down device"), - }); - expect(stuckRunner.calls.map((call) => call.args[1])).not.toContain("clone"); - expect(stuckDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); - - const lateRunner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { match: simctl("shutdown", driverData.udid) }, - { match: cloneInvocation, result: { code: 0, stderr: "", stdout: `${cloneUdid}\n` } }, - { match: sourcesSimctl("shutdown", cloneUdid) }, - { match: sourcesSimctl("delete", cloneUdid) }, - ]); - const lateDatabase = await memorySourceDatabase(); - lateDatabase.exec( - "CREATE TRIGGER refuse_update BEFORE UPDATE ON ios_source_data BEGIN SELECT RAISE(ABORT, 'source row locked'); END", - ); - const late = await createDriver(lateRunner, new FakeClock(), filesystem, lateDatabase); - await expect(late.saveSource(device)).rejects.toMatchObject({ - message: expect.stringContaining("source row locked"), - }); - expect( - lateRunner.calls.slice(3).map((call) => call.args.at(-2)), - "the clone made before the failure is shut down and deleted from the sources set", - ).toEqual(["shutdown", "delete"]); - expect(lateDatabase.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); - }); - }); - - it.each([ - { - name: "copies the seed's disabled.plist to the new simulator", - seedHas: true, - failing: undefined, - }, - { name: "copies nothing when the seed has none", seedHas: false, failing: undefined }, - { - name: "logs a warning and goes on when the copy fails", - seedHas: true, - failing: "copyFile" as const, - }, - ])("a provision $name", async ({ seedHas, failing }) => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - if (seedHas) { - await filesystem.mkdirp(overridesFolder(seedUdid)); - await filesystem.writeFileAtomic(overridesPath(seedUdid), overridesPlist({ "a.b": true })); - } - await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); - filesystem.failing = failing; - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - ]); - const logger = new RecordingLogger(); - const driver = await IosSimctlDriver.create({ - database: await seededDatabase(), - clock: new FakeClock(), - driverConfig: { deviceRoot }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - launchdOverridesRoot: overridesRoot, - logger, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - scopedRunners.push(runner); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - await expect(driver.provision(spec)).resolves.toMatchObject({ deviceId: driverData.udid }); - - expect( - await launchServicesFiles(filesystem, `${deviceRoot}/${driverData.udid}`), - "a copy of a seed loses the LaunchServices database that names the seed's folder", - ).toEqual(["com.apple.LaunchServices.unregisteredtypes-501.db"]); - const copied = failing === undefined && seedHas; - expect(await filesystem.exists(overridesPath(driverData.udid))).toBe(copied); - if (copied) { - expect(await filesystem.readFile(overridesPath(driverData.udid))).toBe( - overridesPlist({ "a.b": true }), - ); - } - expect(logger.warnings).toEqual( - failing === undefined - ? [] - : [ - `could not copy the launchd overrides of seed ${seedUdid} to ${driverData.udid}: disk full`, - ], - ); - }); - - it("keeps no copy of a slim spec's mode in driver data", async () => { - const runner = new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: listFixture }, - }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - { - match: listDevicesInvocation, - result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, - }, - ]); - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); - await driver.resolveSpec({ - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - await expect(driver.provision({ ...spec, mode: "slim" })).resolves.toEqual({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - }); - }); - - it("surfaces simctl failures with stderr", async () => { - const runner = new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: listFixture }, - }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 1, stderr: "Invalid runtime", stdout: "" }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), undefined, await seededDatabase()); - await driver.resolveSpec({ - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - await expect(driver.provision(spec)).rejects.toEqual( - expect.objectContaining({ - message: expect.stringContaining("Invalid runtime"), - name: "DriverCrashError", - }), - ); - }); - - it("boots then checks bootstatus and installer readiness with bounded commands", async () => { - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", driverData.udid) }, - { match: simctl("bootstatus", driverData.udid, "-b") }, - { match: simctl("listapps", driverData.udid) }, - ]); - const driver = await createDriver(runner); - - await expect( - driver.makeReady( - { address: driverData.udid, deviceId: driverData.udid, driverData }, - prepareFull, - ), - ).resolves.toEqual({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - // why: Slim mode is off, so this boot slimmed nothing: the device is full. - mode: "full", - }); - expect(runner.calls).toEqual([ - { ...simctl("boot", driverData.udid), options: { timeoutMs: 300_000 } }, - { ...simctl("bootstatus", driverData.udid, "-b"), options: { timeoutMs: 300_000 } }, - { ...simctl("listapps", driverData.udid), options: { timeoutMs: 30_000 } }, - ]); - }); - - it.each([false, true])( - "keeps makeReady pending until an installer probe succeeds after a failure (hung: %s)", - async (hangs) => { - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", driverData.udid) }, - { match: simctl("bootstatus", driverData.udid, "-b") }, - { - match: simctl("listapps", driverData.udid), - result: { code: 2, stderr: "installer starting", stdout: "" }, - hangs, - }, - { match: simctl("listapps", driverData.udid) }, - ]); - const driver = await createDriver(runner, clock); - const ready = driver.makeReady( - { address: driverData.udid, deviceId: driverData.udid, driverData }, - prepareFull, - ); - const state = promiseState(ready); - await flushMicrotasks221(); - expect(state.state).toBe("pending"); - expect(runner.calls.at(-1)?.args).toEqual(simctlArgs("listapps", driverData.udid)); - if (hangs) { - clock.advance(30_000); - await flushMicrotasks221(); - expect(state.state).toBe("pending"); - } - clock.advance(1_000); - await flushMicrotasks221(); - expect(state.state).toBe("fulfilled"); - expect((await ready).deviceId).toBe(driverData.udid); - }, - ); - - it.each([false, true])( - "shares the bootstatus deadline with a %s-hanging installer probe and shuts down on expiry", - async (hangs) => { - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", driverData.udid) }, - { - match: simctl("bootstatus", driverData.udid, "-b"), - onSpawn: () => clock.advance(299_500), - }, - { - match: simctl("listapps", driverData.udid), - hangs, - result: { code: 2, stderr: "installer starting", stdout: "" }, - }, - { match: simctl("shutdown", driverData.udid) }, - ]); - const driver = await createDriver(runner, clock); - const ready = driver.makeReady( - { address: driverData.udid, deviceId: driverData.udid, driverData }, - prepareFull, - ); - const state = promiseState(ready); - await flushMicrotasks221(); - expect(state.state).toBe("pending"); - expect(runner.calls.at(-1)).toEqual({ - ...simctl("listapps", driverData.udid), - options: { timeoutMs: 500 }, - }); - clock.advance(499); - await flushMicrotasks221(); - expect(state.state).toBe("pending"); - clock.advance(1); - await flushMicrotasks221(); - expect(state.state).toBe("rejected"); - await expect(ready).rejects.toBeInstanceOf(BootTimeoutError); - expect(runner.calls.at(-1)?.args).toEqual(simctlArgs("shutdown", driverData.udid)); - }, - ); - - it.each(["bootstatus", "listapps"])( - "rejects readiness when a successful %s consumes the remaining budget", - async (slowCommand) => { - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", driverData.udid) }, - { - match: simctl("bootstatus", driverData.udid, "-b"), - onSpawn: () => { - if (slowCommand === "bootstatus") clock.advance(300_000); - }, - }, - ...(slowCommand === "listapps" - ? [{ match: simctl("listapps", driverData.udid), onSpawn: () => clock.advance(300_000) }] - : []), - { match: simctl("shutdown", driverData.udid) }, - ]); - const driver = await createDriver(runner, clock); - await expect( - driver.makeReady( - { address: driverData.udid, deviceId: driverData.udid, driverData }, - prepareFull, - ), - ).rejects.toBeInstanceOf(BootTimeoutError); - expect(runner.calls.map((call) => call.args)).toEqual([ - simctlArgs("boot", driverData.udid), - simctlArgs("bootstatus", driverData.udid, "-b"), - ...(slowCommand === "listapps" ? [simctlArgs("listapps", driverData.udid)] : []), - simctlArgs("shutdown", driverData.udid), - ]); - }, - ); - - it("times out bootstatus and issues a best-effort shutdown", async () => { - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", driverData.udid) }, - { hangs: true, match: simctl("bootstatus", driverData.udid, "-b") }, - { match: simctl("shutdown", driverData.udid) }, - ]); - const driver = await createDriver(runner, clock); - const ready = driver.makeReady( - { address: driverData.udid, deviceId: driverData.udid, driverData }, - prepareFull, - ); - // why: Handled up front so a failure below reports itself rather than surfacing as an - // unhandled rejection in whichever test happens to run next. - void ready.catch(() => undefined); - - await waitForCalls(runner, 2); - clock.advance(300_000); - - await expect(ready).rejects.toBeInstanceOf(BootTimeoutError); - expect(runner.calls.map((call) => call.args)).toEqual([ - simctlArgs("boot", driverData.udid), - simctlArgs("bootstatus", driverData.udid, "-b"), - simctlArgs("shutdown", driverData.udid), - ]); - }); - - it("throws virtual devices are not reclaimed for a simulator, from reclaim and reclaimStrategy", async () => { - const runner = new ScriptedProcessRunner([]); - const driver = await createDriver(runner); - const simulator = { address: driverData.udid, deviceId: driverData.udid, driverData }; - - await expect(driver.reclaim(simulator, { clean: "full" })).rejects.toThrow( - "virtual devices are not reclaimed", - ); - expect(() => driver.reclaimStrategy(simulator, { clean: "standard" })).toThrow( - "virtual devices are not reclaimed", - ); - expect(runner.calls).toEqual([]); - }); - - it("shuts down before deleting and uses benchmark estimates", async () => { - const runner = new ScriptedProcessRunner([ - { match: simctl("shutdown", driverData.udid) }, - { match: simctl("delete", driverData.udid) }, - ]); - const driver = await createDriver(runner); - - await driver.destroy({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - }); - - expect(runner.calls.map((call) => call.args)).toEqual([ - simctlArgs("shutdown", driverData.udid), - simctlArgs("delete", driverData.udid), - ]); - expect(driver.estimate({ operation: "provision" }, spec)).toBe(500); - expect(driver.estimate({ operation: "boot" }, spec)).toBe(60_000); - }); - - it.each([ - { name: "removal succeeds", method: "destroy" as const, failing: undefined, deleteCode: 0 }, - { - name: "rm throws, which is logged and does not fail the destroy", - method: "destroy" as const, - failing: "rm" as const, - deleteCode: 0, - }, - { - name: "simctl delete fails, so the folder is kept and the error thrown", - method: "destroy" as const, - failing: undefined, - deleteCode: 1, - }, - { - name: "destroyLegacy removes it too", - method: "destroyLegacy" as const, - failing: undefined, - deleteCode: 0, - }, - ])( - "removes the overrides folder of a destroyed simulator after simctl delete: $name", - async ({ method, failing, deleteCode }) => { - const udid = method === "destroy" ? driverData.udid : legacyUdid; - const args = (verb: string) => - method === "destroy" ? simctlArgs(verb, udid) : ["simctl", verb, udid]; - const runner = new ScriptedProcessRunner([ - { match: { args: args("shutdown"), command: "xcrun" } }, - { - match: { args: args("delete"), command: "xcrun" }, - result: { code: deleteCode, stderr: "no", stdout: "" }, - }, - ]); - const filesystem = new FaultyFilesystem(); - await filesystem.mkdirp(overridesFolder(udid)); - await filesystem.writeFileAtomic(overridesPath(udid), "x"); - await filesystem.mkdirp(overridesFolder("other-udid")); - filesystem.failing = failing; - const logger = new RecordingLogger(); - scopedRunners.push(runner); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - driverConfig: { deviceRoot }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - launchdOverridesRoot: overridesRoot, - logger, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - const device = { address: udid, deviceId: udid, driverData: { ...driverData, udid } }; - - const destroyed = driver[method](device); - - if (deleteCode === 0) { - await expect(destroyed).resolves.toBeUndefined(); - } else { - await expect(destroyed).rejects.toBeInstanceOf(DriverCrashError); - } - const removed = deleteCode === 0 && failing === undefined; - expect(await filesystem.exists(overridesFolder(udid))).toBe(!removed); - expect(await filesystem.exists(overridesFolder("other-udid"))).toBe(true); - expect(logger.warnings).toEqual( - failing === undefined - ? [] - : [`could not remove the launchd overrides folder of ${udid}: disk full`], - ); - }, - ); - - it("lists resolvable models and installed runtimes, defaulting to the newest", async () => { - const driver = await createDriver(scriptedListRunner()); - - // why: Strict: a catalog key an iOS driver does not fill (`images`, `customModels`) must be - // absent, not present as undefined. - await expect(driver.listCatalog()).resolves.toStrictEqual({ - defaultRuntime: "26.5", - modelAliases: {}, - modelClasses: {}, - modelRuntimes: { - "iPhone 15 Pro": ["18.4", "26.5"], - "iPhone 16": ["18.4", "26.5"], - "iPhone 17 Pro": ["18.4", "26.5"], - }, - models: ["iPhone 17 Pro", "iPhone 16", "iPhone 15 Pro"], - runtimes: ["18.4", "26.5"], - slimCategories: expect.any(Array), - }); - }); - - it("lists every slim category with none configured, and only the configured ones otherwise", async () => { - const unset = await createDriver(scriptedListRunner()); - const configured = await createSlimDriver(scriptedListRunner(), new MemoryFilesystem(), { - bootTimeoutMs: 1000, - categories: ["store"], - }); - - expect((await unset.listCatalog()).slimCategories).toEqual( - SLIM_CATEGORIES.map((category) => category.name), - ); - expect((await configured.listCatalog()).slimCategories).toEqual(["store"]); - }); - - describe("model classes", () => { - function classesRunner(deviceTypes: unknown[]): ScriptedProcessRunner { - return new ScriptedProcessRunner([ - { - match: listInvocation, - result: { - code: 0, - stderr: "", - stdout: JSON.stringify({ - devicetypes: deviceTypes, - runtimes: [runtime("26.0", "23A339", true, [])], - }), - }, - }, - ]); - } - - it("classes a device type by its product family: iPhone is phone, iPad is tablet, Apple Watch is watch, Apple TV is tv, Apple Vision is vision", async () => { - const driver = await createDriver( - classesRunner([ - deviceType("iPhone-17", "iPhone 17", 0xffffff, "iPhone"), - deviceType("iPad-Pro", "iPad Pro", 0xffffff, "iPad"), - deviceType("Watch-11", "Apple Watch Series 11 (46mm)", 0xffffff, "Apple Watch"), - deviceType("TV-4K", "Apple TV 4K", 0xffffff, "Apple TV"), - deviceType("Vision-Pro", "Apple Vision Pro", 0xffffff, "Apple Vision"), - ]), - ); - - expect((await driver.listCatalog()).modelClasses).toEqual({ - "Apple TV 4K": "tv", - "Apple Vision Pro": "vision", - "Apple Watch Series 11 (46mm)": "watch", - "iPad Pro": "tablet", - "iPhone 17": "phone", - }); - }); - - it("keeps a device type with no product family, or one outside those five, in models with no modelClasses entry", async () => { - const driver = await createDriver( - classesRunner([ - deviceType("iPhone-17", "iPhone 17", 0xffffff, "iPhone"), - deviceType("Mystery", "Mystery Device"), - deviceType("HomePod", "HomePod", 0xffffff, "HomePod"), - ]), - ); - - const catalog = await driver.listCatalog(); - - expect(catalog.models).toEqual(["iPhone 17", "Mystery Device", "HomePod"]); - expect(catalog.modelClasses).toEqual({ "iPhone 17": "phone" }); - }); - }); - - describe("model and runtime pairing", () => { - function pairingRunner(calls = 1): ScriptedProcessRunner { - return new ScriptedProcessRunner( - Array.from({ length: calls }, () => ({ - match: listInvocation, - result: { code: 0, stderr: "", stdout: modelRuntimesFixture }, - })), - ); - } - - it("lists for each model exactly the installed runtimes that support its device type and fall in its version range", async () => { - const driver = await createDriver(pairingRunner()); - - const catalog = await driver.listCatalog(); - - expect(catalog.runtimes).toEqual(["18.4", "26.0", "26.0"]); - expect(catalog.modelRuntimes).toEqual({ - "iPhone 16": ["18.4", "26.0"], - "iPhone XS": ["18.4"], - "iPad Pro": ["26.0"], - "iPad mini": ["26.0"], - "iPhone 8": [], - }); - }); - - it("resolveSpec accepts every listed pair and refuses every unlisted pair of a listed model and an installed runtime", async () => { - const models = ["iPhone 16", "iPhone XS", "iPad Pro", "iPad mini", "iPhone 8"]; - const installed = ["18.4", "26.0"]; - const driver = await createDriver(pairingRunner(1 + models.length * installed.length)); - const catalog = await driver.listCatalog(); - expect(catalog.models).toEqual(models); - - let accepted = 0; - let refused = 0; - for (const model of models) { - for (const osVersion of installed) { - const resolution = driver.resolveSpec({ - model, - osVersion, - platform: "ios", - }); - if (catalog.modelRuntimes[model]?.includes(osVersion) === true) { - await expect(resolution).resolves.toEqual({ - physical: false, - model, - osVersion, - platform: "ios", - }); - accepted += 1; - } else { - // why: Every runtime here is installed, so the refusal is the range or the pairing - // check -- never a missing runtime, and never a download. - await expect(resolution).rejects.toThrow( - /is out of range|is installed but does not support/, - ); - refused += 1; - } - } - } - expect({ accepted, refused }).toEqual({ accepted: 5, refused: 5 }); - }); - - it("pairs two device types that differ only in letter case through the first, as resolveSpec matches them", async () => { - const fixture = JSON.stringify({ - devicetypes: [ - deviceType("iPhone-16", "iPhone 16"), - deviceType("iphone-16-dup", "iphone 16"), - ], - runtimes: [runtime("26.0", "23A339", true, ["iPhone-16"])], - }); - const runner = new ScriptedProcessRunner( - Array.from({ length: 2 }, () => ({ - match: listInvocation, - result: { code: 0, stderr: "", stdout: fixture }, - })), - ); - const driver = await createDriver(runner); - - const catalog = await driver.listCatalog(); - - expect(catalog.models).toEqual(["iPhone 16", "iphone 16"]); - expect(catalog.modelRuntimes).toEqual({ "iPhone 16": ["26.0"], "iphone 16": ["26.0"] }); - await expect( - driver.resolveSpec({ - model: "iphone 16", - osVersion: "26.0", - platform: "ios", - }), - ).resolves.toEqual({ - physical: false, - model: "iPhone 16", - osVersion: "26.0", - platform: "ios", - }); - }); - }); - - it("shells out to simctl exactly once per listCatalog call", async () => { - const runner = scriptedListRunner(); - const driver = await createDriver(runner); - - await driver.listCatalog(); - - expect(runner.calls).toEqual([{ ...listInvocation, options: { timeoutMs: 30_000 } }]); - }); - - it("reports every device in its set, whatever the device is named", async () => { - const runner = new ScriptedProcessRunner([ - { match: listDevicesInvocation, result: { code: 0, stderr: "", stdout: listDevicesFixture } }, - ]); - const driver = await createDriver(runner); - - const reality = await driver.listManaged(); - - // why: The fixture's last device carries no `simlock-` prefix. Membership in the set is the - // ownership proof (safety rule 8), so it is Simlock's like every other entry here. - expect( - reality.devices.map((device) => ({ deviceId: device.deviceId, runState: device.runState })), - ).toEqual([ - { deviceId: "00000000-0000-0000-0000-000000000101", runState: "running" }, - { deviceId: "00000000-0000-0000-0000-000000000102", runState: "stopped" }, - { deviceId: "00000000-0000-0000-0000-000000000103", runState: "transitioning" }, - { deviceId: "00000000-0000-0000-0000-000000000104", runState: "transitioning" }, - { deviceId: "00000000-0000-0000-0000-000000000105", runState: "running" }, - ]); - }); - - it("populates processes from booted managed devices only", async () => { - const runner = new ScriptedProcessRunner([ - { match: listDevicesInvocation, result: { code: 0, stderr: "", stdout: listDevicesFixture } }, - ]); - const driver = await createDriver(runner); - - const reality = await driver.listManaged(); - - expect(reality.processes.map((device) => device.deviceId)).toEqual([ - "00000000-0000-0000-0000-000000000101", - "00000000-0000-0000-0000-000000000105", - ]); - }); - - it("leaves neither provenance mark when the erasable half cannot be written", async () => { - const filesystem = new MemoryFilesystem(); - const runner = new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: listFixture }, - }, - { - match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), - result: { code: 0, stderr: "", stdout: `${driverData.udid}\n` }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem, await seededDatabase()); - // why: The device directory exists, its data container does not, and `writeFileAtomic` - // creates no parents -- the erasable write cannot land. - await filesystem.mkdirp(`${deviceRoot}/${driverData.udid}`); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - - await expect(driver.provision(spec)).rejects.toThrow( - `ENOENT: no such file or directory, open '${dataPath}'`, - ); - - // why: A durable mark standing alone is exactly what `Doctor` reads as a foreign erase, so - // a half-written pair would have `doctor` accusing the user of erasing the device - // Simlock itself just erased. Neither half is what "never marked" looks like. - expect(await filesystem.exists(durableMarkPath)).toBe(false); - expect(await filesystem.exists(erasableMarkPath)).toBe(false); - }); - - it("reports matching provenance tokens for a healthy managed device", async () => { - const filesystem = new MemoryFilesystem(); - const runner = new ScriptedProcessRunner([ - { - match: listDevicesInvocation, - result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - await plantManagedDevice(filesystem); - await filesystem.writeFileAtomic(durableMarkPath, JSON.stringify({ token: "tok-1" })); - await filesystem.writeFileAtomic(erasableMarkPath, JSON.stringify({ token: "tok-1" })); - - const reality = await driver.listManaged(); - - expect(reality.devices).toEqual([ - expect.objectContaining({ - mark: { durable: "tok-1", erasable: "tok-1", erasableReadable: true }, - }), - ]); - }); - - it("reports erasable undefined when the data-container mark is gone (foreign erase)", async () => { - const filesystem = new MemoryFilesystem(); - const runner = new ScriptedProcessRunner([ - { - match: listDevicesInvocation, - result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - await plantManagedDevice(filesystem); - await filesystem.writeFileAtomic(durableMarkPath, JSON.stringify({ token: "tok-1" })); - - const reality = await driver.listManaged(); - - expect(reality.devices).toEqual([ - expect.objectContaining({ - mark: { durable: "tok-1", erasable: undefined, erasableReadable: true }, - }), - ]); - }); - - it("reports mark undefined when both provenance regions are absent (pre-upgrade device)", async () => { - const filesystem = new MemoryFilesystem(); - const runner = new ScriptedProcessRunner([ - { - match: listDevicesInvocation, - result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - await plantManagedDevice(filesystem); - - const reality = await driver.listManaged(); - - expect(reality.devices[0]?.mark).toBeUndefined(); - }); - - it("reads a corrupt mark file as undefined without throwing", async () => { - const filesystem = new MemoryFilesystem(); - const runner = new ScriptedProcessRunner([ - { - match: listDevicesInvocation, - result: { code: 0, stderr: "", stdout: deviceListResponse("Shutdown") }, - }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - await plantManagedDevice(filesystem); - await filesystem.writeFileAtomic(durableMarkPath, "not json"); - await filesystem.writeFileAtomic(erasableMarkPath, JSON.stringify({ notAToken: true })); - - const reality = await driver.listManaged(); - - expect(reality.devices[0]?.mark).toBeUndefined(); - }); - - it("creates and marks its device root under SIMLOCK_HOME when none is configured", async () => { - const filesystem = new MemoryFilesystem(); - - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - driverConfig: {}, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - simlockHome: "/home/.simlock", - }); - - expect(driver.deviceRoot).toBe("/home/.simlock/devices/ios"); - await expect( - filesystem.exists(`/home/.simlock/devices/ios/${OWNED_ROOT_MARKER_FILE}`), - ).resolves.toBe(true); - }); - - it("does not start when its device root belongs to another Simlock instance", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(deviceRoot); - await filesystem.writeFileAtomic( - `${deviceRoot}/${OWNED_ROOT_MARKER_FILE}`, - JSON.stringify({ - instanceId: "someone-else", - owner: "simlock", - platform: "ios", - schemaVersion: 1, - }), - ); - - const failure = await createDriver(new ScriptedProcessRunner([]), new FakeClock(), filesystem) - .then(() => undefined) - .catch((error: unknown) => error); - - // why: Fails closed: no fallback to the machine's default device set (safety rule 9). - expect(failure).toBeInstanceOf(OwnedRootError); - expect(failure).toMatchObject({ platform: "ios", reason: "wrong-instance" }); - }); - - it("refuses the platform, not the daemon, when the configured device root is not a path", async () => { - const failure = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - driverConfig: { deviceRoot: true }, - filesystem: new MemoryFilesystem(), - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - simlockHome: "/home/.simlock", - }) - .then(() => undefined) - .catch((error: unknown) => error); - - // why: An `OwnedRootError` costs iOS alone; anything else takes the daemon down with it, - // and with it every way of finding out why -- `"deviceRoot": true` and - // `"deviceRoot": "devices/ios"` must not differ by that much. - expect(failure).toBeInstanceOf(OwnedRootError); - expect(failure).toMatchObject({ platform: "ios", reason: "not-absolute" }); - expect(failure).toMatchObject({ message: expect.stringContaining("true") }); - }); - - it("re-proves an intact root by re-running the validation its start was judged by", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(driver.revalidateRoot()).resolves.toBeUndefined(); - }); - - it("refuses to re-prove a root that has become a symlink since startup", async () => { - const filesystem = new MemoryFilesystem(); - const driver = await createDriver(new ScriptedProcessRunner([]), new FakeClock(), filesystem); - // why: The case the re-proof exists for: ownership was proven at startup and the path has - // been swapped since, so `listManaged` now answers from somewhere else entirely. - filesystem.defineSymlink(deviceRoot, "/Users/someone/Library/Developer/CoreSimulator/Devices"); - - const failure = await driver - .revalidateRoot() - .then(() => undefined) - .catch((error: unknown) => error); - - expect(failure).toBeInstanceOf(OwnedRootError); - expect(failure).toMatchObject({ platform: "ios", reason: "symlink" }); - }); - - it("finds a pre-root device in the machine's default set, without --set", async () => { - const runner = new ScriptedProcessRunner([ - { - match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, - result: { - code: 0, - stderr: "", - stdout: JSON.stringify({ - devices: { - "com.apple.CoreSimulator.SimRuntime.iOS-26-5": [ - { - dataPath: `/Library/Devices/${legacyUdid}/data`, - name: "simlock-old", - state: "Shutdown", - udid: legacyUdid, - }, - ], - }, - }), - }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.findLegacy(legacyUdid)).resolves.toMatchObject({ - device: { deviceId: legacyUdid }, - path: `/Library/Devices/${legacyUdid}`, - }); - }); - - it("reports no pre-root device when the default set does not hold the udid", async () => { - const runner = new ScriptedProcessRunner([ - { - match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, - result: { code: 0, stderr: "", stdout: JSON.stringify({ devices: {} }) }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.findLegacy(legacyUdid)).resolves.toBeUndefined(); - }); - - it("destroys a pre-root device through the unscoped path it actually lives on", async () => { - const runner = new ScriptedProcessRunner([ - { match: { args: ["simctl", "shutdown", legacyUdid], command: "xcrun" } }, - { match: { args: ["simctl", "delete", legacyUdid], command: "xcrun" } }, - ]); - const driver = await createDriver(runner); - - await driver.destroyLegacy({ - address: legacyUdid, - deviceId: legacyUdid, - driverData: { ...driverData, udid: legacyUdid }, - }); - - expect(runner.calls.map((call) => call.args)).toEqual([ - ["simctl", "shutdown", legacyUdid], - ["simctl", "delete", legacyUdid], - ]); - }); - - it("answers uninstall for the reclaim strategy of a physical device", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "standard" })).toBe("uninstall"); - expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "full" })).toBe("uninstall"); - }); - - it("scopes a simctl passthrough to the device set the way its own calls are scoped", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, ["install", "booted", "./MyApp.app"])).resolves.toEqual({ - args: ["simctl", "--set", deviceRoot, "install", "booted", "./MyApp.app"], - command: "xcrun", - env: {}, - }); - }); - - it("fills a simctl device verb's missing device with the lease's UDID", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - const udid = "6F1D2A8C-3B4E-4F7A-9C2D-8E5B1A7F3C90"; - const simulator = { address: "", deviceId: udid, driverData: {}, physical: false } as const; - const scoped = ["simctl", "--set", deviceRoot]; - const cases: readonly (readonly [readonly string[], readonly string[]])[] = [ - [ - ["install", "/b/App.app"], - ["install", udid, "/b/App.app"], - ], - [ - ["launch", "--console", "com.x"], - ["launch", "--console", udid, "com.x"], - ], - [ - ["launch", "-a", "arm64", "com.x"], - ["launch", "-a", "arm64", udid, "com.x"], - ], - [ - ["spawn", "-s", "launchctl", "list"], - ["spawn", "-s", udid, "launchctl", "list"], - ], - [ - ["io", "screenshot", "/tmp/s.png"], - ["io", udid, "screenshot", "/tmp/s.png"], - ], - [ - ["install", udid, "/b/App.app"], - ["install", udid, "/b/App.app"], - ], - [ - ["install", udid.toLowerCase(), "/b/App.app"], - ["install", udid.toLowerCase(), "/b/App.app"], - ], - [ - ["list", "devices"], - ["list", "devices"], - ], - [ - ["runtime", "list"], - ["runtime", "list"], - ], - ]; - - for (const [args, expected] of cases) { - await expect( - driver.passthrough("simctl", simulator, args, { hasTerminal: false, platform: "ios" }), - ).resolves.toEqual({ args: [...scoped, ...expected], command: "xcrun", env: {} }); - } - }); - - it("refuses a simctl command that names another device, or that runs on another platform", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - const udid = "6F1D2A8C-3B4E-4F7A-9C2D-8E5B1A7F3C90"; - const other = "0A1B2C3D-0000-4000-8000-123456789ABC"; - const simulator = { address: "", deviceId: udid, driverData: {}, physical: false } as const; - const refusal = (command: string, named: string) => - `Refusing \`simctl ${command}\`: it names device ${named}, and this lease holds ${udid}. Leave the device out; Simlock fills it in.`; - const cases: readonly (readonly [readonly string[], string])[] = [ - [["install", other, "/b/App.app"], refusal(`install ${other} /b/App.app`, other)], - [["install", "booted", "/b/App.app"], refusal("install booted /b/App.app", "booted")], - [ - ["launch", "--console", "booted", "com.x"], - refusal("launch --console booted com.x", "booted"), - ], - ]; - - for (const [args, message] of cases) { - await expect( - driver.passthrough("simctl", simulator, args, { hasTerminal: false, platform: "ios" }), - ).rejects.toMatchObject({ message, tool: "simctl" }); - } - await expect( - driver.passthrough("simctl", simulator, ["install", "/b/App.app"], { - hasTerminal: false, - platform: "android", - }), - ).rejects.toMatchObject({ - message: - "Refusing `simctl`: it only serves an iOS simulator, and this lease is on the android platform.", - tool: "simctl", - }); - }); - - it.each([["create"], ["erase"], ["delete"]])( - "refuses to proxy %s, naming the command that reclaims a device properly", - async (verb) => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow( - PassthroughRefusedError, - ); - await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow(/simlock release/); - await expect(simctlPassthrough(driver, [verb, "ABCD"])).rejects.toThrow(/simlock cleanup/); - }, - ); - - it("finds the refused verb past a leading flag rather than only in first position", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, ["--verbose", "delete", "ABCD"])).rejects.toThrow( - PassthroughRefusedError, - ); - }); - - it("proxies a refused word that is an argument rather than the subcommand", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - expect((await simctlPassthrough(driver, ["spawn", "booted", "log", "erase"])).args).toContain( - "erase", - ); - }); - - it.each([ - [["--profiles", "/tmp", "erase", "all"]], - [["--set", "/tmp", "delete", "all"]], - [["--set=/tmp", "create", "sim"]], - ])( - "refuses a caller-supplied scoping flag rather than reading its value as the subcommand: %j", - async (args) => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - // why: The bug this pins was not the flag itself: the flag's separated value was found as - // the subcommand, so the refused verb behind it was never seen, and the wrapper - // answered with `--set ` prepended -- Simlock handing over the very path - // that contains every other agent's devices. - await expect(simctlPassthrough(driver, args)).rejects.toThrow(PassthroughRefusedError); - await expect(simctlPassthrough(driver, args)).rejects.toThrow( - /supplies the device set itself/, - ); - }, - ); - - it("refuses any unrecognized flag ahead of the subcommand, rather than skipping over it as presumed harmless", async () => { - // why: Round 4, F6: structurally the same escape round 3 closed on the Android side - // (`--reply-fd 9 -H attacker.example`). Before this fix, an unrecognized `-...` token was - // treated as a no-value flag and the scan continued to the next argument -- so a - // caller-supplied global's *value* was read as the subcommand, and the real - // `--set`/`--profiles` behind it was never seen by `#assertProxyable` at all: this exact - // line used to stop the scan at `/tmp` (mistaken for the subcommand), so the trailing - // `--set /evil` was never scanned for refusal and rode through to `xcrun simctl` unchecked. - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect( - simctlPassthrough(driver, ["-x", "/tmp", "--set", "/evil", "boot", "UDID"]), - ).rejects.toThrow(PassthroughRefusedError); - }); - - it("refuses to shut down every device in the set at once", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, ["shutdown", "all"])).rejects.toThrow( - PassthroughRefusedError, - ); - await expect(simctlPassthrough(driver, ["shutdown", "all"])).rejects.toThrow(/simlock release/); - }); - - it("still proxies shutting a single device down by udid", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - expect((await simctlPassthrough(driver, ["shutdown", "ABCD"])).args).toEqual([ - "simctl", - "--set", - deviceRoot, - "shutdown", - "ABCD", - ]); - }); - - it("refuses to delete a runtime shared with Xcode and other tools", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( - PassthroughRefusedError, - ); - await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( - /shared with Xcode and other tools/, - ); - await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( - /through Xcode/, - ); - }); - - it("does not say in the runtime delete refusal that iOS runtimes cannot be downloaded", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - let message = ""; - try { - await simctlPassthrough(driver, ["runtime", "delete", "26.5"]); - } catch (error) { - message = (error as Error).message; - } - - expect(message).toMatch(/shared with Xcode/); - expect(message).not.toMatch(/download|install/i); - }); - - it("points the runtime delete refusal at simlock component remove", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(simctlPassthrough(driver, ["runtime", "delete", "26.5"])).rejects.toThrow( - /use `simlock component remove ios `/, - ); - }); - - it("still proxies the runtime operations that only read", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - expect((await simctlPassthrough(driver, ["runtime", "list"])).args).toEqual([ - "simctl", - "--set", - deviceRoot, - "runtime", - "list", - ]); - }); - - it.skipIf(process.env.SIMLOCK_LIVE_IOS !== "1")( - "runs a provision-to-destroy smoke test against simctl", - async () => { - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new SystemClock(), - driverConfig: {}, - filesystem: new NodeFilesystem(), - idGenerator: { generate: () => `test-${process.pid}` }, - instanceId: `live-${process.pid}`, - processRunner: new NodeProcessRunner(), - simlockHome: join(tmpdir(), `simlock-live-ios-${process.pid}`), - }); - let device: Awaited> | undefined; - - try { - const liveSpec = await driver.resolveSpec({ model: "iPhone 17 Pro", platform: "ios" }); - device = await driver.provision(liveSpec); - await driver.makeReady(device, prepareFull); - } finally { - if (device !== undefined) { - await driver.destroy(device); - } - } - }, - 150_000, - ); - - describe("slim mode", () => { - const widgetsLabels = widgetsLabelList(); - const widgetsSignature = "v1:0dd8c925463fb678"; - const slim18_5 = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-5" }; - - const managedCount = Object.keys(wantedOverrides()).length; - const trueLabel = widgetsLabels[0] ?? ""; - const falseLabel = "com.apple.siriactionsd"; - // why: the label written first is not the first in sorted order, so naming the sorted first - // differs from naming the first one the readback check meets. - const firstWritten = Object.keys(wantedOverrides())[0] ?? ""; - const firstSorted = Object.keys(wantedOverrides()).sort()[0] ?? ""; - function flipped(labels: readonly string[]): Record { - return Object.fromEntries( - labels.map((label) => [label, wantedOverrides()[label] === true ? "enabled" : "disabled"]), - ); - } - - function slimOptions(bootTimeoutMs = 300_000) { - return { bootTimeoutMs, categories: ["widgets"] } as const; - } - - it("parses simctl runtime ids into [major, minor], including edge cases", () => { - expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18-5")).toEqual([ - 18, 5, - ]); - expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-26-0")).toEqual([ - 26, 0, - ]); - expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18")).toEqual([18, 0]); - expect(iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-18-5-1")).toEqual([ - 18, 5, - ]); - expect(iosRuntimeVersionFromId("")).toBeUndefined(); - expect(iosRuntimeVersionFromId("garbage")).toBeUndefined(); - }); - - it("parses a device-specific runtime id (iOS-27-2-iPhone19-4) to the version it names (#1064)", () => { - // why: Xcode 27.2 installs `com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4` next to - // `...iOS-27-2`; the suffix after the version names a device family, not a version part. - expect( - iosRuntimeVersionFromId("com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4"), - ).toEqual([27, 2]); - }); - - it("gates persistent slim on iOS 18.5+ without falling into string comparison", () => { - // why: A naive string/lexicographic compare would put "9.0" ahead of "18.5" -- the gate must - // compare the parsed integers instead. - expect(supportsPersistentSlim([18, 5])).toBe(true); - expect(supportsPersistentSlim([26, 0])).toBe(true); - expect(supportsPersistentSlim([9, 0])).toBe(false); - expect(supportsPersistentSlim([18, 4])).toBe(false); - expect(supportsPersistentSlim(undefined)).toBe(false); - }); - - it.each([ - { name: "no file yet", existing: undefined, expected: wantedOverrides(), rewritten: true }, - { - name: "an unknown category beside a known one, which is reported", - existing: undefined, - expected: wantedOverrides(), - rewritten: true, - configured: ["widgets", "nonsense"], - unknown: ["nonsense"], - }, - { - name: "a file with the runtime's own entries, which are kept", - existing: { "com.apple.runtime.own": true, "com.example.other": false }, - expected: { - ...wantedOverrides(), - "com.apple.runtime.own": true, - "com.example.other": false, - }, - rewritten: true, - }, - { - name: "a file already as wanted, which is not rewritten", - existing: wantedOverrides(), - expected: wantedOverrides(), - rewritten: false, - }, - { - name: "a file slimmed under wider categories, whose dropped labels are now off the list", - existing: Object.fromEntries( - Object.entries(wantedOverrides()).map(([label]) => [ - label, - !neverDisabled.includes(label), - ]), - ), - expected: wantedOverrides(), - rewritten: true, - }, - ])( - "writes the launchd overrides before the one boot and grants slim after the readback: $name", - async ({ existing, expected, rewritten, configured, unknown }) => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - await plantOverrides(filesystem, existing); - const clock = new FakeClock(); - const { runner, fileAtBoot } = runnerReadingFileAtBoot(filesystem, clock); - const onSlimmed = vi.fn(); - const onSlimSkipped = vi.fn(); - const driver = await createSlimDriver( - runner, - filesystem, - { bootTimeoutMs: 300_000, categories: configured ?? ["widgets"] }, - onSlimmed, - onSlimSkipped, - false, - clock, - ); - - const result = await driver.makeReady( - { address: slim18_5.udid, deviceId: slim18_5.udid, driverData: slim18_5 }, - prepareSlim, - ); - - expect(result).toEqual({ - address: slim18_5.udid, - deviceId: slim18_5.udid, - driverData: slim18_5, - mode: "slim", - }); - expect(parseOverrides(fileAtBoot() ?? "")).toEqual(expected); - expect(filesystem.writes).toEqual(rewritten ? [overridesPath(slim18_5.udid)] : []); - expect(filesystem.modes).toEqual(rewritten ? { file: 0o644, folder: 0o700 } : {}); - expect(runner.calls.map((call) => call.args)).toEqual([ - simctlArgs("boot", slim18_5.udid), - simctlArgs("bootstatus", slim18_5.udid, "-b"), - simctlArgs("listapps", slim18_5.udid), - simctlArgs("spawn", slim18_5.udid, "launchctl", "print-disabled", "system"), - ]); - expect(runner.calls[1]?.options).toEqual({ timeoutMs: 260_000 }); - expect(runner.calls[3]?.options).toEqual({ timeoutMs: 120_000 }); - expect(onSlimmed).toHaveBeenCalledTimes(1); - expect(onSlimmed).toHaveBeenCalledWith({ - address: slim18_5.udid, - categories: ["widgets"], - deviceId: slim18_5.udid, - durationMs: 40_000, - labelCount: widgetsLabels.length, - signature: widgetsSignature, - unknownLabels: unknown ?? [], - }); - expect(onSlimSkipped).not.toHaveBeenCalled(); - }, - ); - - it.each([ - { - name: "a true label reads enabled", - readback: () => readbackWith({ [trueLabel]: "enabled" }), - mode: "full", - detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, - }, - { - name: "a true label is missing", - readback: () => readbackWith({ [trueLabel]: undefined }), - mode: "full", - detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, - }, - { - name: "a false label reads disabled", - readback: () => readbackWith({ [falseLabel]: "disabled" }), - mode: "full", - detail: `1 of ${managedCount} managed labels did not read back as written at boot; first: ${falseLabel}`, - }, - { - name: "two labels disagree, the first by sort order is named", - readback: () => readbackWith(flipped([firstWritten, firstSorted])), - mode: "full", - detail: `2 of ${managedCount} managed labels did not read back as written at boot; first: ${firstSorted}`, - }, - { - name: "a true label reads true (older launchd)", - readback: () => readbackWith({ [trueLabel]: "true" }), - mode: "slim", - detail: undefined, - }, - { - name: "a false label is missing", - readback: () => readbackWith({ [falseLabel]: undefined }), - mode: "slim", - detail: undefined, - }, - { - name: "the readback exits non-zero", - readback: () => ({ code: 1, stderr: "launchctl: boom\n" }), - mode: "slim", - detail: "could not read back the launchd overrides: exit 1: launchctl: boom", - }, - { - name: "the readback times out", - readback: () => "hangs" as const, - mode: "slim", - detail: "could not read back the launchd overrides: timed out after 120000ms", - }, - ])( - "grants the mode the readback shows and tells why when it disagrees or fails: $name", - async ({ readback, mode, detail }) => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - const script = readback(); - - const { onSlimSkipped, onSlimmed, result } = await prepareSlimWith( - filesystem, - typeof script === "string" && script !== "hangs" ? { stdout: script } : script, - slimOptions(), - ); - - expect(result.mode).toBe(mode); - if (detail === undefined) { - expect(onSlimmed).toHaveBeenCalledTimes(1); - expect(onSlimSkipped).not.toHaveBeenCalled(); - } else { - expect(onSlimmed).not.toHaveBeenCalled(); - expect(onSlimSkipped.mock.calls).toEqual([ - [{ deviceId: slim18_5.udid, detail, reason: "apply-failed" }], - ]); - } - }, - ); - - it.each([ - { - name: "writeFileAtomic throws and the readback is as wanted", - failing: "writeFileAtomic" as const, - existing: undefined, - readback: { stdout: readbackWith({}) }, - mode: "slim", - detail: "could not write the launchd overrides file: disk full", - }, - { - name: "mkdirp throws and the readback disagrees", - failing: "mkdirp" as const, - existing: undefined, - readback: { stdout: readbackWith({ [trueLabel]: "enabled" }) }, - mode: "full", - detail: `could not write the launchd overrides file: disk full; 1 of ${managedCount} managed labels did not read back as written at boot; first: ${trueLabel}`, - }, - { - name: "the write throws and the readback fails", - failing: "writeFileAtomic" as const, - existing: undefined, - readback: { code: 1, stderr: "boom" }, - mode: "slim", - detail: - "could not write the launchd overrides file: disk full; could not read back the launchd overrides: exit 1: boom", - }, - { - name: "the existing file is not a plist dict, is left unchanged, and the readback decides", - failing: undefined, - existing: "not a plist", - readback: { stdout: readbackWith({}) }, - mode: "slim", - detail: - "could not write the launchd overrides file: the existing file is not a property list dictionary", - }, - ])( - "boots anyway when the overrides file cannot be written and says so: $name", - async ({ failing, existing, readback, mode, detail }) => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - if (existing !== undefined) { - await filesystem.mkdirp(overridesFolder(slim18_5.udid)); - await filesystem.writeFileAtomic(overridesPath(slim18_5.udid), existing); - } - filesystem.failing = failing; - - const { onSlimSkipped, onSlimmed, result, runner } = await prepareSlimWith( - filesystem, - readback, - slimOptions(), - ); - - expect(result.mode).toBe(mode); - expect(runner.calls.map((call) => call.args[3])).toEqual([ - "boot", - "bootstatus", - "listapps", - "spawn", - ]); - expect(onSlimmed).not.toHaveBeenCalled(); - expect(onSlimSkipped.mock.calls).toEqual([ - [{ deviceId: slim18_5.udid, detail, reason: "apply-failed" }], - ]); - if (existing !== undefined) { - expect(await filesystem.readFile(overridesPath(slim18_5.udid))).toBe(existing); - } - }, - ); - - it("ignores a full flag an older daemon stamped into driver data: the spec mode decides", async () => { - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const stamped = { ...slim18_5, full: true }; - const runner = new ScriptedProcessRunner(slimBootCalls(stamped.udid, wantedReadback())); - const driver = await createSlimDriver(runner, filesystem, slimOptions()); - - const result = await driver.makeReady( - { address: stamped.udid, deviceId: stamped.udid, driverData: stamped }, - prepareSlim, - ); - - expect(result.mode).toBe("slim"); - }); - - it.each([ - { - name: "full mode", - runtimeId: slim18_5.runtimeId, - options: { mode: "full", purpose: "prepare" } as const, - categories: ["widgets"], - skipped: undefined, - }, - { - name: "a purpose: 'recover' boot, which only reboots", - runtimeId: slim18_5.runtimeId, - options: { mode: "slim", purpose: "recover" } as const, - categories: ["widgets"], - skipped: undefined, - }, - { - name: "runtime 18.4", - runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-4", - options: prepareSlim, - categories: ["widgets"], - skipped: "runtime-too-old", - }, - { - name: "a runtime id that does not parse", - runtimeId: "not-a-runtime-id", - options: prepareSlim, - categories: ["widgets"], - skipped: "unknown-runtime", - }, - { - name: "configured categories none of which is known", - runtimeId: slim18_5.runtimeId, - options: prepareSlim, - categories: ["nonsense"], - skipped: "apply-failed", - }, - ])( - "touches no overrides file and runs no readback for a device that must not be slimmed: $name", - async ({ runtimeId, options, categories, skipped }) => { - const filesystem = new FaultyFilesystem(); - await plantManagedDevice(filesystem); - filesystem.writes.length = 0; - const data = { ...driverData, runtimeId }; - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", data.udid) }, - { match: simctl("bootstatus", data.udid, "-b") }, - { match: simctl("listapps", data.udid) }, - ]); - const onSlimmed = vi.fn(); - const onSlimSkipped = vi.fn(); - const driver = await createSlimDriver( - runner, - filesystem, - { bootTimeoutMs: 300_000, categories }, - onSlimmed, - onSlimSkipped, - ); - - const result = await driver.makeReady( - { address: data.udid, deviceId: data.udid, driverData: data }, - options, - ); - - expect(result).toEqual({ - address: data.udid, - deviceId: data.udid, - driverData: data, - mode: "full", - }); - expect(runner.calls.map((call) => call.args)).toEqual([ - simctlArgs("boot", data.udid), - simctlArgs("bootstatus", data.udid, "-b"), - simctlArgs("listapps", data.udid), - ]); - expect(filesystem.writes).toEqual([]); - expect(await filesystem.exists(overridesFolder(data.udid))).toBe(false); - expect(onSlimmed).not.toHaveBeenCalled(); - expect(onSlimSkipped.mock.calls.map(([fact]) => (fact as SlimSkippedFact).reason)).toEqual( - skipped === undefined ? [] : [skipped], - ); - }, - ); - - it("times out the slim boot's bootstatus using slim.bootTimeoutMs, not the default 120s", async () => { - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: { command: "xcrun", args: simctlArgs("boot", slim18_5.udid) } }, - { - hangs: true, - match: { command: "xcrun", args: simctlArgs("bootstatus", slim18_5.udid, "-b") }, - }, - { match: { command: "xcrun", args: simctlArgs("shutdown", slim18_5.udid) } }, - ]); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - driverConfig: { deviceRoot }, - instanceId, - simlockHome: "/home/.simlock", - clock, - filesystem, - idGenerator: { generate: () => "device-1" }, - launchdOverridesRoot: overridesRoot, - processRunner: runner, - config: { slim: slimOptions() }, - }); - - const ready = driver.makeReady( - { address: slim18_5.udid, deviceId: slim18_5.udid, driverData: slim18_5 }, - prepareSlim, - ); - - await waitForCalls(runner, 2); - const readyState = promiseState(ready); - - // why: Just past the default 120s: a boot timed against it would have given up by now. - clock.advance(120_001); - await flushMicrotasks221(); - expect(readyState.state).toBe("pending"); - - clock.advance(300_000 - 120_001); - expect(await settledValue(ready)).toBeInstanceOf(BootTimeoutError); - expect(runner.calls[1]?.options).toEqual({ timeoutMs: 300_000 }); - }); - - it("an operator's full iOS boot waits ios.full.bootTimeoutMs, then fails with BootTimeoutError", async () => { - const slim18_4 = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }; - const cases = [ - { data: driverData, full: undefined, options: prepareFull, waitsMs: 300_000 }, - { data: driverData, full: 400_000, options: prepareFull, waitsMs: 400_000 }, - { data: slim18_4, full: 400_000, options: prepareSlim, waitsMs: 400_000 }, - { data: driverData, full: 400_000, options: recoverFull, waitsMs: 400_000 }, - { data: driverData, full: 400_000, options: recoverSlim, waitsMs: 400_000 }, - { data: slim18_5, full: 400_000, options: prepareSlim, waitsMs: 250_000 }, - ]; - for (const { data, full, options, waitsMs } of cases) { - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - { match: { command: "xcrun", args: simctlArgs("boot", data.udid) } }, - { - hangs: true, - match: { command: "xcrun", args: simctlArgs("bootstatus", data.udid, "-b") }, - }, - { match: { command: "xcrun", args: simctlArgs("shutdown", data.udid) } }, - ]); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - driverConfig: { deviceRoot }, - instanceId, - simlockHome: "/home/.simlock", - clock, - filesystem, - idGenerator: { generate: () => "device-1" }, - launchdOverridesRoot: overridesRoot, - processRunner: runner, - config: { - slim: slimOptions(250_000), - ...(full === undefined ? {} : { full: { bootTimeoutMs: full } }), - }, - }); - - const ready = driver.makeReady( - { address: data.udid, deviceId: data.udid, driverData: data }, - options, - ); - await waitForCalls(runner, 2); - const readyState = promiseState(ready); - - clock.advance(waitsMs - 1); - await flushMicrotasks221(); - expect(readyState.state, `${JSON.stringify({ full, options })} still waiting`).toBe( - "pending", - ); - - clock.advance(1); - expect(await settledValue(ready)).toBeInstanceOf(BootTimeoutError); - } - }); - - it.each([ - { name: "slim spec", spec: { ...spec, mode: "slim" } as const }, - { name: "full spec", spec }, - ])( - "quotes the cold-boot cost for a boot of a $name, since a slim boot is one boot", - async (c) => { - const driver = await createSlimDriver( - new ScriptedProcessRunner([]), - new MemoryFilesystem(), - slimOptions(), - ); - - expect(driver.estimate({ operation: "boot" }, c.spec)).toBe(60_000); - }, - ); - - describe("resolveSpec and the device mode", () => { - const iphone16 = { model: "iPhone 16", platform: "ios" } as const; - - it("returns a full spec for a slim request on a runtime below 18.5", async () => { - const driver = await createSlimDriver( - scriptedListRunner(), - new MemoryFilesystem(), - slimOptions(), - ); - - await expect( - driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "18.4" }), - ).resolves.toEqual({ ...iphone16, osVersion: "18.4", physical: false }); - }); - - it("gives two requests with different modes for one model and runtime different specs", async () => { - const list = { - match: listInvocation, - result: { code: 0, stderr: "", stdout: listFixture }, - }; - const driver = await createSlimDriver( - new ScriptedProcessRunner([list, list]), - new MemoryFilesystem(), - slimOptions(), - ); - - const slim = await driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "26.5" }); - const full = await driver.resolveSpec({ ...iphone16, mode: "full", osVersion: "26.5" }); - - expect(slim).toEqual({ ...iphone16, mode: "slim", osVersion: "26.5", physical: false }); - expect(full).toEqual({ ...iphone16, osVersion: "26.5", physical: false }); - }); - - it("returns a full spec for a slim request when slim is not configured", async () => { - const driver = await createDriver(scriptedListRunner()); - - await expect( - driver.resolveSpec({ ...iphone16, mode: "slim", osVersion: "26.5" }), - ).resolves.toEqual({ ...iphone16, osVersion: "26.5", physical: false }); - }); - }); - - describe("advisories()", () => { - it("reports slim-runtime-unsupported when the default mode is slim and an installed runtime predates 18.5", async () => { - const driver = await createSlimDriver( - scriptedListRunner(), - new MemoryFilesystem(), - slimOptions(), - undefined, - undefined, - true, - ); - - await expect(driver.advisories()).resolves.toEqual([ - { - code: "slim-runtime-unsupported", - message: expect.stringContaining("18.4"), - }, - ]); - }); - - it("reports nothing when every installed runtime is 18.5+", async () => { - const onlyNewFixture = JSON.stringify({ - devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, - runtimes: ( - JSON.parse(listFixture) as { runtimes: { version: string }[] } - ).runtimes.filter((runtime) => runtime.version !== "18.4"), - }); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: onlyNewFixture } }, - ]); - const driver = await createSlimDriver( - runner, - new MemoryFilesystem(), - slimOptions(), - undefined, - undefined, - true, - ); - - await expect(driver.advisories()).resolves.toEqual([]); - }); - - it("reports nothing when slim is not configured", async () => { - const driver = await createDriver(scriptedListRunner()); - - await expect(driver.advisories()).resolves.toEqual([]); - }); - - it("reports an installed runtime with an unparseable identifier as unsupported, even though its marketing version looks modern (finding #5, issue #87 review)", async () => { - // why: The marketing `version` string here ("26.5") would pass the persistent-slim floor if - // `advisories()` still parsed it directly -- but this fixture's `identifier` doesn't - // match `iosRuntimeVersionFromId`'s pattern, and `planSlimBoot` (the boot-time decision) - // only ever reads the identifier. `advisories()` must reuse that same call so the two - // can never disagree about whether this runtime actually gets slimmed. - const devicetypes = (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes; - const weirdIdentifierFixture = JSON.stringify({ - devicetypes, - runtimes: [ - { - identifier: "com.apple.CoreSimulator.SimRuntime.iOS-not-a-version", - isAvailable: true, - name: "iOS 26.5", - supportedDeviceTypes: [], - version: "26.5", - }, - ], - }); - const runner = new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: weirdIdentifierFixture }, - }, - ]); - const driver = await createSlimDriver( - runner, - new MemoryFilesystem(), - slimOptions(), - undefined, - undefined, - true, - ); - - await expect(driver.advisories()).resolves.toEqual([ - { - code: "slim-runtime-unsupported", - message: expect.stringContaining("26.5"), - }, - ]); - }); - - it("reports nothing for an iOS 27.2 device-specific runtime installed beside the general one (#1064)", async () => { - const devicetypes = (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes; - const deviceSpecificFixture = JSON.stringify({ - devicetypes, - runtimes: [ - { - buildversion: "24B91016s", - identifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-2-iPhone19-4", - isAvailable: true, - name: "iOS 27.2", - supportedDeviceTypes: [], - version: "27.2", - }, - { - buildversion: "24B8117", - identifier: "com.apple.CoreSimulator.SimRuntime.iOS-27-2", - isAvailable: true, - name: "iOS 27.2", - supportedDeviceTypes: [], - version: "27.2", - }, - ], - }); - const runner = new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: deviceSpecificFixture }, - }, - ]); - const driver = await createSlimDriver( - runner, - new MemoryFilesystem(), - slimOptions(), - undefined, - undefined, - true, - ); - - await expect(driver.advisories()).resolves.toEqual([]); - }); - - it("reports nothing when the default mode is full, even with a runtime below 18.5 installed", async () => { - const driver = await createSlimDriver( - scriptedListRunner(), - new MemoryFilesystem(), - slimOptions(), - undefined, - undefined, - false, - ); - - await expect(driver.advisories()).resolves.toEqual([]); - }); - - it("reports downloaded runtime assets whose runtime is not in the catalog, and only those (#79)", async () => { - // why: `simctl runtime delete` only unregisters a runtime from CoreSimulator: its ~8.5 GiB - // asset bundle stays in mobileassetd's store, tagged `NeverCollected`, on the same - // volume the download preflight measures. So the bytes stay spent and CoreSimulator can - // re-register the runtime from them later. The catalog here installs 18.4 and 26.5 while - // the store still holds two builds nobody deleted the assets for. - const filesystem = new MemoryFilesystem(); - for (const [bundle, simulatorVersion, build] of [ - ["a1.asset", "18.4", "22E238"], - ["b2.asset", "26.5", "23F79"], - ["c3.asset", "18.6", "22G86"], - ["d4.asset", "26.3", "23D60"], - ] as const) { - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, - assetInfoPlist(simulatorVersion, build), - ); - } - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - // why: The advisory code and message shape are a proposal (see the triage report), so this - // pins only what any fix must do: name both orphan builds, name neither installed one. - const advisories = await driver.advisories(); - const text = advisories.map((advisory) => advisory.message).join("\n"); - expect(advisories.map((advisory) => advisory.code)).toContain( - "runtime-cache-unreclaimable", - ); - expect(text).toContain("18.6"); - expect(text).toContain("26.3"); - expect(text).not.toContain("18.4"); - expect(text).not.toContain("26.5"); - }); - - it("tells two downloads of one marketing version apart by build, and reports only the uninstalled one (#79)", async () => { - // why: simctl reports `buildversion`, and a version match alone would call this bundle - // installed: 18.4 is in the catalog, but not from this download. Deleting the runtime - // and re-downloading the same version leaves exactly this -- a stale build nobody can - // reclaim, sitting behind an installed runtime of the same name. - const catalog = JSON.parse(listFixture) as { - devicetypes: unknown; - runtimes: { version: string }[]; - }; - const withBuilds = JSON.stringify({ - devicetypes: catalog.devicetypes, - runtimes: catalog.runtimes.map((runtime) => ({ - ...runtime, - buildversion: runtime.version === "18.4" ? "22E238" : "23F79", - })), - }); - const filesystem = new MemoryFilesystem(); - for (const [bundle, build] of [ - ["a1.asset", "22E238"], - ["stale.asset", "22E247"], - ] as const) { - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, - assetInfoPlist("18.4", build), - ); - } - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: withBuilds } }, - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - const advisories = await driver.advisories(); - - expect(advisories).toEqual([ - { - code: "runtime-cache-unreclaimable", - message: expect.stringContaining("22E247"), - }, - ]); - expect(advisories[0]?.message).not.toContain("22E238"); - }); - - it("names orphaned runtimes oldest first, not in string order (#79)", async () => { - // why: A store that has been collecting for months is read by a human deciding what to - // reclaim; "iOS 9.3, iOS 18.6" is that list, and a plain string sort inverts it. - const filesystem = new MemoryFilesystem(); - for (const [bundle, simulatorVersion, build] of [ - ["new.asset", "18.6", "22G86"], - ["old.asset", "9.3", "13E233"], - ] as const) { - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, - assetInfoPlist(simulatorVersion, build), - ); - } - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - const message = (await driver.advisories())[0]?.message ?? ""; - - expect(message.indexOf("9.3")).toBeLessThan(message.indexOf("18.6")); - }); - - it("states no size in the advisory when a leftover download's size cannot be read", async () => { - const bundle = `${IOS_RUNTIME_ASSET_ROOT}/c3.asset`; - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(`${bundle}/AssetData`); - await filesystem.writeFileAtomic(`${bundle}/Info.plist`, assetInfoPlist("18.6", "22G86")); - filesystem.defineFailure(`${bundle}/AssetData`, "EACCES"); - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - const advisories = await driver.advisories(); - - expect(advisories).toEqual([ - { - code: "runtime-cache-unreclaimable", - message: expect.stringContaining( - "iOS 18.6 (22G86) is no longer installed, but its download is still in", - ), - }, - ]); - expect(advisories[0]?.message).not.toContain("GiB"); - }); - - it("counts two downloads of one build as downloads of one runtime, with their total size", async () => { - const filesystem = new SizedFilesystem( - new Map([ - [`${IOS_RUNTIME_ASSET_ROOT}/c3.asset`, 8_917_036 * 1024], - [`${IOS_RUNTIME_ASSET_ROOT}/c4.asset`, 8_917_036 * 1024], - ]), - ); - for (const bundle of ["c3.asset", "c4.asset"]) { - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, - assetInfoPlist("18.6", "22G86"), - ); - } - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - const message = (await driver.advisories())[0]?.message ?? ""; - - expect(message).toContain( - "iOS 18.6 (22G86) is no longer installed, but its downloads (17.0 GiB in all) are " + - "still in", - ); - }); - - it("states the measured sizes of several leftover downloads as one total", async () => { - // why: The sizes of the iOS 18.6 and 18.4 downloads measured on a real machine; this catalog - // installs 18.4, so the second bundle is labelled 18.3 to be reported. - const filesystem = new SizedFilesystem( - new Map([ - [`${IOS_RUNTIME_ASSET_ROOT}/c3.asset`, 8_917_036 * 1024], - [`${IOS_RUNTIME_ASSET_ROOT}/e5.asset`, 8_916_744 * 1024], - ]), - ); - for (const [bundle, simulatorVersion, build] of [ - ["c3.asset", "18.6", "22G86"], - ["e5.asset", "18.3", "22D8075"], - ] as const) { - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/${bundle}`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/${bundle}/Info.plist`, - assetInfoPlist(simulatorVersion, build), - ); - } - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - const advisories = await driver.advisories(); - - expect(advisories).toEqual([ - { - code: "runtime-cache-unreclaimable", - message: expect.stringContaining( - "iOS 18.3 (22D8075), 18.6 (22G86) are no longer installed, but their downloads " + - "(17.0 GiB in all) are still in", - ), - }, - ]); - }); - - it("reports nothing when every downloaded asset belongs to an installed runtime (#79)", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/a1.asset`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/a1.asset/Info.plist`, - assetInfoPlist("18.4", "22E238"), - ); - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - await expect(driver.advisories()).resolves.toEqual([]); - }); - - it("stays quiet, and asks simctl nothing, when the asset store cannot be read (#79)", async () => { - // why: The store is macOS's, not Simlock's: absent on a machine that never downloaded a - // runtime, and readable only to whoever the OS says. Neither is a `doctor` failure -- - // and with nothing to report, the `simctl list` that would classify the bundles is - // never worth running, which is why the runner below scripts no invocation at all. - const filesystem = new MemoryFilesystem(); - filesystem.defineFailure(IOS_RUNTIME_ASSET_ROOT, "EACCES"); - const runner = new ScriptedProcessRunner([]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - await expect(driver.advisories()).resolves.toEqual([]); - expect(runner.calls).toEqual([]); - }); - - it("skips an asset bundle it cannot identify by build rather than naming it (#79)", async () => { - // why: The build is what decides whether a bundle's runtime is installed, so a bundle - // without one cannot be classified -- and guessing would send the operator to delete - // a runtime they are still using. `missing-build.asset` below names iOS 19.9 and - // nothing else; the store around it is still reported. - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/missing-build.asset`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/missing-build.asset/Info.plist`, - assetInfoPlist("19.9", "22H1").replace("Build22H1", ""), - ); - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/orphan.asset`); - await filesystem.writeFileAtomic( - `${IOS_RUNTIME_ASSET_ROOT}/orphan.asset/Info.plist`, - assetInfoPlist("18.6", "22G86"), - ); - const driver = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - - const advisories = await driver.advisories(); - - expect(advisories).toEqual([ - { - code: "runtime-cache-unreclaimable", - message: expect.stringContaining("18.6"), - }, - ]); - expect(advisories[0]?.message).not.toContain("19.9"); - }); - }); - }); -}); - -describe("IosSimctlDriver listComponents()", () => { - const runtimeListInvocation = simctl("runtime", "list", "-j"); - const defaultSetInvocation = { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }; - - interface Image { - readonly id: string; - readonly runtime: string; - readonly build: string; - readonly version: string; - readonly sizeBytes?: number; - readonly platform?: string; - } - - function image(version: string, build: string, extra: Partial = {}): Image { - return { - build, - id: `IMG-${version}-${build}`, - runtime: `com.apple.CoreSimulator.SimRuntime.iOS-${version.replace(".", "-")}`, - version, - ...extra, - }; - } - - function imagesListed(images: readonly Image[], state = "Ready") { - const stdout = JSON.stringify( - Object.fromEntries( - images.map((entry) => [ - entry.id, - { - build: entry.build, - identifier: entry.id, - platformIdentifier: entry.platform ?? "com.apple.platform.iphonesimulator", - runtimeIdentifier: entry.runtime, - sizeBytes: entry.sizeBytes ?? 1_000, - state, - version: entry.version, - }, - ]), - ), - ); - return { match: runtimeListInvocation, result: { code: 0, stderr: "", stdout } }; - } - - /** - * why: The default set's devices per runtime: a number is that many used devices, or name both - * kinds. A used device carries `lastUsedAt`, as `simctl list -j devices` reports one that has - * been booted; an unused one -- like the batch CoreSimulator creates when a runtime install - * ends -- carries every other key a real entry has and no `lastUsedAt`. - */ - function defaultSetListed( - devices: Readonly< - Record - >, - ) { - const entry = (runtime: string, index: number, used: boolean) => ({ - dataPath: `/Users/me/Library/Developer/CoreSimulator/Devices/${runtime}-${String(index)}/data`, - dataPathSize: used ? 1_500_000_000 : 18_000_000, - deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", - isAvailable: true, - ...(used ? { lastUsedAt: "2026-10-01T12:00:00Z" } : {}), - logPath: `/Users/me/Library/Logs/CoreSimulator/${runtime}-${String(index)}`, - logPathSize: 4096, - name: `user-${String(index)}`, - state: "Shutdown", - udid: `USER-${runtime}-${String(index)}`, - }); - const stdout = JSON.stringify({ - devices: Object.fromEntries( - Object.entries(devices).map(([runtime, counts]) => { - const { unused = 0, used = 0 } = typeof counts === "number" ? { used: counts } : counts; - return [ - runtime, - [ - ...Array.from({ length: used }, (_, index) => entry(runtime, index, true)), - ...Array.from({ length: unused }, (_, index) => entry(runtime, used + index, false)), - ], - ]; - }), - ), - }); - return { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout } }; - } - - it("lists one entry per iOS runtime image with its version, size and build, and no entry for another platform's runtime", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([ - image("18.4", "22E238", { sizeBytes: 8_963_458_461 }), - image("26.0", "23A343", { sizeBytes: 9_100_000_000 }), - image("11.4", "22T250", { - platform: "com.apple.platform.watchsimulator", - runtime: "com.apple.CoreSimulator.SimRuntime.watchOS-11-4", - }), - ]), - defaultSetListed({}), - ]); - const driver = await createDriver(runner); - - await expect(driver.listComponents()).resolves.toEqual([ - { - foreignDevices: 0, - receipt: { build: "22E238", image: "IMG-18.4-22E238" }, - sizeBytes: 8_963_458_461, - variant: "22E238", - version: "18.4", - }, - { - foreignDevices: 0, - receipt: { build: "23A343", image: "IMG-26.0-23A343" }, - sizeBytes: 9_100_000_000, - variant: "23A343", - version: "26.0", - }, - ]); - }); - - it("counts a device in the default device set that has lastUsedAt and uses an image's runtime as foreign, and reads no device in Simlock's own set for it", async () => { - const used = image("18.4", "22E238"); - const unused = image("26.0", "23A343"); - // why: The scripted runner refuses any call it was not given, so a `simctl --set list` - // would fail this test: only the default set is read. - const runner = new ScriptedProcessRunner([ - imagesListed([used, unused]), - defaultSetListed({ [used.runtime]: 2, "com.apple.CoreSimulator.SimRuntime.iOS-17-5": 1 }), - ]); - const driver = await createDriver(runner); - - const listed = await driver.listComponents(); - - expect(listed.map(({ foreignDevices, version }) => ({ foreignDevices, version }))).toEqual([ - { foreignDevices: 2, version: "18.4" }, - { foreignDevices: 0, version: "26.0" }, - ]); - expect(runner.calls.map((call) => call.args)).toEqual([ - runtimeListInvocation.args, - defaultSetInvocation.args, - ]); - }); - - it("does not count a device in the default device set without lastUsedAt, such as the ones macOS creates when a runtime install ends (#241)", async () => { - const fresh = image("18.6", "22G86"); - const runner = new ScriptedProcessRunner([ - imagesListed([fresh]), - // why: What a runtime install leaves on Xcode 26.4.1: about eleven never-booted simulators of - // it. One more the user has booted is the only one that counts. - defaultSetListed({ [fresh.runtime]: { unused: 11, used: 1 } }), - ]); - const driver = await createDriver(runner); - - const listed = await driver.listComponents(); - - expect(listed.map(({ foreignDevices }) => foreignDevices)).toEqual([1]); - }); - - it("counts a default-set entry that is not a device object as a user, so an unreadable entry never unblocks a removal (#241)", async () => { - const fresh = image("18.6", "22G86"); - const runner = new ScriptedProcessRunner([ - imagesListed([fresh]), - { - match: defaultSetInvocation, - result: { - code: 0, - stderr: "", - stdout: JSON.stringify({ - devices: { [fresh.runtime]: ["garbled", null, { name: "fresh", udid: "U-1" }] }, - }), - }, - }, - ]); - const driver = await createDriver(runner); - - const listed = await driver.listComponents(); - - expect(listed.map(({ foreignDevices }) => foreignDevices)).toEqual([2]); - }); - - it("lists a runtime with the receipt installComponent returned for it, and runs no installer to list", async () => { - const before = [ - { build: "22E238", id: "IMG-184", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-4" }, - ]; - const after = [ - ...before, - { build: "22G86", id: "IMG-186", runtime: "com.apple.CoreSimulator.SimRuntime.iOS-18-6" }, - ]; - const asImages = (entries: typeof after): Image[] => - entries.map((entry) => ({ - ...entry, - version: entry.runtime.endsWith("18-6") ? "18.6" : "18.4", - })); - const listed = (stdout: string) => ({ - match: listInvocation, - result: { code: 0, stderr: "", stdout }, - }); - const runner = new ScriptedProcessRunner([ - listed(listFixture), - imagesListed(asImages(before)), - { - match: { - args: ["-downloadPlatform", "iOS", "-buildVersion", "18.6"], - command: "xcodebuild", - }, - }, - listed(listFixtureAfterDownload), - imagesListed(asImages(after)), - imagesListed(asImages(after)), - defaultSetListed({}), - ]); - const driver = await createDriver(runner); - const installed = await driver.installComponent("18.6", { - onProgress: () => undefined, - signal: new AbortController().signal, - }); - const callsBeforeListing = runner.calls.length; - - const components = await driver.listComponents(); - - expect(components.find((component) => component.version === "18.6")?.receipt).toEqual( - installed.receipt, - ); - expect(runner.calls.slice(callsBeforeListing).map((call) => call.command)).toEqual([ - "xcrun", - "xcrun", - ]); - }); - - it("leaves out an image with no version or no platform, leaves out a size that is not a byte count, and counts no device under a runtime that lists none", async () => { - const ios = "com.apple.platform.iphonesimulator"; - const runtimeList = { - "IMG-A": { - build: "22E238", - identifier: "IMG-A", - platformIdentifier: ios, - runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-18-4", - sizeBytes: -1, - version: "18.4", - }, - "IMG-B": { - build: "23A343", - identifier: "IMG-B", - platformIdentifier: ios, - runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-0", - sizeBytes: 1.5, - }, - "IMG-C": { - build: "23A344", - identifier: "IMG-C", - runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-1", - sizeBytes: 10, - version: "26.1", - }, - }; - const runner = new ScriptedProcessRunner([ - { - match: runtimeListInvocation, - result: { code: 0, stderr: "", stdout: JSON.stringify(runtimeList) }, - }, - { - match: defaultSetInvocation, - result: { - code: 0, - stderr: "", - stdout: JSON.stringify({ - devices: { "com.apple.CoreSimulator.SimRuntime.iOS-18-4": "not a list" }, - }), - }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.listComponents()).resolves.toEqual([ - { - foreignDevices: 0, - receipt: { build: "22E238", image: "IMG-A" }, - variant: "22E238", - version: "18.4", - }, - ]); - }); - - it("rejects with DriverCrashError when the default device set answers with something that is not JSON", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([image("18.4", "22E238")]), - { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout: "not json" } }, - ]); - const driver = await createDriver(runner); - - const error = await driver.listComponents().catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(DriverCrashError); - expect((error as Error).message).toContain("Could not parse simctl device list"); - }); - - it("rejects with DriverCrashError when the default device set answers JSON with no devices map", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([image("18.4", "22E238")]), - { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout: "{}" } }, - ]); - const driver = await createDriver(runner); - - const error = await driver.listComponents().catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(DriverCrashError); - expect((error as Error).message).toBe("Invalid simctl device list JSON"); - }); - - it("rejects when the default device set cannot be read, rather than report no foreign devices", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([image("18.4", "22E238")]), - { - match: defaultSetInvocation, - result: { code: 1, stderr: "CoreSimulator is down", stdout: "" }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.listComponents()).rejects.toBeInstanceOf(DriverCrashError); - }); - - describe("removeComponent()", () => { - const target = image("26.4", "23E244", { sizeBytes: 7_000_000_000 }); - const other = image("18.4", "22E238"); - const receipt = { build: "23E244", image: "IMG-26.4-23E244" }; - const deleteInvocation = simctl("runtime", "delete", "IMG-26.4-23E244"); - const deleted = { match: deleteInvocation, result: { code: 0, stderr: "", stdout: "" } }; - const signal = () => new AbortController().signal; - - async function assetStore( - builds: readonly (readonly [string, string])[], - ): Promise { - return addAssets(new MemoryFilesystem(), builds); - } - - async function addAssets( - filesystem: MemoryFilesystem, - builds: readonly (readonly [string, string])[], - ): Promise { - for (const [index, [version, build]] of builds.entries()) { - const bundle = `${IOS_RUNTIME_ASSET_ROOT}/bundle-${String(index)}.asset`; - await filesystem.mkdirp(bundle); - await filesystem.writeFileAtomic(`${bundle}/Info.plist`, assetInfoPlist(version, build)); - } - return filesystem; - } - - it("deletes the image the receipt names with simctl runtime delete and no --keep-asset, and reports no residue when it had no unused default-set device and its download is gone", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target, other]), - defaultSetListed({ [other.runtime]: { unused: 4 } }), - deleted, - imagesListed([other]), - imagesListed([other]), - defaultSetListed({ [other.runtime]: { unused: 4 } }), - ]); - const filesystem = await assetStore([ - ["18.4", "22E238"], - ["26.4", "23E200"], - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - await expect(driver.removeComponent(receipt, { signal: signal() })).resolves.toEqual({ - sizeBytes: 7_000_000_000, - }); - expect(runner.calls[2]?.args).toEqual(deleteInvocation.args); - }); - - it("deletes the seeds made from the removed runtime before the runtime, logs one it cannot delete and goes on, and keeps the seeds of another runtime", async () => { - const [goneSeed, stuckSeed, keptSeed] = [1, 2, 3].map( - (n) => `00000000-0000-0000-0000-0000000002${String(n)}0`, - ) as [string, string, string]; - const database = await memorySourceDatabase(); - const removed = { - deviceTypeId: driverData.deviceTypeId, - runtimeId: target.runtime, - }; - plantSource(database, { ...removed, udid: goneSeed }); - plantSource(database, { - ...removed, - deviceTypeId: "com.apple.CoreSimulator.SimDeviceType.iPad", - udid: stuckSeed, - }); - plantSource(database, { - deviceTypeId: driverData.deviceTypeId, - runtimeId: other.runtime, - udid: keptSeed, - }); - const runner = new ScriptedProcessRunner([ - imagesListed([target, other]), - defaultSetListed({}), - { match: sourcesSimctl("shutdown", goneSeed) }, - { match: sourcesSimctl("delete", goneSeed) }, - { match: sourcesSimctl("shutdown", stuckSeed) }, - { - match: sourcesSimctl("delete", stuckSeed), - result: { code: 1, stderr: "busy", stdout: "" }, - }, - deleted, - imagesListed([other]), - imagesListed([other]), - defaultSetListed({}), - ]); - const logger = new RecordingLogger(); - const driver = await createDriver( - runner, - new FakeClock(), - await assetStore([]), - database, - logger, - ); - - await driver.removeComponent(receipt, { signal: signal() }); - - expect(runner.calls.map((call) => [call.args[3], call.args[4]])).toEqual( - expect.arrayContaining([ - ["shutdown", goneSeed], - ["delete", goneSeed], - ["delete", stuckSeed], - ["runtime", "delete"], - ]), - ); - expect(runner.calls.some((call) => call.args.includes(keptSeed))).toBe(false); - expect( - runner.calls.findIndex((call) => call.args[3] === "runtime" && call.args[4] === "delete"), - ).toBeGreaterThan(runner.calls.findIndex((call) => call.args[4] === stuckSeed)); - expect(logger.warnings).toEqual([ - expect.stringContaining( - `could not remove seed ${stuckSeed} of removed runtime ${target.runtime}:`, - ), - ]); - expect(database.prepare("SELECT udid FROM ios_source_data ORDER BY udid").all()).toEqual([ - { udid: stuckSeed }, - { udid: keptSeed }, - ]); - }); - - it("starts the five-minute deletion budget after the seeds are deleted, so a seed held by a late clone does not use it up", async () => { - const live = image("26.5", "23F77"); - const runtimeDelete = simctl("runtime", "delete", live.id); - const runner = new ScriptedProcessRunner([ - { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, - { hangs: true, match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot) }, - imagesListed([live]), - defaultSetListed({}), - { match: sourcesSimctl("shutdown", seedUdid) }, - { match: sourcesSimctl("delete", seedUdid) }, - { match: runtimeDelete, result: { code: 0, stderr: "", stdout: "" } }, - imagesListed([live], "Deleting"), - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]); - const clock = new FakeClock(); - const driver = await createDriver( - runner, - clock, - await assetStore([]), - await seededDatabase(), - ); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - const provision = driver.provision(spec).catch((error: unknown) => error); - await waitForCalls(runner, 2); - clock.advance(30_000); - await provision; - - const removal = driver - .removeComponent({ build: live.build, image: live.id }, { signal: signal() }) - .catch((caught: unknown) => caught); - await flushMicrotasks221(); - clock.advance(300_000); - const outcome = promiseState(removal); - for (let step = 0; step < 10 && outcome.state === "pending"; step += 1) { - await flushMicrotasks221(); - clock.advance(1_000); - } - - expect(await removal).toEqual({ sizeBytes: 1_000 }); - expect(runner.calls.map((call) => call.args)).toContainEqual(runtimeDelete.args); - }); - - it("reports residue naming Xcode's Settings -> Platforms when the build's download is still in the asset store", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]); - const filesystem = await assetStore([["26.4", "23E244"]]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toContain("26.4 (23E244)"); - expect(removal.residue).toContain(IOS_RUNTIME_ASSET_ROOT); - expect(removal.residue).toContain("Xcode's Settings -> Platforms"); - await expect(filesystem.exists(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`)).resolves.toBe( - true, - ); - }); - - it("states the measured size of the download in residue, 8.5 GiB for an asset of 8,917,036 KiB", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]); - // why: The iOS 18.6 download measured on a real machine. - const filesystem = new SizedFilesystem( - new Map([[`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024]]), - ); - await addAssets(filesystem, [["26.4", "23E244"]]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toContain("its download (8.5 GiB) is still in"); - }); - - it("states the same measured size in the doctor advisory as in the residue of the removal", async () => { - const filesystem = new SizedFilesystem( - new Map([[`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024]]), - ); - await addAssets(filesystem, [["26.4", "23E244"]]); - const remover = await createDriver( - new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]), - new FakeClock(), - filesystem, - ); - const residue = (await remover.removeComponent(receipt, { signal: signal() })).residue; - const doctor = await createDriver(scriptedListRunner(), new FakeClock(), filesystem); - const advisory = (await doctor.advisories()).find( - (candidate) => candidate.code === "runtime-cache-unreclaimable", - ); - - const sizeIn = (text: string | undefined) => /its download \(([^)]+)\)/.exec(text ?? "")?.[1]; - expect(sizeIn(residue)).toBe("8.5 GiB"); - expect(sizeIn(advisory?.message)).toBe(sizeIn(residue)); - }); - - it("states the total size of two downloads the removed build leaves, as downloads of one runtime", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]); - const filesystem = new SizedFilesystem( - new Map([ - [`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset`, 8_917_036 * 1024], - [`${IOS_RUNTIME_ASSET_ROOT}/bundle-1.asset`, 8_916_744 * 1024], - ]), - ); - await addAssets(filesystem, [ - ["26.4", "23E244"], - ["26.4", "23E244"], - ]); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toContain( - "iOS 26.4 (23E244) is no longer installed, but its downloads (17.0 GiB in all) are " + - "still in", - ); - expect(removal.residue).toContain("does not reclaim them"); - }); - - it("states no size in residue when the download's size cannot be read", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({}), - ]); - const filesystem = await assetStore([["26.4", "23E244"]]); - await filesystem.mkdirp(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset/AssetData`); - filesystem.defineFailure(`${IOS_RUNTIME_ASSET_ROOT}/bundle-0.asset/AssetData`, "EACCES"); - const driver = await createDriver(runner, new FakeClock(), filesystem); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toContain("its download is still in"); - expect(removal.residue).not.toContain("GiB"); - }); - - it("removes the runtime when its only default-set devices were never used (#241)", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({ [target.runtime]: { unused: 11 } }), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({ [target.runtime]: { unused: 11 } }), - ]); - const driver = await createDriver(runner, new FakeClock(), await assetStore([])); - - await expect(driver.removeComponent(receipt, { signal: signal() })).resolves.toMatchObject({ - sizeBytes: 7_000_000_000, - }); - expect(runner.calls[2]?.args).toEqual(deleteInvocation.args); - }); - - it.each([ - [ - 3, - "3 never-used simulators in the default device set are now unavailable; Simlock does " + - "not delete simulators there -- `xcrun simctl delete unavailable` clears them", - ], - [ - 1, - "1 never-used simulator in the default device set is now unavailable; Simlock does " + - "not delete simulators there -- `xcrun simctl delete unavailable` clears them", - ], - ])( - "reports %s unused default-set devices as unavailable in residue, naming xcrun simctl delete unavailable, in the singular for one (#241)", - async (unused, residue) => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({ [target.runtime]: { unused } }), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({ [target.runtime]: { unused } }), - ]); - const driver = await createDriver(runner, new FakeClock(), await assetStore([])); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toBe(residue); - }, - ); - - it("carries both the unavailable devices and the download left in the asset store in residue (#241)", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({ [target.runtime]: { unused: 2 } }), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({ [target.runtime]: { unused: 2 } }), - ]); - const driver = await createDriver( - runner, - new FakeClock(), - await assetStore([["26.4", "23E244"]]), - ); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal.residue).toMatch( - /^2 never-used simulators .*`xcrun simctl delete unavailable` clears them\. iOS 26\.4 \(23E244\) is no longer installed, but its download .*Xcode's Settings -> Platforms to get the space back$/, - ); - }); - - it("reports no unavailable simulators when another installed build of the same runtime stays behind (#241)", async () => { - // why: Two builds of iOS 26.4 share one runtime identifier, so its simulators keep working - // on the build that stays. - const sibling = image("26.4", "23E200", { id: "IMG-26.4-23E200" }); - const runner = new ScriptedProcessRunner([ - imagesListed([target, sibling]), - defaultSetListed({ [target.runtime]: { unused: 3 } }), - deleted, - imagesListed([sibling]), - imagesListed([sibling]), - defaultSetListed({ [target.runtime]: { unused: 3 } }), - ]); - const driver = await createDriver(runner, new FakeClock(), await assetStore([])); - - const removal = await driver.removeComponent(receipt, { signal: signal() }); - - expect(removal).toEqual({ sizeBytes: 7_000_000_000 }); - }); - - it("runs no simctl command that writes to the default device set: its only unscoped call is the device list (#241)", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({ [target.runtime]: { unused: 11 } }), - deleted, - imagesListed([]), - imagesListed([]), - defaultSetListed({ [target.runtime]: { unused: 11 } }), - ]); - const driver = await createDriver(runner, new FakeClock(), await assetStore([])); - - await driver.removeComponent(receipt, { signal: signal() }); - - // why: Every call that names no `--set` addresses the default set; each must be the read. - const unscoped = runner.calls.filter( - (call) => call.command === "xcrun" && !call.args.includes("--set"), - ); - expect(unscoped.map((call) => call.args)).toEqual([ - defaultSetInvocation.args, - defaultSetInvocation.args, - ]); - expect(runner.calls).toHaveLength(6); - }); - - it("reports the same foreign-device count in listComponents and in removeComponent's refusal for the same default set (#241)", async () => { - const set = defaultSetListed({ [target.runtime]: { unused: 9, used: 2 } }); - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - set, - imagesListed([target]), - set, - ]); - const driver = await createDriver(runner); - - const [listed] = await driver.listComponents(); - const error = await driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught); - - expect(listed?.foreignDevices).toBe(2); - expect(error).toBeInstanceOf(ComponentInUseError); - expect(error).toMatchObject({ foreignDevices: listed?.foreignDevices }); - }); - - it("fails with simctl's own error when runtime delete exits non-zero", async () => { - // why: No listing after the failure: the scripted runner fails any call it was not given. - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - { - match: deleteInvocation, - result: { code: 1, stderr: "Unable to delete runtime: in use", stdout: "" }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.removeComponent(receipt, { signal: signal() })).rejects.toThrow( - "Unable to delete runtime: in use", - ); - expect(runner.calls).toHaveLength(3); - }); - - it("refuses with ComponentInUseError when one default-set device of the runtime has lastUsedAt, and deletes nothing", async () => { - // why: The scripted runner fails any call it was not given, so a delete would fail this test. - // Ten never-used devices beside the used one: only the used one counts. - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({ [target.runtime]: { unused: 10, used: 1 } }), - ]); - const driver = await createDriver(runner); - - const error = await driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(ComponentInUseError); - expect(error).toMatchObject({ foreignDevices: 1 }); - expect(runner.calls).toHaveLength(2); - }); - - it("refuses with ComponentNotOwnedError when no listed image has the receipt, and deletes nothing", async () => { - const runner = new ScriptedProcessRunner([imagesListed([target]), defaultSetListed({})]); - const driver = await createDriver(runner); - - await expect( - driver.removeComponent({ build: "23E244", image: "IMG-OTHER" }, { signal: signal() }), - ).rejects.toBeInstanceOf(ComponentNotOwnedError); - expect(runner.calls).toHaveLength(2); - }); - - it("fails when the image is still listed after simctl answered", async () => { - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - deleted, - imagesListed([target]), - imagesListed([target]), - defaultSetListed({}), - ]); - const driver = await createDriver(runner); - - // why: The clock never moves, so a removal that waited would never settle. - const error = await settledValue( - driver.removeComponent(receipt, { signal: signal() }).catch((caught: unknown) => caught), - ); - - expect(error).toBeInstanceOf(DriverCrashError); - expect(error).toMatchObject({ message: "ios 26.4 is still installed after it was removed" }); - expect(runner.calls).toHaveLength(6); - }); - - it("answers the removal with its size and residue when simctl runtime list shows the image as Deleting after simctl runtime delete returns and drops it seconds later (#259)", async () => { - // why: As seen on Xcode 26.4.1: `simctl runtime delete` exits while `simctl runtime list` - // still lists the image, in state `Deleting`, for about eight seconds. The runner answers - // by the fake clock rather than by call order, so it does not dictate how a fix waits. - const clock = new FakeClock(); - const deletingForMs = 8_000; - const runner = new ClockedSimctl259(clock, deletingForMs, { - defaultSet: defaultSetListed({ [target.runtime]: { unused: 2 } }), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], "Deleting"), - }); - const driver = await createDriver(runner, clock, await assetStore([])); - - let settled = false; - const removal = driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught) - .finally(() => { - settled = true; - }); - // why: Lets time pass in one-second steps, well inside the five-minute removal budget. - for (let second = 0; second < 30 && !settled; second += 1) { - await flushMicrotasks221(); - clock.advance(1_000); - } - - expect(await settledValue(removal)).toEqual({ - residue: - "2 never-used simulators in the default device set are now unavailable; Simlock does " + - "not delete simulators there -- `xcrun simctl delete unavailable` clears them", - sizeBytes: 7_000_000_000, - }); - expect(runner.deletes).toBe(1); - }); - - it("fails with DriverCrashError five minutes after simctl runtime delete started, not after it answered, when simctl runtime list keeps showing the image as Deleting (#259)", async () => { - // why: The delete itself takes two minutes, so a budget restarted when it answers would end - // two minutes later than one measured from its start. - const clock = new FakeClock(); - const runner = new ClockedSimctl259( - clock, - Number.POSITIVE_INFINITY, - { - defaultSet: defaultSetListed({}), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], "Deleting"), - }, - // why: Half a second off the one-second polls, so the last pause must be cut short. - 120_500, - ); - const driver = await createDriver(runner, clock, await assetStore([])); - - let settledAt: number | undefined; - const removal = driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught) - .finally(() => { - settledAt = clock.now(); - }); - // why: Half-second steps land on the deadline itself, so the time it fails at is exact. - for (let step = 0; step < 1_200 && settledAt === undefined; step += 1) { - await flushMicrotasks221(); - clock.advance(500); - } - - const error = await settledValue(removal); - expect(error).toBeInstanceOf(DriverCrashError); - expect(error).toMatchObject({ - message: "ios 26.4 was still being deleted 300000ms after its removal started", - }); - expect(settledAt).toBe((runner.deleteStartedAt ?? Number.NaN) + 300_000); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("fails with DriverCrashError as soon as the signal fires while simctl runtime list shows the image as Deleting, and lists no more (#259)", async () => { - const clock = new FakeClock(); - const runner = new ClockedSimctl259(clock, Number.POSITIVE_INFINITY, { - defaultSet: defaultSetListed({}), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], "Deleting"), - }); - const driver = await createDriver(runner, clock, await assetStore([])); - const controller = new AbortController(); - - const removal = driver - .removeComponent(receipt, { signal: controller.signal }) - .catch((caught: unknown) => caught); - await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(1)); - await flushMicrotasks221(); - // why: The clock never moves: only the signal can end the wait. - controller.abort(); - - const error = await settledValue(removal); - expect(error).toBeInstanceOf(DriverCrashError); - expect(error).toMatchObject({ - message: "The removal of ios 26.4 was ended while it was still being deleted", - }); - expect(runner.listsAfterDelete).toBe(1); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("lists the images again every second while simctl runtime list shows the image as Deleting, and leaves no abort listener behind (#259)", async () => { - const clock = new FakeClock(); - const runner = new ClockedSimctl259(clock, 3_500, { - defaultSet: defaultSetListed({}), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], "Deleting"), - }); - const driver = await createDriver(runner, clock, await assetStore([])); - const controller = new AbortController(); - const added = vi.spyOn(controller.signal, "addEventListener"); - const removed = vi.spyOn(controller.signal, "removeEventListener"); - - const removal = driver.removeComponent(receipt, { signal: controller.signal }); - await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(1)); - for (const expected of [2, 3, 4]) { - await flushMicrotasks221(); - clock.advance(999); - await flushMicrotasks221(); - expect(runner.listsAfterDelete).toBe(expected - 1); - clock.advance(1); - await vi.waitFor(() => expect(runner.listsAfterDelete).toBe(expected)); - } - // why: Gone at 3.5 s: the listing at 4 s ends the wait. - await flushMicrotasks221(); - clock.advance(1_000); - await expect(removal).resolves.toEqual({ sizeBytes: 7_000_000_000 }); - expect(added.mock.calls.length).toBeGreaterThan(1); - expect(removed.mock.calls.map(([, listener]) => listener)).toEqual( - expect.arrayContaining(added.mock.calls.map(([, listener]) => listener)), - ); - }); - - it.each(["deleting", "DELETING"])( - "waits while simctl runtime list shows the image in state %s, the state compared ignoring case (#259)", - async (state) => { - const clock = new FakeClock(); - const runner = new ClockedSimctl259(clock, 2_000, { - defaultSet: defaultSetListed({}), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], state), - }); - const driver = await createDriver(runner, clock, await assetStore([])); - - let settled = false; - const removal = driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught) - .finally(() => { - settled = true; - }); - for (let second = 0; second < 30 && !settled; second += 1) { - await flushMicrotasks221(); - clock.advance(1_000); - } - - expect(await settledValue(removal)).toEqual({ sizeBytes: 7_000_000_000 }); - }, - ); - - it("fails with DriverCrashError when the signal fires while simctl runtime list is answering that the image is Deleting, and lists no more (#259)", async () => { - const clock = new FakeClock(); - const controller = new AbortController(); - const runner = new ClockedSimctl259( - clock, - Number.POSITIVE_INFINITY, - { - defaultSet: defaultSetListed({}), - deleted, - gone: imagesListed([]), - ready: imagesListed([target]), - deleting: imagesListed([target], "Deleting"), - }, - 0, - () => { - controller.abort(); - }, - ); - const driver = await createDriver(runner, clock, await assetStore([])); - - // why: The clock never moves: only the signal, already fired when the pause starts, ends it. - const error = await settledValue( - driver - .removeComponent(receipt, { signal: controller.signal }) - .catch((caught: unknown) => caught), - ); - - expect(error).toBeInstanceOf(DriverCrashError); - expect(error).toMatchObject({ - message: "The removal of ios 26.4 was ended while it was still being deleted", - }); - expect(runner.listsAfterDelete).toBe(1); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("ends simctl runtime delete after five minutes, SIGTERM then SIGKILL, and answers only once it has exited", async () => { - // why: Ignores SIGTERM: only the SIGKILL ten seconds later ends it. - const stubborn = { hangs: true, ignoresSigterm: true, match: deleteInvocation }; - const clock = new FakeClock(); - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - stubborn, - ]); - const driver = await createDriver(runner, clock); - let settled = false; - const removal = driver - .removeComponent(receipt, { signal: signal() }) - .catch((caught: unknown) => caught) - .finally(() => { - settled = true; - }); - await vi.waitFor(() => expect(runner.calls).toHaveLength(3)); - - clock.advance(5 * 60_000); - await flushMicrotasks221(); - expect(settled).toBe(false); - clock.advance(10_000); - - expect(await settledValue(removal)).toMatchObject({ - message: expect.stringMatching(/timed out after 300000ms/), - }); - }); - - it("ends simctl runtime delete when its signal fires, and starts none when it already has", async () => { - const abort = new AbortController(); - const runner = new ScriptedProcessRunner([ - imagesListed([target]), - defaultSetListed({}), - { hangs: true, match: deleteInvocation }, - ]); - const driver = await createDriver(runner); - const aborted = driver - .removeComponent(receipt, { signal: abort.signal }) - .catch((caught: unknown) => caught); - await vi.waitFor(() => expect(runner.calls).toHaveLength(3)); - abort.abort(); - expect(await settledValue(aborted)).toMatchObject({ - message: expect.stringMatching(/was ended before it finished/), - }); - - // why: The scripted runner fails any call it was not given, so a delete would fail this. - const lateRunner = new ScriptedProcessRunner([imagesListed([target]), defaultSetListed({})]); - const lateDriver = await createDriver(lateRunner); - await expect( - lateDriver.removeComponent(receipt, { signal: AbortSignal.abort() }), - ).rejects.toThrow(/was ended before it finished/); - expect(lateRunner.calls).toHaveLength(2); - }); - }); -}); - -type ScriptedStep259 = ConstructorParameters[0][number]; - -class ClockedSimctl259 extends ScriptedProcessRunner { - deletes = 0; - deleteStartedAt: number | undefined; - listsAfterDelete = 0; - #deletedAt: number | undefined; - - constructor( - private readonly clock: FakeClock, - private readonly deletingForMs: number, - private readonly steps: { - readonly ready: ScriptedStep259; - readonly deleting: ScriptedStep259; - readonly gone: ScriptedStep259; - readonly deleted: ScriptedStep259; - readonly defaultSet: ScriptedStep259; - }, - private readonly deleteTakesMs = 0, - private readonly onListAfterDelete: () => void = () => undefined, - ) { - super([]); - } - - override spawn(...call: Parameters) { - const [command, args] = call; - const argv = args.join(" "); - let step: ScriptedStep259; - if (argv.endsWith("runtime delete IMG-26.4-23E244")) { - this.deletes += 1; - this.deleteStartedAt = this.clock.now(); - this.clock.advance(this.deleteTakesMs); - this.#deletedAt = this.clock.now(); - step = this.steps.deleted; - } else if (argv.endsWith("runtime list -j")) { - if (this.#deletedAt !== undefined) { - this.listsAfterDelete += 1; - this.onListAfterDelete(); - } - step = - this.#deletedAt === undefined - ? this.steps.ready - : this.clock.now() - this.#deletedAt < this.deletingForMs - ? this.steps.deleting - : this.steps.gone; - } else if (argv === "simctl list -j devices") { - step = this.steps.defaultSet; - } else { - throw new Error(`Unexpected process invocation: ${command} ${argv}`); - } - const handle = new ScriptedProcessRunner([step]).spawn(...call); - this.calls.push({ args: [...args], command, options: call[2] ?? {} }); - return handle; - } -} - -describe("IosSimctlDriver screen capture commands", () => { - const physicalDevice = { - address: "UDID-P", - deviceId: "UDID-P", - driverData: {}, - enrolledApps: [], - physical: true, - } as const; - const settings = { bitrateKbps: 1_500, fps: 30, maxDimension: 720 }; - - // why: the daemon hands the driver its packaged programs; `drivers.ios.deviceCaptureCommand` - // replaces only the physical iPhone's program, and an empty string counts as unset exactly as - // `screenCaptureCommand` does, so one mistaken `""` does not turn iPhone streams off. - it.each([ - ["the configured program", "/worker/custom-device-capture", "/worker/custom-device-capture"], - ["the daemon's program when unset", undefined, "/dist/bin/simlock-device-capture"], - ["the daemon's program when set empty", "", "/dist/bin/simlock-device-capture"], - ] as const)("a physical iPhone's stream starts %s", async (_claim, configured, expected) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/worker"); - await filesystem.mkdirp("/dist/bin"); - await filesystem.writeFileAtomic("/worker/custom-device-capture", ""); - await filesystem.writeFileAtomic("/dist/bin/simlock-device-capture", ""); - const launcher = new FakeBytePipeLauncher(); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - driverConfig: configured === undefined ? {} : { deviceCaptureCommand: configured }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - screenCapture: { - baseEnvironment: { PATH: "/usr/bin" }, - command: "/dist/bin/simlock-screen-capture", - deviceCommand: "/dist/bin/simlock-device-capture", - launcher, - }, - simlockHome: "/home/.simlock", - }); - - const capture = driver.screenCapture; - if (capture === undefined) throw new Error("the driver offers no screen capture"); - await capture.open(physicalDevice, settings, new AbortController().signal); - expect(launcher.starts.map((start) => start.command)).toEqual([expected]); - }); -}); - -describe("IosSimctlDriver defaultModels", () => { - it("lists the built-in iOS preference list per class, newest first, with phone starting at iPhone 17", async () => { - const driver = await createDriver(new ScriptedProcessRunner([])); - - expect(driver.defaultModels).toEqual({ - phone: ["iPhone 17", "iPhone 16", "iPhone 15", "iPhone 14", "iPhone 13"], - tablet: [ - "iPad Pro 11-inch (M5)", - "iPad Pro 11-inch (M4)", - "iPad Air 11-inch (M3)", - "iPad (A16)", - "iPad (10th generation)", - ], - tv: ["Apple TV 4K (3rd generation)", "Apple TV"], - vision: ["Apple Vision Pro"], - watch: [ - "Apple Watch Series 11 (46mm)", - "Apple Watch Series 10 (46mm)", - "Apple Watch Series 9 (45mm)", - ], - }); - }); -}); - -describe("IosSimctlDriver toolVersions()", () => { - it("reports the Xcode version and build that xcodebuild -version prints", async () => { - const runner = new ScriptedProcessRunner([ - { - match: { args: ["-version"], command: "xcodebuild" }, - result: { code: 0, stderr: "", stdout: "Xcode 16.4\nBuild version 16F6\n" }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.toolVersions()).resolves.toEqual([ - { build: "16F6", name: "xcode", version: "16.4" }, - ]); - expect(runner.calls[0]?.options.timeoutMs).toBeGreaterThan(0); - }); - - it("reports the Xcode version without a build when xcodebuild prints none", async () => { - const runner = new ScriptedProcessRunner([ - { - match: { args: ["-version"], command: "xcodebuild" }, - result: { code: 0, stderr: "", stdout: "Xcode 16.4\n" }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.toolVersions()).resolves.toEqual([{ name: "xcode", version: "16.4" }]); - }); - - it("reports no Xcode entry when only the command line tools are installed", async () => { - const runner = new ScriptedProcessRunner([ - { - match: { args: ["-version"], command: "xcodebuild" }, - result: { - code: 1, - stderr: - "xcode-select: error: tool 'xcodebuild' requires Xcode, but active developer directory '/Library/Developer/CommandLineTools' is a command line tools instance", - stdout: "", - }, - }, - ]); - const driver = await createDriver(runner); - - await expect(driver.toolVersions()).resolves.toEqual([]); - }); - - it.each([ - // why: Output that would parse, so only the exit code can be what turns it away. - ["exits non-zero", { code: 1, stderr: "error", stdout: "Xcode 16.4\nBuild version 16F6\n" }], - ["is killed by its timeout", { code: null, stderr: "", stdout: "" }], - ["prints no Xcode version", { code: 0, stderr: "", stdout: "something else\n" }], - ])("rejects when xcodebuild %s", async (_case, result) => { - const runner = new ScriptedProcessRunner([ - { match: { args: ["-version"], command: "xcodebuild" }, result }, - ]); - const driver = await createDriver(runner); - - await expect(driver.toolVersions()).rejects.toThrow(/xcodebuild/); - }); - - it("rejects when xcodebuild cannot be started", async () => { - // why: No scripted invocation: the runner refuses to start it, as a spawn failure would. - const driver = await createDriver(new ScriptedProcessRunner([])); - - await expect(driver.toolVersions()).rejects.toThrow(/xcodebuild/); - }); -}); - -function simctlPassthrough(driver: IosSimctlDriver, args: readonly string[]) { - return driver.passthrough("simctl", undefined, args); -} - -async function createDriver( - runner: ScriptedProcessRunner, - clock = new FakeClock(), - filesystem: Filesystem = new MemoryFilesystem(), - database?: SqliteDatabase, - logger?: Logger, - fullBootTimeoutMs?: number, -): Promise { - scopedRunners.push(runner); - return IosSimctlDriver.create({ - database: database ?? (await memorySourceDatabase()), - clock, - driverConfig: { deviceRoot }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - ...(fullBootTimeoutMs === undefined - ? {} - : { config: { full: { bootTimeoutMs: fullBootTimeoutMs } } }), - ...(logger === undefined ? {} : { logger }), - processRunner: runner, - simlockHome: "/home/.simlock", - }); -} - -/** why: the state a case starts from, with the writes and modes the planting made forgotten. */ -async function plantOverrides( - filesystem: FaultyFilesystem, - existing: Readonly> | undefined, -): Promise { - if (existing !== undefined) { - await filesystem.mkdirp(overridesFolder(driverData.udid)); - await filesystem.writeFileAtomic(overridesPath(driverData.udid), overridesPlist(existing)); - } - filesystem.writes.length = 0; - delete filesystem.modes.file; - delete filesystem.modes.folder; -} - -/** why: the boot is where launchd reads the file, so the file is captured as the boot starts. */ -function runnerReadingFileAtBoot(filesystem: FaultyFilesystem, clock: FakeClock) { - let contents: string | undefined; - const calls = slimBootCalls(driverData.udid, wantedReadback()); - const runner = new ScriptedProcessRunner([ - { - ...calls[0]!, - onSpawn: async () => { - clock.advance(40_000); - const path = overridesPath(driverData.udid); - contents = (await filesystem.exists(path)) ? await filesystem.readFile(path) : undefined; - }, - }, - ...calls.slice(1), - ]); - return { fileAtBoot: () => contents, runner }; -} - -/** why: a LaunchServices database file and a neighbour that must stay, in the device folder `device`. */ -async function plantLaunchServices(filesystem: MemoryFilesystem, device: string): Promise { - const folder = `${device}/data/var/db/lsd`; - await filesystem.mkdirp(folder); - await filesystem.writeFileAtomic(`${folder}/com.apple.LaunchServices-33554481-v2.csstore`, "x"); - await filesystem.writeFileAtomic( - `${folder}/com.apple.LaunchServices.unregisteredtypes-501.db`, - "", - ); -} - -async function launchServicesFiles( - filesystem: MemoryFilesystem, - device: string, -): Promise { - return (await filesystem.readdir(`${device}/data/var/db/lsd`)).sort(); -} - -/** why: Records every `warn` message, so the destroy's log line is asserted on its text. */ -class RecordingLogger implements Logger { - readonly warnings: string[] = []; - - debug(): void {} - info(): void {} - warn(message: string): void { - this.warnings.push(message); - } - error(): void {} - child(): Logger { - return this; - } -} - -/** why: Records the mode each `mkdirp` was asked for, because `MemoryFilesystem` creates every directory owner-only by default and so cannot show a caller that left the mode out. */ -class ModeRecordingFilesystem extends MemoryFilesystem { - readonly mkdirpModes = new Map(); - - override async mkdirp( - path: string, - options?: Parameters[1], - ): Promise { - this.mkdirpModes.set(path, options?.mode); - await super.mkdirp(path, options); - } -} - -// why: spelled out here, not imported: the labels simslim v0.13.0 never disables (ADR 0060 §1). -const neverDisabled = ["com.apple.assistantd", "com.apple.gamed", "com.apple.sharingd"]; - -/** why: What the driver must write for categories ["widgets"]: its labels on, every other managed label off. */ -function wantedOverrides(): Record { - const wanted: Record = {}; - for (const category of SLIM_CATEGORIES) { - for (const label of category.labels) wanted[label] = false; - } - for (const label of neverDisabled) wanted[label] = false; - for (const label of widgetsLabelList()) { - if (!neverDisabled.includes(label)) wanted[label] = true; - } - return wanted; -} - -function widgetsLabelList(): readonly string[] { - return SLIM_CATEGORIES.find((category) => category.name === "widgets")?.labels ?? []; -} - -/** why: The readback of a boot that honoured every label as written. */ -function wantedReadback(): string { - return printDisabled( - Object.fromEntries( - Object.entries(wantedOverrides()).map(([label, off]) => [ - label, - off ? "disabled" : "enabled", - ]), - ), - ); -} - -/** why: The readback of a boot that honoured every label, with the given labels changed or dropped (`undefined`). */ -function readbackWith(changes: Readonly>): string { - const entries: Record = Object.fromEntries( - Object.entries(wantedOverrides()).map(([label, off]) => [label, off ? "disabled" : "enabled"]), - ); - for (const [label, state] of Object.entries(changes)) { - if (state === undefined) delete entries[label]; - else entries[label] = state; - } - return printDisabled(entries); -} - -function scriptedListRunner(): ScriptedProcessRunner { - return new ScriptedProcessRunner([ - { - match: listInvocation, - result: { code: 0, stderr: "", stdout: listFixture }, - }, - ]); -} - -/** - * why: Where macOS keeps the simulator runtimes `xcodebuild -downloadPlatform` fetches. Spelled - * out here rather than imported from the driver, which keeps its own copy of the same path: this - * suite reads the store as the OS lays it out, so a fixture stays a fact about macOS rather than - * about what the driver happens to look at (#79). - */ -const IOS_RUNTIME_ASSET_ROOT = "/System/Library/AssetsV2/com_apple_MobileAsset_iOSSimulatorRuntime"; - -/** - * why: A `MemoryFilesystem` whose `directorySize` answers a set size for the directories in `sizes`, - * as a downloaded runtime bundle of several GiB would; it measures every other path as usual. - */ -class SizedFilesystem extends MemoryFilesystem { - constructor(private readonly sizes: ReadonlyMap) { - super(); - } - - override async directorySize(path: string): Promise { - return this.sizes.get(path) ?? super.directorySize(path); - } -} - -/** - * why: mobileassetd's own metadata for one downloaded simulator runtime, cut down to the keys that - * identify it. `NeverCollected` is verbatim from a real bundle on macOS 26.6.1 / Xcode 27: the - * store is told not to reclaim these, which is why deleting the runtime does not shrink it. - */ -function assetInfoPlist(simulatorVersion: string, build: string): string { - return [ - '', - '', - "", - " MobileAssetProperties", - " ", - " __AssetDefaultGarbageCollectionBehaviorNeverCollected", - ` Build${build}`, - ` SimulatorVersion${simulatorVersion}`, - " ", - "", - "", - "", - ].join("\n"); -} - -const prepareFull = { mode: "full", purpose: "prepare" } as const; -const recoverFull = { mode: "full", purpose: "recover" } as const; -const recoverSlim = { mode: "slim", purpose: "recover" } as const; - -describe("IosSimctlDriver device set", () => { - const coreSimulatorRoot = "/CoreSim"; - const runtimeImage = { - build: "23E244", - identifier: "IMG-26.4-23E244", - platformIdentifier: "com.apple.platform.iphonesimulator", - runtimeIdentifier: "com.apple.CoreSimulator.SimRuntime.iOS-26-4", - sizeBytes: 1_000, - state: "Ready", - version: "26.4", - }; - const imageList = JSON.stringify({ [runtimeImage.identifier]: runtimeImage }); - const emptyDevices = JSON.stringify({ devices: {} }); - - interface Step { - readonly args: readonly string[]; - readonly scoped: boolean; - readonly sources?: true; - readonly stdout?: string; - } - - async function plantDevice(filesystem: Filesystem, devicesDir: string): Promise { - await filesystem.mkdirp(`${devicesDir}/${driverData.udid}/data`); - } - - it.each([ - ["unset", {}, deviceRoot, true, `${deviceRoot}/sources`, false], - ["simlock", { deviceSet: "simlock" }, deviceRoot, true, `${deviceRoot}/sources`, false], - [ - "default", - { deviceSet: "default" }, - `${coreSimulatorRoot}/Devices`, - false, - "/home/.simlock/devices/ios-sources", - true, - ], - ] as const)( - "the driver given deviceSet %s selects simctl scope and whether a ready device needs reboot before grant", - async (_case, config, devicesDir, scoped, sourcesSet, rebootBeforeGrant) => { - const steps: Step[] = [ - { args: ["list", "-j", "devicetypes", "runtimes"], scoped: true, stdout: listFixture }, - { - args: ["create", "simlock-seed-device-1", driverData.deviceTypeId, driverData.runtimeId], - scoped: true, - sources: true, - stdout: `${seedUdid}\n`, - }, - { args: ["boot", seedUdid], scoped: true, sources: true }, - { args: ["bootstatus", seedUdid, "-b"], scoped: true, sources: true }, - { args: ["listapps", seedUdid], scoped: true, sources: true }, - { args: ["shutdown", seedUdid], scoped: true, sources: true }, - { - args: ["clone", seedUdid, driverData.name, devicesDir], - scoped: true, - sources: true, - stdout: `${driverData.udid}\n`, - }, - { args: ["boot", driverData.udid], scoped: true }, - { args: ["bootstatus", driverData.udid, "-b"], scoped: true }, - { args: ["listapps", driverData.udid], scoped: true }, - { args: ["shutdown", driverData.udid], scoped: true }, - { args: ["delete", driverData.udid], scoped: true }, - { args: ["list", "-j", "devices"], scoped: true, stdout: emptyDevices }, - { args: ["runtime", "list", "-j"], scoped: true, stdout: imageList }, - { args: ["list", "-j", "devices"], scoped: false, stdout: emptyDevices }, - { args: ["runtime", "delete", runtimeImage.identifier], scoped: true }, - { args: ["runtime", "list", "-j"], scoped: true, stdout: "{}" }, - { args: ["runtime", "list", "-j"], scoped: true, stdout: "{}" }, - { args: ["list", "-j", "devices"], scoped: false, stdout: emptyDevices }, - ]; - const argvOf = (step: Step): string[] => - step.sources === true - ? ["simctl", "--set", sourcesSet, ...step.args] - : step.scoped && scoped - ? simctlArgs(...step.args) - : ["simctl", ...step.args]; - const runner = new ScriptedProcessRunner( - steps.flatMap((step) => [ - ...(!scoped && step.args[0] === "boot" && step.sources !== true - ? deviceHubEdit( - `Set :DevicePreferences:${driverData.udid}:pasteboardSyncEnabled false`, - ).map((match) => ({ match })) - : []), - { - match: { args: argvOf(step), command: "xcrun" }, - ...(step.stdout === undefined - ? {} - : { result: { code: 0, stderr: "", stdout: step.stdout } }), - }, - ...(!scoped && step.args[0] === "delete" - ? deviceHubEdit(`Delete :DevicePreferences:${driverData.udid}`).map((match) => ({ - match, - })) - : []), - ]), - ); - const filesystem = new MemoryFilesystem(); - await plantDevice(filesystem, devicesDir); - if (scoped) await plantManagedDevice(filesystem); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: scoped ? { deviceRoot, ...config } : config, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - expect(driver.rebootReadyBeforeGrant).toBe(rebootBeforeGrant); - const device = { address: driverData.udid, deviceId: driverData.udid, driverData }; - - await expect( - driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }), - ).resolves.toBeDefined(); - await driver.provision(spec); - await expect(filesystem.exists(`${sourcesSet}/${OWNED_ROOT_MARKER_FILE}`)).resolves.toBe( - !scoped, - ); - await driver.makeReady(device, prepareFull); - await driver.destroy(device); - await driver.listManaged(); - await driver.removeComponent( - { build: runtimeImage.build, image: runtimeImage.identifier }, - { signal: new AbortController().signal }, - ); - - expect( - runner.calls.filter((call) => call.command === "xcrun").map((call) => call.args), - ).toEqual(steps.map(argvOf)); - }, - ); - - it.each([ - ["Default", { deviceSet: "Default" }, "Default", 'but it is "Default"'], - ["the empty string", { deviceSet: "" }, "", 'but it is ""'], - ["a boolean", { deviceSet: true }, "true", "but it is true"], - [ - "default with a deviceRoot", - { deviceSet: "default", deviceRoot: "/Volumes/sims" }, - "default", - 'drivers.ios.deviceRoot names Simlock\'s own set, but drivers.ios.deviceSet is "default". Remove one of them.', - ], - ] as const)( - "refuses iOS with invalid-device-set when deviceSet is %s", - async (_case, config, root, detail) => { - const create = async (driverConfig: Record) => - IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig, - filesystem: new MemoryFilesystem(), - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - simlockHome: "/home/.simlock", - }); - - await expect(create({ deviceSet: "default" })).resolves.toBeInstanceOf(IosSimctlDriver); - const failure = await create(config) - .then(() => undefined) - .catch((error: unknown) => error); - - expect(failure).toBeInstanceOf(OwnedRootError); - expect(failure).toMatchObject({ - message: expect.stringContaining(detail), - path: root, + ).resolves.toEqual({ + physical: false, + model: "iPhone 16", + osVersion: "26.0", platform: "ios", - reason: "invalid-device-set", - }); - expect((failure as Error).message).toMatch(/^Refusing the ios device set: /); - }, - ); - - it("an operator who switches with leftovers gets device-set-changed, and an unreadable folder gets unreadable", async () => { - const ownRoot = "/home/.simlock/devices/ios"; - const xcodeDevices = `${coreSimulatorRoot}/Devices`; - const [first, second, third] = [1, 2, 3].map( - (n) => `00000000-0000-0000-0000-00000000000${String(n)}`, - ); - const mark = (owner: string) => JSON.stringify({ instanceId: owner, token: "t", udid: "u" }); - const steps: readonly [ - string, - Record, - (fs: MemoryFilesystem) => Promise, - { readonly message?: string; readonly path?: string; readonly reason: string } | undefined, - ][] = [ - [ - "default with two devices in Simlock's set", - { deviceSet: "default" }, - async (fs) => { - await fs.mkdirp(`${ownRoot}/${second}`); - await fs.mkdirp(`${ownRoot}/${first}`); - await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); - }, - { - message: `Refusing the ios device set: Simlock's own set at ${ownRoot} still holds 2 simulator(s) (${first}, ${second}). Set drivers.ios.deviceSet back to simlock, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot} delete \`, or, for a seed in the sources set, \`xcrun simctl --set ${ownRoot}/sources delete \`.`, - path: ownRoot, - reason: "device-set-changed", - }, - ], - [ - "default with a seed in the sources set Simlock's own set held", - { deviceSet: "default" }, - async (fs) => { - await fs.mkdirp(`${ownRoot}/sources/${third}`); - await fs.mkdirp(`${ownRoot}/${first}`); - await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); - }, - { - message: `Refusing the ios device set: Simlock's own set at ${ownRoot} still holds 2 simulator(s) (${first}, ${third}). Set drivers.ios.deviceSet back to simlock, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot} delete \`, or, for a seed in the sources set, \`xcrun simctl --set ${ownRoot}/sources delete \`.`, - path: ownRoot, - reason: "device-set-changed", - }, - ], - [ - "simlock with a seed in the sources set deviceSet default used", - { deviceSet: "simlock" }, - async (fs) => { - await fs.mkdirp(`${ownRoot}-sources/${second}`); - }, - { - message: `Refusing the ios device set: Simlock's sources set at ${ownRoot}-sources still holds 1 simulator(s) (${second}). Set drivers.ios.deviceSet back to default, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl --set ${ownRoot}-sources delete \`. Run \`simlock nuke --delete-devices\` again afterwards, so Simlock drops its record of them.`, - path: `${ownRoot}-sources`, - reason: "device-set-changed", - }, - ], - [ - "default with Simlock's set empty", - { deviceSet: "default" }, - async (fs) => { - await fs.mkdirp(ownRoot); - await fs.writeFileAtomic(`${ownRoot}/${OWNED_ROOT_MARKER_FILE}`, "{}"); - }, - undefined, - ], - ["default with no root folder", { deviceSet: "default" }, async () => undefined, undefined], - [ - "simlock with a device of this instance in Xcode's set", - { deviceSet: "simlock" }, - async (fs) => { - await fs.mkdirp(`${xcodeDevices}/${third}`); - await fs.writeFileAtomic(`${xcodeDevices}/${third}/simlock-mark.json`, mark(instanceId)); - }, - { - message: `Refusing the ios device set: Xcode's device set still holds 1 of this Simlock's simulator(s) (${third}). Set drivers.ios.deviceSet back to default, run \`simlock nuke --delete-devices\` and \`simlock doctor --purge-orphans\`, then switch again. Delete any that remain with \`xcrun simctl delete \`.`, - path: xcodeDevices, - reason: "device-set-changed", - }, - ], - [ - "simlock with only unmarked, other-instance and corrupt devices in Xcode's set", - { deviceSet: "simlock" }, - async (fs) => { - await fs.mkdirp(`${xcodeDevices}/${first}`); - await fs.mkdirp(`${xcodeDevices}/${second}`); - await fs.writeFileAtomic(`${xcodeDevices}/${second}/simlock-mark.json`, mark("other")); - await fs.mkdirp(`${xcodeDevices}/${third}`); - await fs.writeFileAtomic(`${xcodeDevices}/${third}/simlock-mark.json`, "{broken"); - }, - undefined, - ], - [ - "default with Simlock's folder unreadable", - { deviceSet: "default" }, - async (fs) => { - await fs.mkdirp(ownRoot); - fs.defineFailure(ownRoot, "EACCES"); - }, - { path: ownRoot, reason: "unreadable" }, - ], - [ - "simlock with Xcode's folder unreadable", - { deviceSet: "simlock" }, - async (fs) => { - await fs.mkdirp(xcodeDevices); - fs.defineFailure(xcodeDevices, "EACCES"); - }, - { path: xcodeDevices, reason: "unreadable" }, - ], - ]; - - for (const [name, config, plant, expected] of steps) { - const filesystem = new MemoryFilesystem(); - await plant(filesystem); - const runner = new ScriptedProcessRunner([]); - const failure = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: config, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }) - .then(() => undefined) - .catch((error: unknown) => error); - - expect(runner.calls, name).toEqual([]); - if (expected === undefined) { - expect(failure, name).toBeUndefined(); - } else { - expect(failure, name).toBeInstanceOf(OwnedRootError); - expect(failure, name).toMatchObject({ ...expected, platform: "ios" }); - } - } - }); - - it.each([ - ["simlock", { deviceRoot, deviceSet: "simlock" }, deviceRoot], - ["default", { deviceSet: "default" }, `${coreSimulatorRoot}/Devices`], - ] as const)( - "a new simulator's marks name this instance under deviceSet %s", - async (setting, config, devicesDir) => { - const scoped = setting === "simlock"; - const argv = (...args: string[]) => (scoped ? simctlArgs(...args) : ["simctl", ...args]); - const marks = [ - `${devicesDir}/${driverData.udid}/data/simlock-mark.json`, - `${devicesDir}/${driverData.udid}/simlock-mark.json`, - ]; - const driverFor = async (steps: readonly (readonly [string[], string?])[]) => { - const runner = new ScriptedProcessRunner( - steps.map(([args, stdout]) => ({ - match: { args, command: "xcrun" }, - ...(stdout === undefined ? {} : { result: { code: 0, stderr: "", stdout } }), - })), - ); - const filesystem = new MemoryFilesystem(); - if (scoped) await plantManagedDevice(filesystem); - else await filesystem.mkdirp(`${devicesDir}/${driverData.udid}/data`); - const driver = await IosSimctlDriver.create({ - database: await seededDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: config, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - return { driver, filesystem, runner }; - }; - const created = [ - argv("list", "-j", "devicetypes", "runtimes"), - [ - "simctl", - "--set", - scoped ? `${deviceRoot}/sources` : "/home/.simlock/devices/ios-sources", - "clone", - seedUdid, - driverData.name, - devicesDir, - ], - ]; - - const { driver, filesystem } = await driverFor([ - [created[0] ?? [], listFixture], - [created[1] ?? [], `${driverData.udid}\n`], - ]); - await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - await driver.provision(spec); - for (const path of marks) { - expect(JSON.parse(await filesystem.readFile(path)), path).toMatchObject({ instanceId }); - } - - if (scoped) return; - const failing = await driverFor([ - [created[0] ?? [], listFixture], - [created[1] ?? [], `${driverData.udid}\n`], - [argv("delete", driverData.udid)], - ]); - await failing.filesystem.rm(`${devicesDir}/${driverData.udid}/data`); - await failing.driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); - await expect(failing.driver.provision(spec)).rejects.toThrow(); - expect(failing.runner.calls.at(-1)).toEqual({ - args: argv("delete", driverData.udid), - command: "xcrun", - options: { timeoutMs: 30_000 }, - }); - }, - ); - - it("under deviceSet default listManaged reports only devices whose durable mark names this instance", async () => { - const devicesDir = `${coreSimulatorRoot}/Devices`; - const udid = (n: number) => `00000000-0000-0000-0000-00000000010${String(n)}`; - const mark = (owner?: string) => - JSON.stringify({ ...(owner === undefined ? {} : { instanceId: owner }), token: "t" }); - const filesystem = new MemoryFilesystem(); - const cases = [ - { durable: mark(instanceId), erasable: mark(instanceId), listed: true, name: "simlock-0" }, - { durable: mark(instanceId), erasable: undefined, listed: true, name: "simlock-9" }, - { durable: undefined, erasable: mark(instanceId), listed: false, name: "no-durable" }, - { durable: mark(), erasable: mark(), listed: false, name: "no-instance" }, - { durable: mark("other"), erasable: mark("other"), listed: false, name: "another" }, - { durable: undefined, erasable: undefined, listed: false, name: "simlock-1" }, - ]; - for (const [index, entry] of cases.entries()) { - await filesystem.mkdirp(`${devicesDir}/${udid(index)}/data`); - if (entry.durable !== undefined) { - await filesystem.writeFileAtomic( - `${devicesDir}/${udid(index)}/simlock-mark.json`, - entry.durable, - ); - } - if (entry.erasable !== undefined) { - await filesystem.writeFileAtomic( - `${devicesDir}/${udid(index)}/data/simlock-mark.json`, - entry.erasable, - ); - } - } - const runner = new ScriptedProcessRunner([ - { - match: { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }, - result: { - code: 0, - stderr: "", - stdout: JSON.stringify({ - devices: { - "com.apple.CoreSimulator.SimRuntime.iOS-26-5": cases.map((entry, index) => ({ - dataPath: `${devicesDir}/${udid(index)}/data`, - name: entry.name, - state: "Shutdown", - udid: udid(index), - })), - }, - }), - }, - }, - ]); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: { deviceSet: "default" }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - - const reality = await driver.listManaged(); - - expect(reality.devices.map((device) => [device.deviceId, device.mark?.erasable])).toEqual([ - [udid(0), "t"], - [udid(1), undefined], - ]); - await expect(driver.findLegacy(udid(0))).resolves.toBeUndefined(); - expect(runner.calls).toHaveLength(1); - }); - - it("under deviceSet default foreignDevices and the removal's unused count take only devices this instance's mark does not name", async () => { - const devicesDir = `${coreSimulatorRoot}/Devices`; - const udids = ["A", "B", "C"].map((n) => `0000000${n}-0000-0000-0000-000000000000`); - const devices = (used: boolean) => - JSON.stringify({ - devices: { - [runtimeImage.runtimeIdentifier]: udids.map((udid) => ({ - ...(used ? { lastUsedAt: "2026-10-01T12:00:00Z" } : {}), - name: "d", - state: "Shutdown", - udid, - })), - }, - }); - const driverWith = async (steps: readonly (readonly [readonly string[], string?])[]) => { - const filesystem = new MemoryFilesystem(); - const marked = [instanceId, undefined, "other"] as const; - for (const [index, udid] of udids.entries()) { - await filesystem.mkdirp(`${devicesDir}/${udid}`); - const owner = marked[index]; - if (owner !== undefined) { - await filesystem.writeFileAtomic( - `${devicesDir}/${udid}/simlock-mark.json`, - JSON.stringify({ instanceId: owner, token: "t" }), - ); - } - } - return IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: { deviceSet: "default" }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner( - steps.map(([args, stdout]) => ({ - match: { args: [...args], command: "xcrun" }, - ...(stdout === undefined ? {} : { result: { code: 0, stderr: "", stdout } }), - })), - ), - simlockHome: "/home/.simlock", - }); - }; - const runtimeList = [["simctl", "runtime", "list", "-j"], imageList] as const; - const deviceList = (used: boolean) => - [["simctl", "list", "-j", "devices"], devices(used)] as const; - - const listing = await driverWith([runtimeList, deviceList(true)]); - const [component] = await listing.listComponents(); - expect(component?.foreignDevices).toBe(2); - - const removing = await driverWith([ - runtimeList, - deviceList(false), - [["simctl", "runtime", "delete", runtimeImage.identifier]], - [["simctl", "runtime", "list", "-j"], "{}"], - [["simctl", "runtime", "list", "-j"], "{}"], - deviceList(false), - ]); - const removal = await removing.removeComponent( - { build: runtimeImage.build, image: runtimeImage.identifier }, - { signal: new AbortController().signal }, - ); - expect(removal.residue).toContain("2 never-used simulators"); - }); - - it.each([ - ["simlock", { deviceRoot, deviceSet: "simlock" }, { SIMLOCK_IOS_DEVICE_SET: deviceRoot }], - ["default", { deviceSet: "default" }, {}], - ] as const)( - "a lease holder and the screen capture get the device set only under deviceSet %s", - async (_setting, config, leaseEnvironment) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/dist/bin"); - await filesystem.writeFileAtomic("/dist/bin/simlock-screen-capture", ""); - const launcher = new FakeBytePipeLauncher(); - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: config, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - screenCapture: { - baseEnvironment: { PATH: "/usr/bin", SIMLOCK_IOS_DEVICE_SET: "/from/the/daemon" }, - command: "/dist/bin/simlock-screen-capture", - launcher, - }, - simlockHome: "/home/.simlock", - }); - const capture = driver.screenCapture; - if (capture === undefined) throw new Error("the driver offers no screen capture"); - - await capture.open( - { address: driverData.udid, deviceId: driverData.udid, driverData, physical: false }, - { bitrateKbps: 1_500, fps: 30, maxDimension: 720 }, - new AbortController().signal, - ); - - expect(driver.virtualLeaseEnvironment()).toEqual(leaseEnvironment); - expect( - driver.leaseEnvironment({ - address: driverData.udid, - deviceId: driverData.udid, - driverData, - }), - ).toEqual(leaseEnvironment); - expect(driver.leaseEnvironment(PHYSICAL_DEVICE)).toEqual({ - SIMLOCK_DEVICE_UDID: PHYSICAL_DEVICE.deviceId, - }); - expect(launcher.starts[0]?.env).toEqual({ - PATH: "/usr/bin", - ...leaseEnvironment, }); - }, - ); - - it("simlock simctl under deviceSet default runs plain xcrun simctl and keeps its refusals", async () => { - const driver = await IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: new FakeClock(), - coreSimulatorRoot, - driverConfig: { deviceSet: "default" }, - filesystem: new MemoryFilesystem(), - idGenerator: { generate: () => "device-1" }, - instanceId, - processRunner: new ScriptedProcessRunner([]), - simlockHome: "/home/.simlock", - }); - - await expect( - simctlPassthrough(driver, ["install", driverData.udid, "app.app"]), - ).resolves.toEqual({ - args: ["simctl", "install", driverData.udid, "app.app"], - command: "xcrun", - env: {}, }); - for (const refused of [ - ["erase", driverData.udid], - ["shutdown", "all"], - ["--set", "/tmp", "list"], - ]) { - await expect(simctlPassthrough(driver, refused), refused.join(" ")).rejects.toBeInstanceOf( - PassthroughRefusedError, - ); - } }); }); - -describe("IosSimctlDriver simulator audio", () => { - const volumeState = "com.apple.springboard.volumestate"; - const udid = driverData.udid; - const slimData = { ...driverData, runtimeId: "com.apple.CoreSimulator.SimRuntime.iOS-18-5" }; - - const silence = [ - simctlArgs("spawn", udid, "notifyutil", "-s", volumeState, "0"), - simctlArgs("spawn", udid, "notifyutil", "-p", volumeState), - ]; - const bootCalls = [ - simctlArgs("boot", udid), - simctlArgs("bootstatus", udid, "-b"), - simctlArgs("listapps", udid), - ]; - const readbackCall = simctlArgs("spawn", udid, "launchctl", "print-disabled", "system"); - - it.each([ - { - name: "prepare", - options: { mode: "full", purpose: "prepare" } as const, - expected: [...bootCalls, ...silence], - }, - { - name: "recover", - options: { mode: "full", purpose: "recover" } as const, - expected: [...bootCalls, ...silence], - }, - { - name: "a slim prepare, silenced before the readback", - data: slimData, - options: { mode: "slim", purpose: "prepare" } as const, - expected: [...bootCalls, ...silence, readbackCall], - }, - { - name: "the setting true", - audio: true, - options: { mode: "full", purpose: "prepare" } as const, - expected: bootCalls, - }, - { - name: "a volume command that fails", - volume: { code: 1, stderr: "boom", stdout: "" }, - options: { mode: "full", purpose: "prepare" } as const, - expected: [...bootCalls, silence[0]], - failure: "boom", - }, - { - name: "a volume command that times out", - volume: "hangs" as const, - options: { mode: "full", purpose: "prepare" } as const, - expected: [...bootCalls, silence[0]], - failure: "timed out", - }, - ])( - "sets a simulator's volume to 0 before handing it on when ios.simulator.audio is false: $name", - async (c) => { - const data = c.data ?? driverData; - const silenceCall = (index: number, result?: ProcessResult) => ({ - match: { args: silence[index]!, command: "xcrun" }, - ...(result === undefined ? {} : { result }), - }); - const runner = new ScriptedProcessRunner([ - { match: simctl("boot", udid) }, - { match: simctl("bootstatus", udid, "-b") }, - { match: simctl("listapps", udid) }, - ...(c.audio === true - ? [] - : c.volume === "hangs" - ? [{ ...silenceCall(0), hangs: true }] - : c.volume !== undefined - ? [silenceCall(0, c.volume)] - : [silenceCall(0), silenceCall(1)]), - ...(c.options.mode === "slim" - ? [ - { - match: simctl("spawn", udid, "launchctl", "print-disabled", "system"), - result: { code: 0, stderr: "", stdout: wantedReadback() }, - }, - ] - : []), - ]); - scopedRunners.push(runner); - const clock = new FakeClock(); - const failures: SilenceFailedFact[] = []; - const driver = await IosSimctlDriver.create({ - clock, - database: await memorySourceDatabase(), - driverConfig: { deviceRoot }, - filesystem: new MemoryFilesystem(), - idGenerator: { generate: () => "device-1" }, - instanceId, - launchdOverridesRoot: overridesRoot, - onSilenceFailed: (fact) => failures.push(fact), - processRunner: runner, - simlockHome: "/home/.simlock", - config: { - simulator: { audio: c.audio ?? false, createTimeoutMs: 30_000, headless: true }, - slim: { bootTimeoutMs: 300_000, categories: ["widgets"] }, - }, - }); - - const ready = driver.makeReady( - { address: udid, deviceId: udid, driverData: data }, - c.options, - ); - if (c.volume === "hangs") { - const state = promiseState(ready); - await waitForCalls(runner, c.expected.length); - clock.advance(29_999); - await flushMicrotasks221(); - expect(state.state).toBe("pending"); - clock.advance(1); - await flushMicrotasks221(); - expect(state.state).toBe("fulfilled"); - } - const device = await ready; - - expect(device.address).toBe(udid); - expect(runner.calls.map((call) => call.args)).toEqual(c.expected); - expect(failures).toEqual( - c.failure === undefined ? [] : [{ deviceId: udid, detail: c.failure }], - ); - }, - ); -}); diff --git a/src/drivers/ios/test-index.ts b/src/drivers/ios/test-index.ts new file mode 100644 index 000000000..4c50a1da3 --- /dev/null +++ b/src/drivers/ios/test-index.ts @@ -0,0 +1,360 @@ +import { readFileSync } from "node:fs"; +import { + ScriptedProcessRunner, + type SqliteDatabase, + FakeClock, + type Filesystem, + MemoryFilesystem, + type Logger, +} from "../../ports/index.js"; +import { IosSimctlDriver, SLIM_CATEGORIES } from "./index.js"; +import { plantReadySeed, memorySourceDatabase } from "./testing.js"; +import { + simctl, + deviceRoot, + driverData, + dataPath, + instanceId, + printDisabled, +} from "./test-slim.js"; + +export const listFixture = readFileSync( + new URL("./fixtures/simctl-list.json", import.meta.url), + "utf8", +); + +export const listInvocation = simctl("list", "-j", "devicetypes", "runtimes"); + +export const listDevicesInvocation = simctl("list", "-j", "devices"); + +// why: iPhone 16 pairs with both installed runtimes; iPhone Xs pairs only with the older one (iOS 26 +// dropped it, mirroring real Xcode 27) and its range caps out at iOS 18.6; iPhone 7 caps out at +// iOS 15.0, below the 16.0 auto-download floor. +export const pairingFixture = readFileSync( + new URL("./fixtures/simctl-list-pairing.json", import.meta.url), + "utf8", +); + +// why: Same catalog as `listFixture`, plus an iOS 18.6 runtime that has just finished downloading -- +// stands in for the re-scanned catalog `resolveSpec` reads after a successful `xcodebuild` call. +export const listFixtureAfterDownload = JSON.stringify({ + devicetypes: (JSON.parse(listFixture) as { devicetypes: unknown }).devicetypes, + runtimes: [ + ...(JSON.parse(listFixture) as { runtimes: unknown[] }).runtimes, + { + identifier: "com.apple.CoreSimulator.SimRuntime.iOS-18-6", + isAvailable: true, + name: "iOS 18.6", + supportedDeviceTypes: [ + { identifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", name: "iPhone 16" }, + ], + version: "18.6", + }, + ], +}); + +export const spec = { + physical: false, + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", +} as const; + +/** why: Every simctl call the driver makes is scoped to its device set; so is every expectation. */ + +export const scopedRunners: ScriptedProcessRunner[] = []; + +export const legacyUdid = "00000000-0000-0000-0000-0000000000ff"; + +/** + * why: The complete argv of every call the pre-root path is allowed to make without `--set`, + * spelled out rather than pattern-matched: `findLegacy` / `destroyLegacy` deliberately + * address the machine's default set (ADR 0001, Migration), and `listComponents` reads it with + * the same `list` to count foreign devices (ADR 0010 §8). Every *other* call must still fail + * the invariant below if its scoping ever goes missing. + */ +export const UNSCOPED_LEGACY_CALLS = new Set( + [ + ["simctl", "list", "-j", "devices"], + ["simctl", "shutdown", legacyUdid], + ["simctl", "delete", legacyUdid], + ].map((argv) => argv.join(" ")), +); + +export const seedUdid = "00000000-0000-0000-0000-0000000000aa"; + +/** why: A seed lives in the sources set, a sibling of the lease device set, and its calls are scoped to that. */ +export function sourcesSimctl(...args: readonly string[]): { + readonly command: string; + readonly args: string[]; +} { + return { args: ["simctl", "--set", `${deviceRoot}/sources`, ...args], command: "xcrun" }; +} + +/** why: A database that already holds the seed of the kind `spec` names, so a test of the copy step starts at the clone. */ +export async function seededDatabase( + seed: { readonly runtimeBuild?: string; readonly udid?: string } = {}, +): Promise { + return plantReadySeed(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + runtimeId: driverData.runtimeId, + udid: seedUdid, + ...seed, + }); +} + +export function deviceListResponse(state: string): string { + return JSON.stringify({ + devices: { + "com.apple.CoreSimulator.SimRuntime.iOS-26-5": [ + { name: driverData.name, udid: driverData.udid, state, dataPath }, + ], + }, + }); +} + +export const PHYSICAL_DEVICE = { + address: "00008110-001A2B3C4D5E", + deviceId: "00008110-001A2B3C4D5E", + driverData: {}, + enrolledApps: ["com.example.kept"], + physical: true, +} as const; + +export function slimOptions(bootTimeoutMs = 300_000) { + return { bootTimeoutMs, categories: ["widgets"] } as const; +} + +export const runtimeListInvocation = simctl("runtime", "list", "-j"); + +export const defaultSetInvocation = { args: ["simctl", "list", "-j", "devices"], command: "xcrun" }; + +export interface Image { + readonly id: string; + readonly runtime: string; + readonly build: string; + readonly version: string; + readonly sizeBytes?: number; + readonly platform?: string; +} + +export function image(version: string, build: string, extra: Partial = {}): Image { + return { + build, + id: `IMG-${version}-${build}`, + runtime: `com.apple.CoreSimulator.SimRuntime.iOS-${version.replace(".", "-")}`, + version, + ...extra, + }; +} + +export function imagesListed(images: readonly Image[], state = "Ready") { + const stdout = JSON.stringify( + Object.fromEntries( + images.map((entry) => [ + entry.id, + { + build: entry.build, + identifier: entry.id, + platformIdentifier: entry.platform ?? "com.apple.platform.iphonesimulator", + runtimeIdentifier: entry.runtime, + sizeBytes: entry.sizeBytes ?? 1_000, + state, + version: entry.version, + }, + ]), + ), + ); + return { match: runtimeListInvocation, result: { code: 0, stderr: "", stdout } }; +} + +/** + * why: The default set's devices per runtime: a number is that many used devices, or name both + * kinds. A used device carries `lastUsedAt`, as `simctl list -j devices` reports one that has + * been booted; an unused one -- like the batch CoreSimulator creates when a runtime install + * ends -- carries every other key a real entry has and no `lastUsedAt`. + */ +export function defaultSetListed( + devices: Readonly>, +) { + const entry = (runtime: string, index: number, used: boolean) => ({ + dataPath: `/Users/me/Library/Developer/CoreSimulator/Devices/${runtime}-${String(index)}/data`, + dataPathSize: used ? 1_500_000_000 : 18_000_000, + deviceTypeIdentifier: "com.apple.CoreSimulator.SimDeviceType.iPhone-16", + isAvailable: true, + ...(used ? { lastUsedAt: "2026-10-01T12:00:00Z" } : {}), + logPath: `/Users/me/Library/Logs/CoreSimulator/${runtime}-${String(index)}`, + logPathSize: 4096, + name: `user-${String(index)}`, + state: "Shutdown", + udid: `USER-${runtime}-${String(index)}`, + }); + const stdout = JSON.stringify({ + devices: Object.fromEntries( + Object.entries(devices).map(([runtime, counts]) => { + const { unused = 0, used = 0 } = typeof counts === "number" ? { used: counts } : counts; + return [ + runtime, + [ + ...Array.from({ length: used }, (_, index) => entry(runtime, index, true)), + ...Array.from({ length: unused }, (_, index) => entry(runtime, used + index, false)), + ], + ]; + }), + ), + }); + return { match: defaultSetInvocation, result: { code: 0, stderr: "", stdout } }; +} + +export function simctlPassthrough(driver: IosSimctlDriver, args: readonly string[]) { + return driver.passthrough("simctl", undefined, args); +} + +export async function createDriver( + runner: ScriptedProcessRunner, + clock = new FakeClock(), + filesystem: Filesystem = new MemoryFilesystem(), + database?: SqliteDatabase, + logger?: Logger, + fullBootTimeoutMs?: number, +): Promise { + scopedRunners.push(runner); + return IosSimctlDriver.create({ + database: database ?? (await memorySourceDatabase()), + clock, + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + ...(fullBootTimeoutMs === undefined + ? {} + : { config: { full: { bootTimeoutMs: fullBootTimeoutMs } } }), + ...(logger === undefined ? {} : { logger }), + processRunner: runner, + simlockHome: "/home/.simlock", + }); +} + +/** why: a LaunchServices database file and a neighbour that must stay, in the device folder `device`. */ +export async function plantLaunchServices( + filesystem: MemoryFilesystem, + device: string, +): Promise { + const folder = `${device}/data/var/db/lsd`; + await filesystem.mkdirp(folder); + await filesystem.writeFileAtomic(`${folder}/com.apple.LaunchServices-33554481-v2.csstore`, "x"); + await filesystem.writeFileAtomic( + `${folder}/com.apple.LaunchServices.unregisteredtypes-501.db`, + "", + ); +} + +export async function launchServicesFiles( + filesystem: MemoryFilesystem, + device: string, +): Promise { + return (await filesystem.readdir(`${device}/data/var/db/lsd`)).sort(); +} + +/** why: Records every `warn` message, so the destroy's log line is asserted on its text. */ +export class RecordingLogger implements Logger { + readonly warnings: string[] = []; + + debug(): void {} + info(): void {} + warn(message: string): void { + this.warnings.push(message); + } + error(): void {} + child(): Logger { + return this; + } +} + +// why: spelled out here, not imported: the labels simslim v0.13.0 never disables (ADR 0060 §1). +export const neverDisabled = ["com.apple.assistantd", "com.apple.gamed", "com.apple.sharingd"]; + +/** why: What the driver must write for categories ["widgets"]: its labels on, every other managed label off. */ +export function wantedOverrides(): Record { + const wanted: Record = {}; + for (const category of SLIM_CATEGORIES) { + for (const label of category.labels) wanted[label] = false; + } + for (const label of neverDisabled) wanted[label] = false; + for (const label of widgetsLabelList()) { + if (!neverDisabled.includes(label)) wanted[label] = true; + } + return wanted; +} + +export function widgetsLabelList(): readonly string[] { + return SLIM_CATEGORIES.find((category) => category.name === "widgets")?.labels ?? []; +} + +/** why: The readback of a boot that honoured every label as written. */ +export function wantedReadback(): string { + return printDisabled( + Object.fromEntries( + Object.entries(wantedOverrides()).map(([label, off]) => [ + label, + off ? "disabled" : "enabled", + ]), + ), + ); +} + +export function scriptedListRunner(): ScriptedProcessRunner { + return new ScriptedProcessRunner([ + { + match: listInvocation, + result: { code: 0, stderr: "", stdout: listFixture }, + }, + ]); +} + +/** + * why: Where macOS keeps the simulator runtimes `xcodebuild -downloadPlatform` fetches. Spelled + * out here rather than imported from the driver, which keeps its own copy of the same path: this + * suite reads the store as the OS lays it out, so a fixture stays a fact about macOS rather than + * about what the driver happens to look at (#79). + */ +export const IOS_RUNTIME_ASSET_ROOT = + "/System/Library/AssetsV2/com_apple_MobileAsset_iOSSimulatorRuntime"; + +/** + * why: A `MemoryFilesystem` whose `directorySize` answers a set size for the directories in `sizes`, + * as a downloaded runtime bundle of several GiB would; it measures every other path as usual. + */ +export class SizedFilesystem extends MemoryFilesystem { + constructor(private readonly sizes: ReadonlyMap) { + super(); + } + + override async directorySize(path: string): Promise { + return this.sizes.get(path) ?? super.directorySize(path); + } +} + +/** + * why: mobileassetd's own metadata for one downloaded simulator runtime, cut down to the keys that + * identify it. `NeverCollected` is verbatim from a real bundle on macOS 26.6.1 / Xcode 27: the + * store is told not to reclaim these, which is why deleting the runtime does not shrink it. + */ +export function assetInfoPlist(simulatorVersion: string, build: string): string { + return [ + '', + '', + "", + " MobileAssetProperties", + " ", + " __AssetDefaultGarbageCollectionBehaviorNeverCollected", + ` Build${build}`, + ` SimulatorVersion${simulatorVersion}`, + " ", + "", + "", + "", + ].join("\n"); +} + +export const prepareFull = { mode: "full", purpose: "prepare" } as const; diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index d950c8237..d8887b03d 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -28,7 +28,6 @@ type SizedFile = { path: string; lines: number }; */ const SHRINK_ONLY: Record = { "src/cli/index.test.ts": 11674, - "src/drivers/ios/index.test.ts": 6208, "src/drivers/ios/index.ts": 2700, }; From 2a24e5c22ac020d0ff1d6bb5c73417710c9143ed Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:25:09 +0100 Subject: [PATCH 25/83] test(ios): split physical.test.ts by topic --- .../ios/physical-files-and-limits.test.ts | 142 ++++ src/drivers/ios/physical-presence.test.ts | 315 +++++++ src/drivers/ios/physical.test.ts | 780 +----------------- src/drivers/ios/test-physical.ts | 355 ++++++++ 4 files changed, 842 insertions(+), 750 deletions(-) create mode 100644 src/drivers/ios/physical-files-and-limits.test.ts create mode 100644 src/drivers/ios/physical-presence.test.ts create mode 100644 src/drivers/ios/test-physical.ts diff --git a/src/drivers/ios/physical-files-and-limits.test.ts b/src/drivers/ios/physical-files-and-limits.test.ts new file mode 100644 index 000000000..069f67f1c --- /dev/null +++ b/src/drivers/ios/physical-files-and-limits.test.ts @@ -0,0 +1,142 @@ +import { readFileSync } from "node:fs"; +import { describe, it, expect } from "vitest"; +import { promiseState } from "../../test-support/promise-state.js"; +import { + DEVICE_ROOT, + Lab, + PHONE, + PHONE_IDENTIFIER, + PHONE_UDID, + READ_LIMIT_MS, + ReadScript, + SIMLOCK_HOME, + UDID_A, + UDID_B, + enrollableLab, + expireAfter, + inspect, + listDocument, + listed, + lockDocument, + turns, + waitForCalls, + wire, +} from "./test-physical.js"; + +const simctlListFixture = readFileSync( + new URL("./fixtures/simctl-list.json", import.meta.url), + "utf8", +); + +const JSON_DIR = `${SIMLOCK_HOME}/devicectl`; + +describe("IosSimctlDriver physical handler: files and limits", () => { + it("every devicectl read passes --json-output naming a new file in /devicectl/ (the driver creates it, mode 0700), and the driver reads that file and then deletes it", async () => { + const lab = enrollableLab([wire(PHONE)], PHONE_UDID, ["com.example.MyApp"]); + + const result = await inspect(lab, PHONE_UDID); + + expect(result).toMatchObject({ apps: ["com.example.MyApp"], enrollable: true }); + expect(lab.written).toHaveLength(3); + expect(new Set(lab.written).size).toBe(3); + for (const path of lab.written) { + expect(path).toMatch(/^\/home\/\.simlock\/devicectl\/[^/]+$/); + expect(await lab.filesystem.exists(path)).toBe(false); + } + const directory = await lab.filesystem.lstat(JSON_DIR); + expect(directory.kind).toBe("directory"); + expect(directory.mode & 0o777).toBe(0o700); + }); + + it.each([ + { expires: false, script: { exit: 1 } as ReadScript, title: "exits non-zero" }, + { expires: true, script: { hangs: true } as ReadScript, title: "passes its limit" }, + ])("the file is deleted also after a read that $title", async ({ expires, script }) => { + const lab = new Lab().list(listDocument([wire(PHONE)]), { + file: listDocument([wire(PHONE)]), + ...script, + }); + const driver = await lab.driver(); + + await expireAfter( + lab, + driver.inspectPhysical(PHONE_UDID), + expires ? 1 : 0, + expires ? READ_LIMIT_MS : 0, + ); + + expect(lab.written).toHaveLength(1); + expect(await lab.filesystem.exists(lab.written[0] ?? "")).toBe(false); + }); + + it("two lock reads in parallel name two different files", async () => { + const lab = new Lab() + .list([listed(UDID_A), listed(UDID_B)]) + .lock(UDID_A) + .lock(UDID_B); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); + + expect(result).toMatchObject([{ present: true }, { present: true }]); + expect(lab.written).toHaveLength(3); + expect(new Set(lab.written).size).toBe(3); + }); + + it.each([ + { + lab: () => + new Lab() + .list([wire(PHONE)]) + .lock(PHONE_UDID, { file: lockDocument({ passcodeRequired: "false" }) }) + .apps(PHONE_UDID, ["com.example.MyApp"]), + name: "lockState", + title: "passcodeRequired as a string", + }, + { + lab: () => + new Lab() + .list([wire({ ...PHONE, udid: 8110 })]) + .lock(PHONE_UDID) + .apps(PHONE_UDID, ["com.example.MyApp"]), + name: "list devices", + title: "udid as a number", + }, + ])( + "a read whose file holds a field of the wrong JSON type ($title) is a failed read", + async ({ lab, name }) => { + const accepted = await inspect(enrollableLab(), PHONE_IDENTIFIER); + expect(accepted).toMatchObject({ enrollable: true }); + + const result = await inspect(lab(), PHONE_IDENTIFIER); + + expect(result).toMatchObject({ + enrollable: false, + message: expect.stringContaining(name), + reason: "unreadable", + }); + }, + ); + + it("a devicectl read that hangs does not delay a simctl call made at the same time", async () => { + const lab = new Lab().list([wire(PHONE)], { hangs: true }).add({ + match: { + args: ["simctl", "--set", DEVICE_ROOT, "list", "-j", "devicetypes", "runtimes"], + command: "xcrun", + }, + result: { code: 0, stderr: "", stdout: simctlListFixture }, + }); + const driver = await lab.driver(); + const read = driver.inspectPhysical(PHONE_UDID); + void read.catch(() => undefined); + await waitForCalls(lab.runner, 1); + const readState = promiseState(read); + + const resolving = driver.resolveSpec({ model: "iPhone 16", platform: "ios" }); + const resolvingState = promiseState(resolving); + await turns(5); + + expect(readState.state).toBe("pending"); + expect(resolvingState.state).toBe("fulfilled"); + expect(await resolving).toMatchObject({ model: "iPhone 16", physical: false }); + }); +}); diff --git a/src/drivers/ios/physical-presence.test.ts b/src/drivers/ios/physical-presence.test.ts new file mode 100644 index 000000000..713e26f7c --- /dev/null +++ b/src/drivers/ios/physical-presence.test.ts @@ -0,0 +1,315 @@ +import { describe, it, expect } from "vitest"; +import { + type ProcessRunner, + ScriptedProcessRunner, + FakeClock, + type ProcessRunOptions, + type ProcessResult, + type ProcessHandle, + type ProcessStreamOptions, + type StreamingProcessHandle, +} from "../../ports/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { + FAILURES, + LIST_ARGV, + LOCKED, + Lab, + OPEN_LOCK, + READ_LIMIT_MS, + UDID_A, + UDID_B, + expireAfter, + listDocument, + listed, + lockArgv, + lockDocument, + turns, + waitForCalls, +} from "./test-physical.js"; + +class LockReadsTakeTime implements ProcessRunner { + constructor( + private readonly inner: ScriptedProcessRunner, + private readonly clock: FakeClock, + private readonly takesMs: number, + ) {} + + run( + command: string, + args: readonly string[], + options?: ProcessRunOptions, + ): Promise { + return this.spawn(command, args, options).wait(); + } + + spawn(command: string, args: readonly string[], options?: ProcessRunOptions): ProcessHandle { + const handle = this.inner.spawn(command, args, options); + if (!args.includes("lockState")) return handle; + return { + kill: (signal) => { + handle.kill(signal); + }, + pid: handle.pid, + stderr: handle.stderr, + stdout: handle.stdout, + unref: () => { + handle.unref(); + }, + wait: () => + new Promise((resolve, reject) => { + this.clock.setTimer(this.takesMs, () => { + handle.wait().then(resolve, reject); + }); + }), + }; + } + + spawnStreaming( + command: string, + args: readonly string[], + options: ProcessStreamOptions, + ): StreamingProcessHandle { + return this.inner.spawnStreaming(command, args, options); + } +} + +const PHONE_LOCK_READ_FAILURES = FAILURES.filter((failure) => !failure.expires); + +const UDID_C = "00008110-00000000000C"; + +const UDID_D = "00008110-00000000000D"; + +describe("IosSimctlDriver physical handler: listAttachedPhysical", () => { + it("lists the wired phones in any pairing state, once each, none on the network, on this machine or simulated, and throws when the list read times out, exits non-zero or has no result.devices", async () => { + const lab = new Lab().list([ + listed(UDID_B, { pairingState: "unpaired" }), + listed(UDID_A), + listed(UDID_A, { transportType: "localNetwork" }), + listed(UDID_C, { transportType: "localNetwork" }), + listed(UDID_D, { transportType: "sameMachine" }), + listed("00008110-00000000000E", { reality: "simulated" }), + ]); + + const attached = await (await lab.driver()).listAttachedPhysical(); + + expect(attached).toEqual([UDID_A, UDID_B]); + expect(lab.calls()).toEqual([LIST_ARGV]); + + const document = listDocument([listed(UDID_A)]); + const reads = [ + { + lab: new Lab().list([], { file: document, hangs: true }), + reason: /timed out after 30000ms/, + }, + { lab: new Lab().list([], { exit: 1, file: document }), reason: /exited 1/ }, + { lab: new Lab().list('{"result":{}}'), reason: /no result\.devices list/ }, + ]; + for (const { lab, reason } of reads) { + const pending = (await lab.driver()).listAttachedPhysical(); + + const settled = reason.test("timed out after 30000ms") + ? expireAfter(lab, pending, 1, READ_LIMIT_MS) + : pending; + + await expect(settled).rejects.toThrow(reason); + } + }); +}); + +describe("IosSimctlDriver physical handler: presence", () => { + it("listPresentPhysical answers a paired, wired, unlocked given iPhone present, with its OS version", async () => { + const lab = new Lab().list([listed(UDID_A)]).lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A]); + + expect(result).toEqual([{ driverDeviceId: UDID_A, osVersion: "18.4", present: true }]); + }); + + it("listPresentPhysical answers a paired, wired, unlocked given iPhone with no reality present", async () => { + const lab = new Lab().list([listed(UDID_A, { reality: undefined })]).lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A]); + + expect(result).toEqual([{ driverDeviceId: UDID_A, osVersion: "18.4", present: true }]); + }); + + it("listPresentPhysical answers a wired, paired, simulated entry not present not-present, with no lock read", async () => { + const lab = new Lab().list([listed(UDID_A, { reality: "simulated" })]); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A]); + + expect(result).toEqual([{ driverDeviceId: UDID_A, present: false, reason: "not-present" }]); + expect(lab.calls()).toEqual([LIST_ARGV]); + }); + + it("listPresentPhysical answers a locked iPhone not present locked, an unpaired one untrusted, and a Wi-Fi-only one and one not listed not-present", async () => { + const lab = new Lab() + .list([ + listed(UDID_A), + listed(UDID_B, { pairingState: "unpaired" }), + listed(UDID_C, { transportType: "localNetwork" }), + ]) + .lock(UDID_A, { file: LOCKED }); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B, UDID_C, UDID_D]); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, present: false, reason: "locked" }, + { driverDeviceId: UDID_B, present: false, reason: "untrusted" }, + { driverDeviceId: UDID_C, present: false, reason: "not-present" }, + { driverDeviceId: UDID_D, present: false, reason: "not-present" }, + ]); + expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); + }); + + it("listPresentPhysical gives no entry for, and never runs a lock read for, a physical device not among the UDIDs it was given", async () => { + const lab = new Lab().list([listed(UDID_A), listed(UDID_B)]).lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A]); + + expect(result).toMatchObject([{ driverDeviceId: UDID_A, present: true }]); + expect(result).toHaveLength(1); + expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); + }); + + it("listPresentPhysical ignores a list entry with no udid: it runs no lock read for it, and every given UDID reads as it would without that entry", async () => { + const lab = new Lab() + .list([ + listed(UDID_A, { udid: undefined }), + listed(UDID_A), + listed(UDID_B, { udid: undefined }), + ]) + .lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, present: true }, + { driverDeviceId: UDID_B, present: false, reason: "not-present" }, + ]); + expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); + }); + + it("listPresentPhysical of a given UDID listed once over Wi-Fi and once wired reads the wired entry; one listed wired twice reads not-present with no lock read", async () => { + const lab = new Lab() + .list([ + listed(UDID_A, { osVersionNumber: "17.0", transportType: "localNetwork" }), + listed(UDID_A), + listed(UDID_B), + listed(UDID_B), + ]) + .lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, osVersion: "18.4", present: true }, + { driverDeviceId: UDID_B, present: false, reason: "not-present" }, + ]); + expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); + }); + + it.each(FAILURES)( + "listPresentPhysical whose list read $title answers every given UDID not present not-present, and does not reject", + async (failure) => { + const lab = new Lab().list([listed(UDID_A), listed(UDID_B)], { + ...failure.script(listDocument([listed(UDID_A), listed(UDID_B)])), + }); + const driver = await lab.driver(); + const pending = driver.listPresentPhysical([UDID_A, UDID_B]); + + const result = await expireAfter( + lab, + pending, + failure.expires ? 1 : 0, + failure.expires ? READ_LIMIT_MS : 0, + ); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, present: false, reason: "not-present" }, + { driverDeviceId: UDID_B, present: false, reason: "not-present" }, + ]); + expect(lab.timeouts()).toEqual([READ_LIMIT_MS]); + }, + ); + + it("one lock read past 30 s answers that device not present not-present and the others as read", async () => { + const lab = new Lab() + .list([listed(UDID_A), listed(UDID_B)]) + .lock(UDID_A, { file: OPEN_LOCK, hangs: true }) + .lock(UDID_B); + const driver = await lab.driver(); + + const result = await expireAfter( + lab, + driver.listPresentPhysical([UDID_A, UDID_B]), + 3, + READ_LIMIT_MS, + ); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, present: false, reason: "not-present" }, + { driverDeviceId: UDID_B, osVersion: "18.4", present: true }, + ]); + expect(lab.timeouts()).toEqual([READ_LIMIT_MS, READ_LIMIT_MS, READ_LIMIT_MS]); + }); + + it.each([ + ...PHONE_LOCK_READ_FAILURES.map((failure) => ({ + script: failure.script(OPEN_LOCK), + title: failure.title, + })), + { + script: { file: lockDocument({ passcodeRequired: "false" }) }, + title: "lacks a boolean passcodeRequired", + }, + ])("a lock read that $title answers that device not present not-present", async ({ script }) => { + const lab = new Lab() + .list([listed(UDID_A), listed(UDID_B)]) + .lock(UDID_A, script) + .lock(UDID_B); + + const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_A, present: false, reason: "not-present" }, + { driverDeviceId: UDID_B, present: true }, + ]); + }); + + it("two lock reads that each take 25 s (fake clock) both finish, and listPresentPhysical resolves before 30 s have passed after the list read: they run in parallel", async () => { + const lab = new Lab() + .list([listed(UDID_A), listed(UDID_B)]) + .lock(UDID_A) + .lock(UDID_B); + const driver = await lab.driver(new LockReadsTakeTime(lab.runner, lab.clock, 25_000)); + const pending = driver.listPresentPhysical([UDID_A, UDID_B]); + const state = promiseState(pending); + void pending.catch(() => undefined); + + await waitForCalls(lab.runner, 3); + lab.clock.advance(25_000); + await turns(5); + + expect(state.state).toBe("fulfilled"); + expect(await pending).toMatchObject([ + { driverDeviceId: UDID_A, present: true }, + { driverDeviceId: UDID_B, present: true }, + ]); + }); + + it("a list entry for a given UDID with no osVersionNumber answers that UDID not present not-present", async () => { + const lab = new Lab() + .list([listed(UDID_B), listed(UDID_A, { osVersionNumber: undefined })]) + .lock(UDID_B) + .lock(UDID_A); + + const result = await (await lab.driver()).listPresentPhysical([UDID_B, UDID_A]); + + expect(result).toMatchObject([ + { driverDeviceId: UDID_B, osVersion: "18.4", present: true }, + { driverDeviceId: UDID_A, present: false, reason: "not-present" }, + ]); + }); +}); diff --git a/src/drivers/ios/physical.test.ts b/src/drivers/ios/physical.test.ts index fbfaaf9b1..807280550 100644 --- a/src/drivers/ios/physical.test.ts +++ b/src/drivers/ios/physical.test.ts @@ -1,75 +1,45 @@ import { readFileSync } from "node:fs"; - -import { describe, expect, it } from "vitest"; - -import type { DeviceSpec, PhysicalDriverDevice, PhysicalInspection } from "../../core/index.js"; -import { - FakeBytePipeLauncher, - FakeClock, - MemoryFilesystem, - ScriptedProcessRunner, - type ProcessHandle, - type ProcessResult, - type ProcessRunner, - type ProcessRunOptions, - type ProcessStreamOptions, - type ScriptedProcessExpectation, - type StreamingProcessHandle, -} from "../../ports/index.js"; -import { promiseState } from "../../test-support/promise-state.js"; +import { expect, describe, it } from "vitest"; +import type { PhysicalDriverDevice, PhysicalInspection, DeviceSpec } from "../../core/index.js"; +import { FakeBytePipeLauncher } from "../../ports/index.js"; import { IosSimctlDriver } from "./index.js"; -import { memorySourceDatabase } from "./testing.js"; - -type Json = string | number | boolean | null | undefined | Json[] | { [key: string]: Json }; -type Fields = Record; +import { + FAILURES, + Fields, + LIST_ARGV, + LOCKED, + Lab, + OPEN_LOCK, + PHONE, + PHONE_ECID, + PHONE_IDENTIFIER, + PHONE_SERIAL, + PHONE_UDID, + PROPERTIES_GROUP, + READ_LIMIT_MS, + ReadScript, + appsDocument, + enrollableLab, + expireAfter, + inspect, + listDocument, + listed, + lockArgv, + lockDocument, + waitForCalls, + wire, +} from "./test-physical.js"; const listFixture = readFileSync( new URL("./fixtures/devicectl-list-devices.json", import.meta.url), "utf8", ); -const lockFixture = JSON.parse( - readFileSync(new URL("./fixtures/devicectl-lock-state.json", import.meta.url), "utf8"), -) as Fields; -const appsFixture = JSON.parse( - readFileSync(new URL("./fixtures/devicectl-info-apps.json", import.meta.url), "utf8"), -) as { readonly result: { readonly apps: readonly Fields[] } }; -const simctlListFixture = readFileSync( - new URL("./fixtures/simctl-list.json", import.meta.url), - "utf8", -); -const SIMLOCK_HOME = "/home/.simlock"; -const JSON_DIR = `${SIMLOCK_HOME}/devicectl`; -const DEVICE_ROOT = "/Devices"; -const READ_LIMIT_MS = 30_000; const UNINSTALL_LIMIT_MS = 60_000; -const PHONE_UDID = "00008110-001A2B3C4D5E"; -const PHONE_IDENTIFIER = "C9F5A3E2-6D7B-4A8C-9E4F-3B2A0D9C8E73"; -const PHONE_SERIAL = "F4GXYZ1234"; -const PHONE_ECID = 6368729128779814; const IPAD_WIFI_UDID = "00008112-000B3344556677AB"; -const IPAD_WIRED_UDID = "00008027-00112233445566AA"; -const PHONE: Fields = { - deviceType: "iPhone", - developerModeStatus: "enabled", - ecid: PHONE_ECID, - identifier: PHONE_IDENTIFIER, - isMobileDeviceOnly: false, - localHostnames: ["Szymons-iPhone.coredevice.local"], - marketingName: "iPhone 15 Pro", - name: "Szymon's iPhone", - osVersionNumber: "18.4", - pairingState: "paired", - platform: "iOS", - productType: "iPhone16,1", - reality: "physical", - serialNumber: PHONE_SERIAL, - transportType: "wired", - tunnelState: "disconnected", - udid: PHONE_UDID, -}; +const IPAD_WIRED_UDID = "00008027-00112233445566AA"; const enrolledWithGone = (): PhysicalDriverDevice => ({ ...PHYSICAL_DEVICE, @@ -84,98 +54,6 @@ const PHYSICAL_DEVICE: PhysicalDriverDevice = { physical: true, }; -const DEPRECATED_GROUP: Readonly> = { - deviceType: "hardwareProperties", - developerModeStatus: "deviceProperties", - ecid: "hardwareProperties", - isMobileDeviceOnly: "connectionProperties", - localHostnames: "connectionProperties", - marketingName: "hardwareProperties", - name: "deviceProperties", - osVersionNumber: "deviceProperties", - pairingState: "connectionProperties", - platform: "hardwareProperties", - productType: "hardwareProperties", - reality: "hardwareProperties", - serialNumber: "hardwareProperties", - transportType: "connectionProperties", - tunnelState: "connectionProperties", - udid: "hardwareProperties", -}; - -const PROPERTIES_GROUP: Readonly> = { - deviceType: "hardware", - developerModeStatus: "state", - ecid: "hardware", - marketingName: "hardware", - osVersionNumber: "software", - pairingState: "connection", - platform: "hardware", - productType: "hardware", - reality: "hardware", - serialNumber: "hardware", - transportType: "connection", - udid: "hardware", -}; - -function wire(fields: Fields, onlyInProperties: readonly string[] = []): Fields { - const deprecated: Record = { - connectionProperties: {}, - deviceProperties: {}, - hardwareProperties: {}, - }; - const properties: Record = { - connection: {}, - hardware: {}, - software: {}, - state: {}, - }; - for (const [field, value] of Object.entries(fields)) { - if (value === undefined || field === "identifier") continue; - const group = DEPRECATED_GROUP[field]; - const fallback = PROPERTIES_GROUP[field]; - const deprecatedTarget = group === undefined ? undefined : deprecated[group]; - const propertiesTarget = fallback === undefined ? undefined : properties[fallback]; - if (deprecatedTarget !== undefined && !onlyInProperties.includes(field)) { - deprecatedTarget[field] = value; - } - if (propertiesTarget !== undefined) propertiesTarget[field] = value; - } - return { - ...deprecated, - identifier: fields["identifier"], - properties, - visibilityClass: "default", - }; -} - -function listDocument(devices: readonly Fields[]): string { - // why: JSON.parse would round an ECID above 2^53, so a test writes those digits as raw text. - return JSON.stringify({ - info: { commandType: "devicectl.list.devices", jsonVersion: 2, outcome: "success" }, - result: { devices }, - }).replace(/"@ecid:(\d+)"/g, "$1"); -} - -function lockDocument(result: Fields): string { - return JSON.stringify({ ...lockFixture, result }); -} - -function appsDocument(bundles: readonly string[]): string { - const template = appsFixture.result.apps[0] ?? {}; - return JSON.stringify({ - ...appsFixture, - result: { - ...appsFixture.result, - appCount: bundles.length, - apps: bundles.map((bundle) => ({ ...template, bundleIdentifier: bundle, name: bundle })), - }, - }); -} - -const OPEN_LOCK = lockDocument({ passcodeRequired: false, unlockedSinceBoot: true }); -const LOCKED = lockDocument({ passcodeRequired: true, unlockedSinceBoot: true }); - function fixtureDevices(): Fields[] { return (JSON.parse(listFixture) as { result: { devices: Fields[] } }).result.devices; } @@ -198,22 +76,6 @@ function withOsVersion(device: Fields, osVersionNumber: string): Fields { return copy; } -const LIST_ARGV = ["xcrun", "devicectl", "list", "devices", "--json-output", ""]; - -function lockArgv(udid: string): string[] { - return [ - "xcrun", - "devicectl", - "device", - "info", - "lockState", - "--device", - udid, - "--json-output", - "", - ]; -} - function appsArgv(udid: string): string[] { return [ "xcrun", @@ -232,207 +94,6 @@ function uninstallArgv(udid: string, bundle: string): string[] { return ["xcrun", "devicectl", "device", "uninstall", "app", "--device", udid, bundle]; } -class UmaskFilesystem extends MemoryFilesystem { - override async mkdirp(path: string): Promise { - const existed = await this.exists(path); - await super.mkdirp(path); - // why: a real mkdirp leaves 0755 under a 022 umask, so only an explicit 0700 passes. - if (!existed && path.endsWith("/devicectl")) await this.chmod(path, 0o755); - } -} - -interface ReadScript { - readonly file?: string; - readonly exit?: number; - readonly hangs?: boolean; -} - -class Lab { - readonly filesystem: MemoryFilesystem = new UmaskFilesystem(); - readonly clock = new FakeClock(); - readonly written: string[] = []; - readonly #expectations: ScriptedProcessExpectation[] = []; - #runner: ScriptedProcessRunner | undefined; - - get runner(): ScriptedProcessRunner { - this.#runner ??= new ScriptedProcessRunner(this.#expectations); - return this.#runner; - } - - add(expectation: ScriptedProcessExpectation): this { - this.#expectations.push(expectation); - return this; - } - - list(devices: readonly Fields[] | string, script?: ReadScript): this { - const file = typeof devices === "string" ? devices : listDocument(devices); - return this.#read(["list", "devices"], script ?? { file }); - } - - lock(udid: string, script: ReadScript = { file: OPEN_LOCK }): this { - return this.#read(["device", "info", "lockState", "--device", udid], script); - } - - apps(udid: string, bundles: readonly string[] | ReadScript): this { - const script = Array.isArray(bundles) - ? { file: appsDocument(bundles as readonly string[]) } - : (bundles as ReadScript); - return this.#read(["device", "info", "apps", "--device", udid], script); - } - - uninstall(udid: string, bundle: string, script: ReadScript = {}): this { - return this.add({ - match: { - args: ["devicectl", "device", "uninstall", "app", "--device", udid, bundle], - command: "xcrun", - }, - result: { code: script.exit ?? 0, stderr: "", stdout: "" }, - ...(script.hangs === true ? { hangs: true } : {}), - }); - } - - async driver( - runner: ProcessRunner = this.runner, - screenCapture?: NonNullable[0]["screenCapture"]>, - ): Promise { - await this.filesystem.mkdirp(SIMLOCK_HOME); - return IosSimctlDriver.create({ - database: await memorySourceDatabase(), - clock: this.clock, - driverConfig: { deviceRoot: DEVICE_ROOT }, - filesystem: this.filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId: "instance-1", - processRunner: runner, - ...(screenCapture === undefined ? {} : { screenCapture }), - simlockHome: SIMLOCK_HOME, - }); - } - - calls(): string[][] { - return this.runner.calls.map((call) => { - const argv = [call.command, ...call.args]; - const index = argv.indexOf("--json-output"); - if (index === -1) return argv; - return argv.map((argument, position) => (position === index + 1 ? "" : argument)); - }); - } - - timeouts(): (number | undefined)[] { - return this.runner.calls.map((call) => call.options.timeoutMs); - } - - #read(argv: readonly string[], script: ReadScript): this { - return this.add({ - match: { - args: ["devicectl", ...argv, "--json-output", /^\/home\/\.simlock\/devicectl\/[^/]+$/], - command: "xcrun", - }, - onSpawn: async (args) => { - const index = args.indexOf("--json-output"); - const path = args[index + 1]; - if (index === -1 || path === undefined) return; - this.written.push(path); - if (script.file === undefined) return; - try { - await this.filesystem.writeFileAtomic(path, script.file); - } catch { - // why: a driver that never created the directory gets no file, and its read fails on its own. - } - }, - result: { - code: script.exit ?? 0, - stderr: script.exit === undefined ? "" : "devicectl: failed", - stdout: "", - }, - ...(script.hangs === true ? { hangs: true } : {}), - }); - } -} - -class LockReadsTakeTime implements ProcessRunner { - constructor( - private readonly inner: ScriptedProcessRunner, - private readonly clock: FakeClock, - private readonly takesMs: number, - ) {} - - run( - command: string, - args: readonly string[], - options?: ProcessRunOptions, - ): Promise { - return this.spawn(command, args, options).wait(); - } - - spawn(command: string, args: readonly string[], options?: ProcessRunOptions): ProcessHandle { - const handle = this.inner.spawn(command, args, options); - if (!args.includes("lockState")) return handle; - return { - kill: (signal) => { - handle.kill(signal); - }, - pid: handle.pid, - stderr: handle.stderr, - stdout: handle.stdout, - unref: () => { - handle.unref(); - }, - wait: () => - new Promise((resolve, reject) => { - this.clock.setTimer(this.takesMs, () => { - handle.wait().then(resolve, reject); - }); - }), - }; - } - - spawnStreaming( - command: string, - args: readonly string[], - options: ProcessStreamOptions, - ): StreamingProcessHandle { - return this.inner.spawnStreaming(command, args, options); - } -} - -async function turns(count: number): Promise { - for (let turn = 0; turn < count; turn += 1) { - await new Promise((resolve) => { - setImmediate(resolve); - }); - } -} - -async function waitForCalls(runner: ScriptedProcessRunner, count: number): Promise { - for (let turn = 0; turn < 50 && runner.calls.length < count; turn += 1) await turns(1); - expect(runner.calls.length, "devicectl calls the driver made").toBeGreaterThanOrEqual(count); -} - -async function expireAfter( - lab: Lab, - pending: Promise, - calls: number, - ms: number, -): Promise { - void pending.catch(() => undefined); - await waitForCalls(lab.runner, calls); - lab.clock.advance(ms); - return pending; -} - -function enrollableLab( - devices: readonly Fields[] | string = [wire(PHONE)], - udid = PHONE_UDID, - bundles: readonly string[] = ["com.example.MyApp"], -): Lab { - return new Lab().list(devices).lock(udid).apps(udid, bundles); -} - -async function inspect(lab: Lab, id: string): Promise { - return (await lab.driver()).inspectPhysical(id); -} - interface Scenario { readonly change?: Fields; readonly id?: string; @@ -453,42 +114,6 @@ async function refusedAfterAccepting( return { calls: lab.calls(), inspection }; } -const FAILURES = [ - { - expires: false, - script: (file: string): ReadScript => ({ exit: 1, file }), - title: "exits non-zero", - }, - { expires: false, script: (): ReadScript => ({}), title: "leaves no file" }, - { - expires: false, - script: (): ReadScript => ({ file: "devicectl: not json {" }), - title: "leaves a file that is not JSON", - }, - { - expires: true, - script: (file: string): ReadScript => ({ file, hangs: true }), - title: "passes 30 s", - }, -] as const; - -const PHONE_LOCK_READ_FAILURES = FAILURES.filter((failure) => !failure.expires); - -function listed(udid: string, extra: Fields = {}): Fields { - return wire({ - ...PHONE, - identifier: `ID-${udid}`, - serialNumber: `SN-${udid}`, - udid, - ...extra, - }); -} - -const UDID_A = "00008110-00000000000A"; -const UDID_B = "00008110-00000000000B"; -const UDID_C = "00008110-00000000000C"; -const UDID_D = "00008110-00000000000D"; - describe("IosSimctlDriver physical handler: enrollment", () => { it.each([ { id: PHONE_IDENTIFIER, label: "CoreDevice ID" }, @@ -1091,240 +716,6 @@ describe("IosSimctlDriver physical handler: stream iOS version", () => { }); }); -describe("IosSimctlDriver physical handler: listAttachedPhysical", () => { - it("lists the wired phones in any pairing state, once each, none on the network, on this machine or simulated, and throws when the list read times out, exits non-zero or has no result.devices", async () => { - const lab = new Lab().list([ - listed(UDID_B, { pairingState: "unpaired" }), - listed(UDID_A), - listed(UDID_A, { transportType: "localNetwork" }), - listed(UDID_C, { transportType: "localNetwork" }), - listed(UDID_D, { transportType: "sameMachine" }), - listed("00008110-00000000000E", { reality: "simulated" }), - ]); - - const attached = await (await lab.driver()).listAttachedPhysical(); - - expect(attached).toEqual([UDID_A, UDID_B]); - expect(lab.calls()).toEqual([LIST_ARGV]); - - const document = listDocument([listed(UDID_A)]); - const reads = [ - { - lab: new Lab().list([], { file: document, hangs: true }), - reason: /timed out after 30000ms/, - }, - { lab: new Lab().list([], { exit: 1, file: document }), reason: /exited 1/ }, - { lab: new Lab().list('{"result":{}}'), reason: /no result\.devices list/ }, - ]; - for (const { lab, reason } of reads) { - const pending = (await lab.driver()).listAttachedPhysical(); - - const settled = reason.test("timed out after 30000ms") - ? expireAfter(lab, pending, 1, READ_LIMIT_MS) - : pending; - - await expect(settled).rejects.toThrow(reason); - } - }); -}); - -describe("IosSimctlDriver physical handler: presence", () => { - it("listPresentPhysical answers a paired, wired, unlocked given iPhone present, with its OS version", async () => { - const lab = new Lab().list([listed(UDID_A)]).lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A]); - - expect(result).toEqual([{ driverDeviceId: UDID_A, osVersion: "18.4", present: true }]); - }); - - it("listPresentPhysical answers a paired, wired, unlocked given iPhone with no reality present", async () => { - const lab = new Lab().list([listed(UDID_A, { reality: undefined })]).lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A]); - - expect(result).toEqual([{ driverDeviceId: UDID_A, osVersion: "18.4", present: true }]); - }); - - it("listPresentPhysical answers a wired, paired, simulated entry not present not-present, with no lock read", async () => { - const lab = new Lab().list([listed(UDID_A, { reality: "simulated" })]); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A]); - - expect(result).toEqual([{ driverDeviceId: UDID_A, present: false, reason: "not-present" }]); - expect(lab.calls()).toEqual([LIST_ARGV]); - }); - - it("listPresentPhysical answers a locked iPhone not present locked, an unpaired one untrusted, and a Wi-Fi-only one and one not listed not-present", async () => { - const lab = new Lab() - .list([ - listed(UDID_A), - listed(UDID_B, { pairingState: "unpaired" }), - listed(UDID_C, { transportType: "localNetwork" }), - ]) - .lock(UDID_A, { file: LOCKED }); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B, UDID_C, UDID_D]); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, present: false, reason: "locked" }, - { driverDeviceId: UDID_B, present: false, reason: "untrusted" }, - { driverDeviceId: UDID_C, present: false, reason: "not-present" }, - { driverDeviceId: UDID_D, present: false, reason: "not-present" }, - ]); - expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); - }); - - it("listPresentPhysical gives no entry for, and never runs a lock read for, a physical device not among the UDIDs it was given", async () => { - const lab = new Lab().list([listed(UDID_A), listed(UDID_B)]).lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A]); - - expect(result).toMatchObject([{ driverDeviceId: UDID_A, present: true }]); - expect(result).toHaveLength(1); - expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); - }); - - it("listPresentPhysical ignores a list entry with no udid: it runs no lock read for it, and every given UDID reads as it would without that entry", async () => { - const lab = new Lab() - .list([ - listed(UDID_A, { udid: undefined }), - listed(UDID_A), - listed(UDID_B, { udid: undefined }), - ]) - .lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, present: true }, - { driverDeviceId: UDID_B, present: false, reason: "not-present" }, - ]); - expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); - }); - - it("listPresentPhysical of a given UDID listed once over Wi-Fi and once wired reads the wired entry; one listed wired twice reads not-present with no lock read", async () => { - const lab = new Lab() - .list([ - listed(UDID_A, { osVersionNumber: "17.0", transportType: "localNetwork" }), - listed(UDID_A), - listed(UDID_B), - listed(UDID_B), - ]) - .lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, osVersion: "18.4", present: true }, - { driverDeviceId: UDID_B, present: false, reason: "not-present" }, - ]); - expect(lab.calls()).toEqual([LIST_ARGV, lockArgv(UDID_A)]); - }); - - it.each(FAILURES)( - "listPresentPhysical whose list read $title answers every given UDID not present not-present, and does not reject", - async (failure) => { - const lab = new Lab().list([listed(UDID_A), listed(UDID_B)], { - ...failure.script(listDocument([listed(UDID_A), listed(UDID_B)])), - }); - const driver = await lab.driver(); - const pending = driver.listPresentPhysical([UDID_A, UDID_B]); - - const result = await expireAfter( - lab, - pending, - failure.expires ? 1 : 0, - failure.expires ? READ_LIMIT_MS : 0, - ); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, present: false, reason: "not-present" }, - { driverDeviceId: UDID_B, present: false, reason: "not-present" }, - ]); - expect(lab.timeouts()).toEqual([READ_LIMIT_MS]); - }, - ); - - it("one lock read past 30 s answers that device not present not-present and the others as read", async () => { - const lab = new Lab() - .list([listed(UDID_A), listed(UDID_B)]) - .lock(UDID_A, { file: OPEN_LOCK, hangs: true }) - .lock(UDID_B); - const driver = await lab.driver(); - - const result = await expireAfter( - lab, - driver.listPresentPhysical([UDID_A, UDID_B]), - 3, - READ_LIMIT_MS, - ); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, present: false, reason: "not-present" }, - { driverDeviceId: UDID_B, osVersion: "18.4", present: true }, - ]); - expect(lab.timeouts()).toEqual([READ_LIMIT_MS, READ_LIMIT_MS, READ_LIMIT_MS]); - }); - - it.each([ - ...PHONE_LOCK_READ_FAILURES.map((failure) => ({ - script: failure.script(OPEN_LOCK), - title: failure.title, - })), - { - script: { file: lockDocument({ passcodeRequired: "false" }) }, - title: "lacks a boolean passcodeRequired", - }, - ])("a lock read that $title answers that device not present not-present", async ({ script }) => { - const lab = new Lab() - .list([listed(UDID_A), listed(UDID_B)]) - .lock(UDID_A, script) - .lock(UDID_B); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_A, present: false, reason: "not-present" }, - { driverDeviceId: UDID_B, present: true }, - ]); - }); - - it("two lock reads that each take 25 s (fake clock) both finish, and listPresentPhysical resolves before 30 s have passed after the list read: they run in parallel", async () => { - const lab = new Lab() - .list([listed(UDID_A), listed(UDID_B)]) - .lock(UDID_A) - .lock(UDID_B); - const driver = await lab.driver(new LockReadsTakeTime(lab.runner, lab.clock, 25_000)); - const pending = driver.listPresentPhysical([UDID_A, UDID_B]); - const state = promiseState(pending); - void pending.catch(() => undefined); - - await waitForCalls(lab.runner, 3); - lab.clock.advance(25_000); - await turns(5); - - expect(state.state).toBe("fulfilled"); - expect(await pending).toMatchObject([ - { driverDeviceId: UDID_A, present: true }, - { driverDeviceId: UDID_B, present: true }, - ]); - }); - - it("a list entry for a given UDID with no osVersionNumber answers that UDID not present not-present", async () => { - const lab = new Lab() - .list([listed(UDID_B), listed(UDID_A, { osVersionNumber: undefined })]) - .lock(UDID_B) - .lock(UDID_A); - - const result = await (await lab.driver()).listPresentPhysical([UDID_B, UDID_A]); - - expect(result).toMatchObject([ - { driverDeviceId: UDID_B, osVersion: "18.4", present: true }, - { driverDeviceId: UDID_A, present: false, reason: "not-present" }, - ]); - }); -}); - describe("IosSimctlDriver physical handler: reclaim", () => { it("reclaim of a physical device uninstalls each listed bundle ID missing from enrolledApps, with --device , and returns strategy uninstall", async () => { const lab = new Lab() @@ -1426,114 +817,3 @@ describe("IosSimctlDriver physical handler: reclaim", () => { expect(driver.estimate({ clean: "full", operation: "reclaim" }, virtual)).toBe(34_000); }); }); - -describe("IosSimctlDriver physical handler: files and limits", () => { - it("every devicectl read passes --json-output naming a new file in /devicectl/ (the driver creates it, mode 0700), and the driver reads that file and then deletes it", async () => { - const lab = enrollableLab([wire(PHONE)], PHONE_UDID, ["com.example.MyApp"]); - - const result = await inspect(lab, PHONE_UDID); - - expect(result).toMatchObject({ apps: ["com.example.MyApp"], enrollable: true }); - expect(lab.written).toHaveLength(3); - expect(new Set(lab.written).size).toBe(3); - for (const path of lab.written) { - expect(path).toMatch(/^\/home\/\.simlock\/devicectl\/[^/]+$/); - expect(await lab.filesystem.exists(path)).toBe(false); - } - const directory = await lab.filesystem.lstat(JSON_DIR); - expect(directory.kind).toBe("directory"); - expect(directory.mode & 0o777).toBe(0o700); - }); - - it.each([ - { expires: false, script: { exit: 1 } as ReadScript, title: "exits non-zero" }, - { expires: true, script: { hangs: true } as ReadScript, title: "passes its limit" }, - ])("the file is deleted also after a read that $title", async ({ expires, script }) => { - const lab = new Lab().list(listDocument([wire(PHONE)]), { - file: listDocument([wire(PHONE)]), - ...script, - }); - const driver = await lab.driver(); - - await expireAfter( - lab, - driver.inspectPhysical(PHONE_UDID), - expires ? 1 : 0, - expires ? READ_LIMIT_MS : 0, - ); - - expect(lab.written).toHaveLength(1); - expect(await lab.filesystem.exists(lab.written[0] ?? "")).toBe(false); - }); - - it("two lock reads in parallel name two different files", async () => { - const lab = new Lab() - .list([listed(UDID_A), listed(UDID_B)]) - .lock(UDID_A) - .lock(UDID_B); - - const result = await (await lab.driver()).listPresentPhysical([UDID_A, UDID_B]); - - expect(result).toMatchObject([{ present: true }, { present: true }]); - expect(lab.written).toHaveLength(3); - expect(new Set(lab.written).size).toBe(3); - }); - - it.each([ - { - lab: () => - new Lab() - .list([wire(PHONE)]) - .lock(PHONE_UDID, { file: lockDocument({ passcodeRequired: "false" }) }) - .apps(PHONE_UDID, ["com.example.MyApp"]), - name: "lockState", - title: "passcodeRequired as a string", - }, - { - lab: () => - new Lab() - .list([wire({ ...PHONE, udid: 8110 })]) - .lock(PHONE_UDID) - .apps(PHONE_UDID, ["com.example.MyApp"]), - name: "list devices", - title: "udid as a number", - }, - ])( - "a read whose file holds a field of the wrong JSON type ($title) is a failed read", - async ({ lab, name }) => { - const accepted = await inspect(enrollableLab(), PHONE_IDENTIFIER); - expect(accepted).toMatchObject({ enrollable: true }); - - const result = await inspect(lab(), PHONE_IDENTIFIER); - - expect(result).toMatchObject({ - enrollable: false, - message: expect.stringContaining(name), - reason: "unreadable", - }); - }, - ); - - it("a devicectl read that hangs does not delay a simctl call made at the same time", async () => { - const lab = new Lab().list([wire(PHONE)], { hangs: true }).add({ - match: { - args: ["simctl", "--set", DEVICE_ROOT, "list", "-j", "devicetypes", "runtimes"], - command: "xcrun", - }, - result: { code: 0, stderr: "", stdout: simctlListFixture }, - }); - const driver = await lab.driver(); - const read = driver.inspectPhysical(PHONE_UDID); - void read.catch(() => undefined); - await waitForCalls(lab.runner, 1); - const readState = promiseState(read); - - const resolving = driver.resolveSpec({ model: "iPhone 16", platform: "ios" }); - const resolvingState = promiseState(resolving); - await turns(5); - - expect(readState.state).toBe("pending"); - expect(resolvingState.state).toBe("fulfilled"); - expect(await resolving).toMatchObject({ model: "iPhone 16", physical: false }); - }); -}); diff --git a/src/drivers/ios/test-physical.ts b/src/drivers/ios/test-physical.ts new file mode 100644 index 000000000..9db57fa3a --- /dev/null +++ b/src/drivers/ios/test-physical.ts @@ -0,0 +1,355 @@ +import { readFileSync } from "node:fs"; +import { expect } from "vitest"; +import type { PhysicalInspection } from "../../core/index.js"; +import { + MemoryFilesystem, + FakeClock, + type ScriptedProcessExpectation, + ScriptedProcessRunner, + type ProcessRunner, +} from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase } from "./testing.js"; + +export type Json = string | number | boolean | null | undefined | Json[] | { [key: string]: Json }; + +export type Fields = Record; + +export const lockFixture = JSON.parse( + readFileSync(new URL("./fixtures/devicectl-lock-state.json", import.meta.url), "utf8"), +) as Fields; + +export const appsFixture = JSON.parse( + readFileSync(new URL("./fixtures/devicectl-info-apps.json", import.meta.url), "utf8"), +) as { readonly result: { readonly apps: readonly Fields[] } }; + +export const SIMLOCK_HOME = "/home/.simlock"; + +export const DEVICE_ROOT = "/Devices"; + +export const READ_LIMIT_MS = 30_000; + +export const PHONE_UDID = "00008110-001A2B3C4D5E"; + +export const PHONE_IDENTIFIER = "C9F5A3E2-6D7B-4A8C-9E4F-3B2A0D9C8E73"; + +export const PHONE_SERIAL = "F4GXYZ1234"; + +export const PHONE_ECID = 6368729128779814; + +export const PHONE: Fields = { + deviceType: "iPhone", + developerModeStatus: "enabled", + ecid: PHONE_ECID, + identifier: PHONE_IDENTIFIER, + isMobileDeviceOnly: false, + localHostnames: ["Szymons-iPhone.coredevice.local"], + marketingName: "iPhone 15 Pro", + name: "Szymon's iPhone", + osVersionNumber: "18.4", + pairingState: "paired", + platform: "iOS", + productType: "iPhone16,1", + reality: "physical", + serialNumber: PHONE_SERIAL, + transportType: "wired", + tunnelState: "disconnected", + udid: PHONE_UDID, +}; + +export const DEPRECATED_GROUP: Readonly> = { + deviceType: "hardwareProperties", + developerModeStatus: "deviceProperties", + ecid: "hardwareProperties", + isMobileDeviceOnly: "connectionProperties", + localHostnames: "connectionProperties", + marketingName: "hardwareProperties", + name: "deviceProperties", + osVersionNumber: "deviceProperties", + pairingState: "connectionProperties", + platform: "hardwareProperties", + productType: "hardwareProperties", + reality: "hardwareProperties", + serialNumber: "hardwareProperties", + transportType: "connectionProperties", + tunnelState: "connectionProperties", + udid: "hardwareProperties", +}; + +export const PROPERTIES_GROUP: Readonly> = { + deviceType: "hardware", + developerModeStatus: "state", + ecid: "hardware", + marketingName: "hardware", + osVersionNumber: "software", + pairingState: "connection", + platform: "hardware", + productType: "hardware", + reality: "hardware", + serialNumber: "hardware", + transportType: "connection", + udid: "hardware", +}; + +export function wire(fields: Fields, onlyInProperties: readonly string[] = []): Fields { + const deprecated: Record = { + connectionProperties: {}, + deviceProperties: {}, + hardwareProperties: {}, + }; + const properties: Record = { + connection: {}, + hardware: {}, + software: {}, + state: {}, + }; + for (const [field, value] of Object.entries(fields)) { + if (value === undefined || field === "identifier") continue; + const group = DEPRECATED_GROUP[field]; + const fallback = PROPERTIES_GROUP[field]; + const deprecatedTarget = group === undefined ? undefined : deprecated[group]; + const propertiesTarget = fallback === undefined ? undefined : properties[fallback]; + if (deprecatedTarget !== undefined && !onlyInProperties.includes(field)) { + deprecatedTarget[field] = value; + } + if (propertiesTarget !== undefined) propertiesTarget[field] = value; + } + return { + ...deprecated, + identifier: fields["identifier"], + properties, + visibilityClass: "default", + }; +} + +export function listDocument(devices: readonly Fields[]): string { + // why: JSON.parse would round an ECID above 2^53, so a test writes those digits as raw text. + return JSON.stringify({ + info: { commandType: "devicectl.list.devices", jsonVersion: 2, outcome: "success" }, + result: { devices }, + }).replace(/"@ecid:(\d+)"/g, "$1"); +} + +export function lockDocument(result: Fields): string { + return JSON.stringify({ ...lockFixture, result }); +} + +export function appsDocument(bundles: readonly string[]): string { + const template = appsFixture.result.apps[0] ?? {}; + return JSON.stringify({ + ...appsFixture, + result: { + ...appsFixture.result, + appCount: bundles.length, + apps: bundles.map((bundle) => ({ ...template, bundleIdentifier: bundle, name: bundle })), + }, + }); +} + +export const OPEN_LOCK = lockDocument({ passcodeRequired: false, unlockedSinceBoot: true }); + +export const LOCKED = lockDocument({ passcodeRequired: true, unlockedSinceBoot: true }); + +export const LIST_ARGV = ["xcrun", "devicectl", "list", "devices", "--json-output", ""]; + +export function lockArgv(udid: string): string[] { + return [ + "xcrun", + "devicectl", + "device", + "info", + "lockState", + "--device", + udid, + "--json-output", + "", + ]; +} + +export class UmaskFilesystem extends MemoryFilesystem { + override async mkdirp(path: string): Promise { + const existed = await this.exists(path); + await super.mkdirp(path); + // why: a real mkdirp leaves 0755 under a 022 umask, so only an explicit 0700 passes. + if (!existed && path.endsWith("/devicectl")) await this.chmod(path, 0o755); + } +} + +export interface ReadScript { + readonly file?: string; + readonly exit?: number; + readonly hangs?: boolean; +} + +export class Lab { + readonly filesystem: MemoryFilesystem = new UmaskFilesystem(); + readonly clock = new FakeClock(); + readonly written: string[] = []; + readonly #expectations: ScriptedProcessExpectation[] = []; + #runner: ScriptedProcessRunner | undefined; + + get runner(): ScriptedProcessRunner { + this.#runner ??= new ScriptedProcessRunner(this.#expectations); + return this.#runner; + } + + add(expectation: ScriptedProcessExpectation): this { + this.#expectations.push(expectation); + return this; + } + + list(devices: readonly Fields[] | string, script?: ReadScript): this { + const file = typeof devices === "string" ? devices : listDocument(devices); + return this.#read(["list", "devices"], script ?? { file }); + } + + lock(udid: string, script: ReadScript = { file: OPEN_LOCK }): this { + return this.#read(["device", "info", "lockState", "--device", udid], script); + } + + apps(udid: string, bundles: readonly string[] | ReadScript): this { + const script = Array.isArray(bundles) + ? { file: appsDocument(bundles as readonly string[]) } + : (bundles as ReadScript); + return this.#read(["device", "info", "apps", "--device", udid], script); + } + + uninstall(udid: string, bundle: string, script: ReadScript = {}): this { + return this.add({ + match: { + args: ["devicectl", "device", "uninstall", "app", "--device", udid, bundle], + command: "xcrun", + }, + result: { code: script.exit ?? 0, stderr: "", stdout: "" }, + ...(script.hangs === true ? { hangs: true } : {}), + }); + } + + async driver( + runner: ProcessRunner = this.runner, + screenCapture?: NonNullable[0]["screenCapture"]>, + ): Promise { + await this.filesystem.mkdirp(SIMLOCK_HOME); + return IosSimctlDriver.create({ + database: await memorySourceDatabase(), + clock: this.clock, + driverConfig: { deviceRoot: DEVICE_ROOT }, + filesystem: this.filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId: "instance-1", + processRunner: runner, + ...(screenCapture === undefined ? {} : { screenCapture }), + simlockHome: SIMLOCK_HOME, + }); + } + + calls(): string[][] { + return this.runner.calls.map((call) => { + const argv = [call.command, ...call.args]; + const index = argv.indexOf("--json-output"); + if (index === -1) return argv; + return argv.map((argument, position) => (position === index + 1 ? "" : argument)); + }); + } + + timeouts(): (number | undefined)[] { + return this.runner.calls.map((call) => call.options.timeoutMs); + } + + #read(argv: readonly string[], script: ReadScript): this { + return this.add({ + match: { + args: ["devicectl", ...argv, "--json-output", /^\/home\/\.simlock\/devicectl\/[^/]+$/], + command: "xcrun", + }, + onSpawn: async (args) => { + const index = args.indexOf("--json-output"); + const path = args[index + 1]; + if (index === -1 || path === undefined) return; + this.written.push(path); + if (script.file === undefined) return; + try { + await this.filesystem.writeFileAtomic(path, script.file); + } catch { + // why: a driver that never created the directory gets no file, and its read fails on its own. + } + }, + result: { + code: script.exit ?? 0, + stderr: script.exit === undefined ? "" : "devicectl: failed", + stdout: "", + }, + ...(script.hangs === true ? { hangs: true } : {}), + }); + } +} + +export async function turns(count: number): Promise { + for (let turn = 0; turn < count; turn += 1) { + await new Promise((resolve) => { + setImmediate(resolve); + }); + } +} + +export async function waitForCalls(runner: ScriptedProcessRunner, count: number): Promise { + for (let turn = 0; turn < 50 && runner.calls.length < count; turn += 1) await turns(1); + expect(runner.calls.length, "devicectl calls the driver made").toBeGreaterThanOrEqual(count); +} + +export async function expireAfter( + lab: Lab, + pending: Promise, + calls: number, + ms: number, +): Promise { + void pending.catch(() => undefined); + await waitForCalls(lab.runner, calls); + lab.clock.advance(ms); + return pending; +} + +export function enrollableLab( + devices: readonly Fields[] | string = [wire(PHONE)], + udid = PHONE_UDID, + bundles: readonly string[] = ["com.example.MyApp"], +): Lab { + return new Lab().list(devices).lock(udid).apps(udid, bundles); +} + +export async function inspect(lab: Lab, id: string): Promise { + return (await lab.driver()).inspectPhysical(id); +} + +export const FAILURES = [ + { + expires: false, + script: (file: string): ReadScript => ({ exit: 1, file }), + title: "exits non-zero", + }, + { expires: false, script: (): ReadScript => ({}), title: "leaves no file" }, + { + expires: false, + script: (): ReadScript => ({ file: "devicectl: not json {" }), + title: "leaves a file that is not JSON", + }, + { + expires: true, + script: (file: string): ReadScript => ({ file, hangs: true }), + title: "passes 30 s", + }, +] as const; + +export function listed(udid: string, extra: Fields = {}): Fields { + return wire({ + ...PHONE, + identifier: `ID-${udid}`, + serialNumber: `SN-${udid}`, + udid, + ...extra, + }); +} + +export const UDID_A = "00008110-00000000000A"; + +export const UDID_B = "00008110-00000000000B"; From f9511c7b47ae81d6c9c7ea0f3dc50e788c8545fb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:28:09 +0100 Subject: [PATCH 26/83] chore(agents): the test lint excuses a test line or internal import a split carries over unchanged --- .agents/scripts/test-lint.mjs | 70 +++++++++++++++++++++++++++++------ src/test-lint.test.ts | 33 +++++++++++++++++ 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/.agents/scripts/test-lint.mjs b/.agents/scripts/test-lint.mjs index 47de65120..a01b7628e 100644 --- a/.agents/scripts/test-lint.mjs +++ b/.agents/scripts/test-lint.mjs @@ -84,17 +84,18 @@ function sourceOf({ path: file, lines, added }) { return { file, lines, added, text, code, codeLines: code.split("\n"), starts: lineStarts(text) }; } -function testFindings(changed) { +function testFindings(changed, split) { const source = sourceOf(changed); const rules = LINE_RULES.filter((entry) => entry.applies(source.file)); const out = [...source.added] .sort((a, b) => a - b) + .filter((n) => !takeMovedLine(split.removed, source.lines[n - 1])) .flatMap((n) => rules .filter((entry) => entry.breaks(source, n)) .map((entry) => ({ path: source.file, line: n, rule: entry.rule })), ); - const internal = internalImportLine(source); + const internal = internalImportLine(source, split.imported); if (internal !== undefined) out.push({ path: source.file, line: internal, rule: "internal" }); return out; } @@ -143,20 +144,22 @@ function optionalCallInExpect(code, offset, line) { }); } -function internalImportLine(source) { - if (!exists(`${path.posix.dirname(source.file)}/index.ts`)) return undefined; - const line = firstAddedInternalImport(source); +function internalImportLine(source, imported) { + const dir = path.posix.dirname(source.file); + if (!exists(`${dir}/index.ts`)) return undefined; + const line = firstAddedInternalImport(source, imported.get(dir) ?? new Set()); if (line === undefined) return undefined; const first = firstTestBlock(source.codeLines); if (first === -1) return undefined; return hasWhyAbove(source.lines, source.codeLines, first) ? undefined : line; } -function firstAddedInternalImport({ text, code, starts, added }) { - const importRe = - /^[ \t]*import\s+(type\s+)?(?:[^;]*?)\bfrom\s*["']\.\/([^"'/]+?)(?:\.[jt]sx?)?["']/gm; - return [...text.matchAll(importRe)] - .filter((m) => isInternalValueImport(m, code)) +const IMPORT_RE = + /^[ \t]*import\s+(type\s+)?(?:[^;]*?)\bfrom\s*["']\.\/([^"'/]+?)(?:\.[jt]sx?)?["']/gm; + +function firstAddedInternalImport({ text, code, starts, added }, carried) { + return [...text.matchAll(IMPORT_RE)] + .filter((m) => isInternalValueImport(m, code) && !carried.has(m[2])) .map((m) => lineOf(starts, m.index + m[0].length - 1)) .find((line) => added.has(line)); } @@ -172,6 +175,50 @@ function firstTestBlock(codeLines) { return codeLines.findIndex((l) => /^\s*(?:it|test)\b[\w.]*\s*\(/.test(l)); } +function takeMovedLine(removed, line) { + const text = line.trim(); + const n = removed.get(text) ?? 0; + if (n === 0) return false; + removed.set(text, n - 1); + return true; +} + +// why: a split moves tests between files unchanged, so a line the diff adds to one test file while +// it removes the same line from another is not new (one removal excuses one addition), and a test +// file that imports an internal sibling a test file of its directory already imported before the +// split carries that file's reason with it rather than adding a new one. +function splitTests(base, head) { + const from = gitText(["merge-base", base, head ?? "HEAD"]).trim(); + const range = head === undefined ? [from] : [from, head]; + const removed = new Map(); + const diff = gitText([ + "diff", + "-U0", + "--no-color", + "--no-renames", + ...range, + "--", + "*.test.ts", + "*.test.tsx", + ]); + for (const line of diff.split("\n")) { + if (!line.startsWith("-") || line.startsWith("--- ")) continue; + const text = line.slice(1).trim(); + if (text !== "") removed.set(text, (removed.get(text) ?? 0) + 1); + } + const imported = new Map(); + const shrunk = gitText(["diff", "--name-only", "--no-renames", "--diff-filter=MD", ...range]); + for (const file of shrunk.split("\n").filter((f) => TEST_FILE.test(f))) { + const text = revisionText(from, file); + const code = mask(text); + const dir = path.posix.dirname(file); + const names = imported.get(dir) ?? new Set(); + for (const m of text.matchAll(IMPORT_RE)) if (isInternalValueImport(m, code)) names.add(m[2]); + imported.set(dir, names); + } + return { removed, imported }; +} + function contractFindings(changed, moved) { const source = sourceOf(changed); const excused = movedDeclarations(source, moved.get(source.file)); @@ -503,9 +550,10 @@ if (base === undefined || (args.includes("--head") && head === undefined)) usage const findings = []; const moved = movedContractDeclarations(base, head); +const split = splitTests(base, head); for (const file of changedFiles({ base, head, files: files.length > 0 ? files : null })) { if (PIN_FILE.test(file.path)) continue; - if (TEST_FILE.test(file.path)) findings.push(...testFindings(file)); + if (TEST_FILE.test(file.path)) findings.push(...testFindings(file, split)); else if (CONTRACT_FILE.test(file.path) && !HELPER_FILE.test(file.path)) { findings.push(...contractFindings(file, moved)); } diff --git a/src/test-lint.test.ts b/src/test-lint.test.ts index e0cc06b5d..da2866873 100644 --- a/src/test-lint.test.ts +++ b/src/test-lint.test.ts @@ -314,6 +314,39 @@ describe("test-lint.mjs", () => { expect(lint(repo(before, after))).toEqual({ status: 1, lines: [`${line}: ${BOUND}`] }); }); + it("testing 5, 6 and 10: passes a split that moves tests unchanged into a new file, and flags a line or an internal import the split adds", () => { + const dir = repo( + { + ...LEASING_INDEX, + "src/leasing/queue.test.ts": file( + 'import { admit } from "./queue.js";', + "it('admits', () => {});", + "it('rejects', () => { expect(admit()).rejects.toBeInstanceOf(Error); });", + ), + }, + { + "src/leasing/queue.test.ts": file( + 'import { admit } from "./queue.js";', + "it('admits', () => {});", + ), + "src/leasing/queue-rejects.test.ts": file( + 'import { admit } from "./queue.js";', + 'import { plan } from "./planner.js";', + "it('rejects', () => { expect(admit()).rejects.toBeInstanceOf(Error); });", + "it.skip('plans', () => {});", + ), + }, + ); + + const result = lint(dir); + + expect(result.status).toBe(1); + expect(result.lines).toEqual([ + `src/leasing/queue-rejects.test.ts:4: ${SKIP}`, + `src/leasing/queue-rejects.test.ts:2: ${INTERNAL}`, + ]); + }); + it("judges only the lines the diff adds in the files given, and never a pattern inside a string or comment", () => { const dir = repo( { "src/a/old.test.ts": file("it.skip('old', () => {});") }, From 3139b652c864cb7c01d13489650f3cf6e764b244 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:20:08 +0100 Subject: [PATCH 27/83] test(gateway): split dispatcher.test.ts by topic --- .../dispatcher-alerts-and-refusals.test.ts | 400 ++++ src/gateway/dispatcher-baselines.test.ts | 520 +++++ src/gateway/dispatcher-events-tokens.test.ts | 153 ++ .../dispatcher-lease-lifecycle.test.ts | 526 +++++ src/gateway/dispatcher-leases.test.ts | 419 ++++ src/gateway/dispatcher.test.ts | 1991 +---------------- 6 files changed, 2020 insertions(+), 1989 deletions(-) create mode 100644 src/gateway/dispatcher-alerts-and-refusals.test.ts create mode 100644 src/gateway/dispatcher-baselines.test.ts create mode 100644 src/gateway/dispatcher-events-tokens.test.ts create mode 100644 src/gateway/dispatcher-lease-lifecycle.test.ts create mode 100644 src/gateway/dispatcher-leases.test.ts diff --git a/src/gateway/dispatcher-alerts-and-refusals.test.ts b/src/gateway/dispatcher-alerts-and-refusals.test.ts new file mode 100644 index 000000000..d22877c40 --- /dev/null +++ b/src/gateway/dispatcher-alerts-and-refusals.test.ts @@ -0,0 +1,400 @@ +import { describe, expect, it, vi } from "vitest"; +import { createHmac } from "node:crypto"; +import { createServer } from "node:net"; +import { type AlertDestination } from "../core/index.js"; +import { FetchWebhookPoster, type WebhookPoster } from "../ports/index.js"; +import { ScriptedWorkerClient } from "./test-support.js"; +import { harness, session } from "./test-dispatcher.js"; + +type SentRequest = Parameters[0]; + +describe("GatewayDispatcher", () => { + describe("alerts.test", () => { + const generic: AlertDestination = { + name: "ops", + format: "generic", + url: "https://hooks.example.com/ops-token", + events: ["worker.connected"], + secret: "s3cr3t", + }; + const slack: AlertDestination = { + name: "chat", + format: "slack", + url: "https://hooks.slack.example.com/chat-token", + events: ["worker.connected"], + }; + + it("alerts.test posts one sample to the named destination and answers its status", async () => { + const { dispatcher, poster } = harness({ alertDestinations: [generic, slack] }); + + await expect( + dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), + ).resolves.toEqual({ destination: "ops", status: 200 }); + await expect( + dispatcher.dispatch("alerts.test", { destination: "chat" }, session()), + ).resolves.toEqual({ destination: "chat", status: 200 }); + + expect(poster.requests).toHaveLength(2); + const [first, second] = poster.requests as [SentRequest, SentRequest]; + expect(first.url).toBe("https://hooks.example.com/ops-token"); + expect(JSON.parse(first.body)).toEqual({ + id: "evt_sample-id", + seq: 0, + timestamp: 1_000, + event: "alerts.test-sent", + payload: { destination: "ops" }, + module: "alerts", + }); + expect(first.headers).toEqual({ + "Content-Type": "application/json", + "User-Agent": "simlock/9.9.9", + "X-Simlock-Event": "alerts.test-sent", + "X-Simlock-Delivery": "evt_sample-id", + "X-Simlock-Signature": `sha256=${createHmac("sha256", "s3cr3t") + .update(first.body) + .digest("hex")}`, + }); + expect(second.url).toBe("https://hooks.slack.example.com/chat-token"); + expect(JSON.parse(second.body)).toEqual({ + text: "🔔 Test message from the simlock gateway to destination chat.", + }); + expect(second.headers).toEqual({ + "Content-Type": "application/json", + "User-Agent": "simlock/9.9.9", + }); + + const other = harness({ alertDestinations: [generic] }); + other.poster.respond = () => Promise.resolve({ status: 204 }); + await expect( + other.dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), + ).resolves.toEqual({ destination: "ops", status: 204 }); + }); + + it("alerts.test fails with ALERT_SEND_FAILED on a non-2xx status, a rejection and no answer within 10 s, and tries once", async () => { + const refused = Object.assign(new TypeError("fetch failed"), { + cause: { code: "ECONNREFUSED" }, + }); + const cases = [ + [() => Promise.resolve({ status: 404 }), "ops answered HTTP 404", { status: 404 }], + [() => Promise.resolve({ status: 199 }), "ops answered HTTP 199", { status: 199 }], + [() => Promise.resolve({ status: 301 }), "ops answered HTTP 301", { status: 301 }], + [() => Promise.resolve({ status: 503 }), "ops answered HTTP 503", { status: 503 }], + [ + () => Promise.reject(refused), + "ops could not be reached: ECONNREFUSED", + { error: "ECONNREFUSED" }, + ], + [ + () => Promise.reject(new TypeError("fetch failed")), + "ops could not be reached: network-error", + { error: "network-error" }, + ], + ] as const; + for (const [respond, message, details] of cases) { + const { clock, dispatcher, poster } = harness({ alertDestinations: [generic] }); + poster.respond = respond; + + await expect( + dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), + ).rejects.toMatchObject({ + code: "ALERT_SEND_FAILED", + message, + details: { destination: "ops", ...details }, + }); + clock.advance(60_000); + expect(poster.requests).toHaveLength(1); + } + + const { clock, dispatcher, poster } = harness({ alertDestinations: [generic] }); + poster.respond = () => new Promise(() => {}); + let outcome: unknown = "pending"; + void dispatcher.dispatch("alerts.test", { destination: "ops" }, session()).then( + (value) => (outcome = value), + (error: unknown) => (outcome = error), + ); + await vi.waitFor(() => expect(poster.requests).toHaveLength(1)); + clock.advance(9_999); + await new Promise((resolve) => setImmediate(resolve)); + expect(outcome).toBe("pending"); + expect(poster.requests[0]?.signal.aborted).toBe(false); + + clock.advance(1); + await new Promise((resolve) => setImmediate(resolve)); + expect(outcome).toMatchObject({ + code: "ALERT_SEND_FAILED", + message: "ops did not answer within 10 s", + details: { destination: "ops", error: "timeout" }, + }); + expect(poster.requests[0]?.signal.aborted).toBe(true); + clock.advance(60_000); + expect(poster.requests).toHaveLength(1); + }); + + it("alerts.test answers UNKNOWN_ALERT_DESTINATION for a name not configured and on a gateway with no destinations", async () => { + const unknown = { + code: "UNKNOWN_ALERT_DESTINATION", + message: + 'No alert destination named "nope". Check gateway.alerts.destinations in the gateway\'s config.', + details: { destination: "nope" }, + }; + const configured = harness({ alertDestinations: [generic] }); + await expect( + configured.dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), + ).resolves.toEqual({ destination: "ops", status: 200 }); + + await expect( + configured.dispatcher.dispatch("alerts.test", { destination: "nope" }, session()), + ).rejects.toMatchObject(unknown); + await expect( + harness().dispatcher.dispatch("alerts.test", { destination: "nope" }, session()), + ).rejects.toMatchObject(unknown); + await expect( + configured.dispatcher.dispatch("alerts.test", { destination: "o" }, session()), + ).rejects.toMatchObject({ code: "UNKNOWN_ALERT_DESTINATION", details: { destination: "o" } }); + + expect(configured.poster.requests).toHaveLength(1); + }); + + it("a refused connection answers could not be reached with the network code, and neither the answer nor the failed call's operation log line holds the URL", async () => { + const closedPort = await new Promise((resolve) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as { port: number }; + server.close(() => resolve(port)); + }); + }); + const url = `http://127.0.0.1:${closedPort}/hook-token-xyz`; + const { dispatcher, logSink } = harness({ + alertDestinations: [{ ...generic, url }], + webhookPoster: new FetchWebhookPoster(), + }); + + const failure = await dispatcher + .dispatch("alerts.test", { destination: "ops" }, session()) + .then( + () => undefined, + (error: unknown) => error as { code: string; message: string; details: unknown }, + ); + + expect(failure).toMatchObject({ + code: "ALERT_SEND_FAILED", + message: "ops could not be reached: ECONNREFUSED", + details: { destination: "ops", error: "ECONNREFUSED" }, + }); + expect( + logSink.records + .filter((record) => record.message === "operation") + .map((record) => record.fields), + ).toMatchObject([ + { + operation: "alerts.test", + code: "ALERT_SEND_FAILED", + message: "ops could not be reached: ECONNREFUSED", + }, + ]); + for (const text of [JSON.stringify(failure), JSON.stringify(logSink.records)]) { + expect(text).not.toContain("hook-token-xyz"); + expect(text).not.toContain(String(closedPort)); + } + }); + + it("alerts.test answers BAD_REQUEST for a destination of no characters or more than 32, and looks up one of 32", async () => { + const { dispatcher, poster } = harness({ alertDestinations: [generic] }); + const longest = "n".repeat(32); + + for (const destination of ["", "n".repeat(33)]) { + await expect( + dispatcher.dispatch("alerts.test", { destination }, session()), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + } + await expect( + dispatcher.dispatch("alerts.test", { destination: longest }, session()), + ).rejects.toMatchObject({ code: "UNKNOWN_ALERT_DESTINATION" }); + expect(poster.requests).toEqual([]); + }); + + it("stop during a test answers DAEMON_STOPPING", async () => { + const { alerts, dispatcher, poster } = harness({ alertDestinations: [generic] }); + poster.respond = () => new Promise(() => {}); + let outcome: unknown = "pending"; + void dispatcher.dispatch("alerts.test", { destination: "ops" }, session()).then( + (value) => (outcome = value), + (error: unknown) => (outcome = error), + ); + await vi.waitFor(() => expect(poster.requests).toHaveLength(1)); + expect(outcome).toBe("pending"); + + alerts?.stop(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(outcome).toMatchObject({ code: "DAEMON_STOPPING" }); + }); + }); + + describe("worker.install-component (ADR 0010 §7)", () => { + function withWorker() { + const fixture = harness(); + const worker = new ScriptedWorkerClient(); + fixture.workers.connected("wrk_1", undefined, "0.3.0"); + fixture.workers.refresh("wrk_1", { + downloads: { policy: "on-request", timeoutMs: 60_000 }, + }); + fixture.directory.add("wrk_1", worker); + return { ...fixture, worker }; + } + + it("refuses an agent session with FORBIDDEN and asks no worker", async () => { + const { dispatcher, worker } = withWorker(); + + await expect( + dispatcher.dispatch( + "worker.install-component", + { platform: "android", version: "35", workers: "all" }, + session({ role: "agent" }), + ), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + expect(worker.calls.filter((call) => call.startsWith("component.install"))).toEqual([]); + }); + + it("opens the session's stream before asking a worker, and hands it each update with the worker's id", async () => { + const { dispatcher, worker } = withWorker(); + const seen: unknown[] = []; + worker.installComponentHandler = async (input, options) => { + seen.push("asked"); + options.onProgress?.({ stage: "waiting" }); + return { + component: input.version, + outcome: "installed", + platform: input.platform, + version: input.version, + }; + }; + + const result = await dispatcher.dispatch( + "worker.install-component", + { platform: "android", version: "35", workers: ["wrk_1"] }, + session({ + onComponentProgress: (progress, workerId) => seen.push({ progress, workerId }), + onStarted: () => seen.push("started"), + }), + ); + + expect(seen).toEqual([ + "started", + "asked", + { progress: { stage: "waiting" }, workerId: "wrk_1" }, + ]); + expect(result).toEqual({ + results: [{ outcome: "installed", version: "35", workerId: "wrk_1" }], + }); + }); + }); + + describe("UNSUPPORTED_IN_GATEWAY_MODE", () => { + it.each([ + "nuke.run", + "cleanup.run", + "doctor.run", + "driver.passthrough", + "warm.forget", + ] as const)("%s stays per-worker, permanently", async (operation) => { + const { dispatcher } = harness(); + const input = + operation === "driver.passthrough" + ? { args: ["devices"], tool: "adb" } + : operation === "warm.forget" + ? { model: "Pixel", platform: "android" } + : { fix: true }; + + await expect(dispatcher.dispatch(operation, input, session())).rejects.toMatchObject({ + code: "UNSUPPORTED_IN_GATEWAY_MODE", + details: { operation }, + }); + }); + + it.each([ + [ + "component.install", + { platform: "android", version: "35" }, + "admin", + "component.install installs on one machine; on a gateway, name the workers to install on " + + "(simlock component install --worker or --all-workers)", + ], + [ + "component.list", + {}, + "agent", + "component.list lists one machine's components; run it against a worker", + ], + [ + "component.remove", + { platform: "ios", version: "26.4" }, + "admin", + "component.remove removes a component from one machine; run it against a worker", + ], + [ + "device.add", + { id: "00008110-001A2B3C4D5E", platform: "ios" }, + "admin", + "device.add enrols a device plugged into one machine; run it on the worker the device is plugged into", + ], + [ + "device.remove", + { id: "00008110-001A2B3C4D5E" }, + "admin", + "device.remove unenrols a device plugged into one machine; run it on the worker the device is plugged into", + ], + ] as const)( + "%s answers UNSUPPORTED_IN_GATEWAY_MODE with a message pointing at the worker", + async (operation, input, role, message) => { + const { dispatcher } = harness(); + + await expect( + dispatcher.dispatch(operation, input, session({ role })), + ).rejects.toMatchObject({ + code: "UNSUPPORTED_IN_GATEWAY_MODE", + details: { operation }, + message, + }); + }, + ); + }); + + describe("daemon.update", () => { + it("asks the gateway's own update coordinator and answers with its outcome, the gateway's version and its health, and status.get carries the update and its last failure", async () => { + const { dispatcher, update } = harness(); + const run = { + forced: true, + from: "0.0.0-test", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.0.0-test", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + update.progress = { lastUpdateFailure: failure, latestVersion: "0.4.0", update: run }; + + await expect( + dispatcher.dispatch("daemon.update", { force: true }, session({ role: "admin" })), + ).resolves.toEqual({ + health: "running", + lastUpdateFailure: failure, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: run, + upToDate: false, + version: "0.0.0-test", + }); + expect(update.requests).toEqual([{ force: true }]); + + const status = await dispatcher.dispatch("status.get", {}, session({ role: "admin" })); + expect(status.daemon).toMatchObject({ lastUpdateFailure: failure, update: run }); + }); + }); +}); diff --git a/src/gateway/dispatcher-baselines.test.ts b/src/gateway/dispatcher-baselines.test.ts new file mode 100644 index 000000000..d23206ddf --- /dev/null +++ b/src/gateway/dispatcher-baselines.test.ts @@ -0,0 +1,520 @@ +import { describe, expect, it, vi } from "vitest"; +import { type BaselineEntry, SimlockError } from "../contract/index.js"; +import { + catalogFixture, + grantFixture, + leaseFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { GATEWAY_REQUESTER_PREFIX, harness, session } from "./test-dispatcher.js"; + +describe("GatewayDispatcher", () => { + describe("a lease request naming a baseline (#1208)", () => { + const CATALOG = catalogFixture([ + { + modelClasses: { "Pixel 9": "phone" } as const, + models: ["Pixel 9"], + platform: "android", + runtimes: ["36"], + }, + { + modelClasses: { "iPhone 17": "phone" } as const, + models: ["iPhone 17"], + platform: "ios", + runtimes: ["27.0"], + }, + ]).platforms; + const checkout: BaselineEntry = { + mode: "full", + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android", + savedAt: "2026-11-02T13:10:00Z", + version: 2, + }; + const smoke: BaselineEntry = { + ...checkout, + mode: "slim", + model: "iPhone 17", + name: "smoke", + osVersion: "27.0", + platform: "ios", + version: 1, + }; + const stale = (entry: BaselineEntry) => ({ + ...entry, + staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + }); + + function fleet(baselinesByWorker: Record) { + const rig = harness(); + const clients: Record = {}; + for (const [workerId, baselines] of Object.entries(baselinesByWorker)) { + const client = new ScriptedWorkerClient(); + clients[workerId] = client; + rig.directory.add(workerId, client); + rig.workers.connected(workerId, undefined, "0.3.0"); + rig.workers.refresh(workerId, { + baselines: [...baselines], + capacity: statusFixture().capacity, + catalog: CATALOG, + devices: [], + downloads: { policy: "on-request" }, + health: "running", + leases: [], + queueDepth: 0, + }); + } + return { ...rig, clients }; + } + + type Rejected = { readonly code: string; readonly message: string }; + const cases: readonly [ + string, + Record, + Record, + { readonly grantedBy: string } | { readonly rejected: Rejected }, + string[]?, + ][] = [ + [ + "one worker holds it, another has none", + { wrk_a: [], wrk_b: [checkout] }, + { baseline: "checkout", platform: "android" }, + { grantedBy: "wrk_b" }, + ], + [ + "an OS range that holds the baseline's OS", + { wrk_a: [], wrk_b: [checkout] }, + { baseline: "checkout", osVersion: ">=35", platform: "android" }, + { grantedBy: "wrk_b" }, + ], + [ + "no worker holds a baseline of the name", + { wrk_a: [checkout], wrk_b: [] }, + { baseline: "nothere", platform: "android" }, + { + rejected: { + code: "UNKNOWN_BASELINE", + message: 'no worker holds a baseline named "nothere"', + }, + }, + ], + [ + "every holder differs from the request: the first holder's message", + { wrk_a: [checkout], wrk_b: [{ ...checkout, osVersion: "35" }] }, + { baseline: "checkout", osVersion: "34", platform: "android" }, + { + rejected: { + code: "BASELINE_MISMATCH", + message: "baseline checkout is Android 36, the request asks for 34", + }, + }, + ], + [ + "every holder is stale: the first holder's reason", + { wrk_a: [stale(smoke)], wrk_b: [{ ...smoke, staleReason: "image is gone" }] }, + { baseline: "smoke", platform: "ios" }, + { + rejected: { + code: "BASELINE_STALE", + message: + "baseline smoke is stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + }, + }, + ], + [ + "every holder is stale and differs: the mismatch is checked first", + { wrk_a: [stale(smoke)] }, + { baseline: "smoke", mode: "full", platform: "ios" }, + { + rejected: { + code: "BASELINE_MISMATCH", + message: "baseline smoke is slim, the request asks for full", + }, + }, + ], + [ + "the only holder is disconnected", + { wrk_a: [checkout], wrk_b: [] }, + { baseline: "checkout", platform: "android" }, + { + rejected: { + code: "NO_CAPACITY", + message: "No worker in the fleet can currently serve this request", + }, + }, + ["wrk_a"], + ], + [ + "the only fresh holder is disconnected and the other holder is stale", + { wrk_a: [checkout], wrk_b: [stale(checkout)] }, + { baseline: "checkout", platform: "android" }, + { + rejected: { + code: "NO_CAPACITY", + message: "No worker in the fleet can currently serve this request", + }, + }, + ["wrk_a"], + ], + ]; + + it.each(cases)( + "sends a request naming a baseline only to a worker that holds it, fresh: %s", + async (_label, baselines, ask, outcome, disconnected = []) => { + const { clients, dispatcher, workers } = fleet(baselines); + for (const workerId of disconnected) workers.disconnected(workerId); + for (const client of Object.values(clients)) { + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + } + + // why: a rejection must come at once, so the request does not opt out of waiting and the answer is read after the queue is flushed; a request that waited instead would have none. + const answer = dispatcher.dispatch( + "lease.request", + ("rejected" in outcome ? ask : { ...ask, noWait: true }) as never, + session({ role: "agent" }), + ); + + if ("rejected" in outcome) { + let settled: unknown; + void answer.catch((error: unknown) => { + settled = error; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toMatchObject(outcome.rejected); + for (const client of Object.values(clients)) { + expect(client.lastRequestLeaseInput).toBeUndefined(); + } + return; + } + const grant = await answer; + expect(grant.lease.id).toMatch(new RegExp(`^${outcome.grantedBy}\\.`)); + for (const [workerId, client] of Object.entries(clients)) { + if (workerId === outcome.grantedBy) { + expect(client.lastRequestLeaseInput).toMatchObject({ baseline: ask["baseline"] }); + } else { + expect(client.lastRequestLeaseInput).toBeUndefined(); + } + } + }, + ); + + it.each([ + ["UNKNOWN_BASELINE", "no-baseline"], + ["BASELINE_MISMATCH", "baseline-mismatch"], + ["BASELINE_STALE", "baseline-stale"], + ] as const)( + "leaves out a holder that answers %s, asks the next holder, ends with that answer when none is left, and names %s for it while the request waits", + async (code, reason) => { + const refusal = () => + new SimlockError(code, "domain", `${code} from the worker`, {} as never); + const ask = { baseline: "checkout", noWait: true, platform: "android" } as const; + const next = fleet({ wrk_a: [checkout], wrk_b: [checkout] }); + next.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); + next.clients["wrk_b"]?.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await next.dispatcher.dispatch( + "lease.request", + ask, + session({ role: "agent" }), + ); + + expect(grant.lease.id).toMatch(/^wrk_b\./); + expect(next.directory.catalogRefreshCalls).toContain("wrk_a"); + + const alone = fleet({ wrk_a: [checkout] }); + alone.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); + + await expect( + alone.dispatcher.dispatch("lease.request", ask, session({ role: "agent" })), + ).rejects.toMatchObject({ code, message: `${code} from the worker` }); + + const busy = fleet({ wrk_a: [checkout], wrk_b: [checkout] }); + busy.workers.refresh("wrk_b", { queueDepth: 1 }); + busy.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); + void busy.dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform: "android" }, + session({ role: "agent" }), + ); + await vi.waitFor(async () => { + const status = await busy.dispatcher.dispatch("status.get", {}, session()); + expect(status.waiting?.[0]?.workers).toContainEqual({ reason, workerId: "wrk_a" }); + }); + }, + ); + }); + + describe("a request naming a baseline that waits (#1208)", () => { + it("waits while its only holder is busy, and status.get names why each other worker is passed over", async () => { + const entry: BaselineEntry = { + mode: "full", + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android", + savedAt: "2026-11-02T13:10:00Z", + version: 2, + }; + const held: Record = { + wrk_a: { baselines: [entry], queueDepth: 1 }, + wrk_b: { baselines: [], queueDepth: 0 }, + wrk_c: { baselines: [{ ...entry, staleReason: "image is gone" }], queueDepth: 0 }, + wrk_d: { baselines: [{ ...entry, mode: "slim" }], queueDepth: 0 }, + }; + const { directory, dispatcher, workers } = harness(); + for (const [workerId, view] of Object.entries(held)) { + directory.add(workerId, new ScriptedWorkerClient()); + workers.connected(workerId, undefined, "0.3.0"); + workers.refresh(workerId, { + ...view, + capacity: statusFixture().capacity, + catalog: catalogFixture([ + { + modelClasses: { "Pixel 9": "phone" } as const, + models: ["Pixel 9"], + platform: "android", + runtimes: ["36"], + }, + ]).platforms, + devices: [], + downloads: { policy: "on-request" }, + health: "running", + leases: [], + }); + } + + void dispatcher.dispatch( + "lease.request", + { baseline: "checkout", mode: "full", platform: "android" }, + session({ role: "agent" }), + ); + await vi.waitFor(async () => { + const status = await dispatcher.dispatch("status.get", {}, session()); + expect(status.waiting?.[0]?.workers).toEqual([ + { reason: "busy", workerId: "wrk_a" }, + { reason: "no-baseline", workerId: "wrk_b" }, + { reason: "baseline-stale", workerId: "wrk_c" }, + { reason: "baseline-mismatch", workerId: "wrk_d" }, + ]); + }); + }); + }); + + describe("a request naming a baseline held by a starting worker (#1208)", () => { + it("keeps the starting worker's last baselines for routing, so the request waits instead of failing, and shows none in its view", async () => { + const { directory, dispatcher, workers } = harness(); + directory.add("wrk_a", new ScriptedWorkerClient()); + workers.connected("wrk_a", undefined, "0.3.0"); + workers.refresh("wrk_a", { + baselines: [ + { + mode: "full", + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android", + savedAt: "2026-11-02T13:10:00Z", + version: 1, + }, + ], + capacity: statusFixture().capacity, + catalog: catalogFixture([ + { + modelClasses: { "Pixel 9": "phone" } as const, + models: ["Pixel 9"], + platform: "android", + runtimes: ["36"], + }, + ]).platforms, + devices: [], + health: "running", + leases: [], + queueDepth: 0, + }); + workers.refresh("wrk_a", { health: "starting" }); + + void dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform: "android" }, + session({ role: "agent" }), + ); + + await vi.waitFor(async () => { + const status = await dispatcher.dispatch("status.get", {}, session()); + expect(status.waiting).toHaveLength(1); + }); + expect(workers.view("wrk_a")).not.toHaveProperty("baselines"); + }); + }); + + describe("baselines through a gateway (#1208)", () => { + const entry = (name: string, overrides: Partial = {}): BaselineEntry => ({ + mode: "full", + model: "Pixel 9", + name, + osVersion: "36", + platform: "android", + savedAt: "2026-11-02T13:10:00Z", + version: 1, + ...overrides, + }); + + function workersWith(baselinesByWorker: Record) { + const rig = harness(); + const clients: Record = {}; + for (const [workerId, baselines] of Object.entries(baselinesByWorker)) { + const client = new ScriptedWorkerClient(); + client.baselines = baselines; + clients[workerId] = client; + rig.directory.add(workerId, client); + rig.workers.connected(workerId, undefined, "0.3.0"); + rig.workers.refresh(workerId, { + baselines, + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + }); + } + return { ...rig, clients }; + } + + it("saves a baseline on the worker that holds the lease, with the worker's lease id, and drops the lease from its index, or keeps it when the worker has no live link", async () => { + const { clients, directory, dispatcher, leaseIndex, workers } = workersWith({ + wrk_a: [], + wrk_b: [], + }); + workers.refresh("wrk_a", { + leases: [ + { + ...leaseFixture("L7", "dev_1"), + ownerId: "alice-principal", + requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, + }, + ], + }); + + const saved = await dispatcher.dispatch( + "baseline.save", + { leaseId: "wrk_a.L7", name: "checkout", note: "app 4.3" }, + session(), + ); + + expect(clients["wrk_a"]?.savedBaselines).toEqual([ + { leaseId: "L7", name: "checkout", note: "app 4.3" }, + ]); + expect(clients["wrk_b"]?.savedBaselines).toEqual([]); + expect(saved).toMatchObject({ baseline: { name: "checkout" }, leaseId: "wrk_a.L7" }); + expect(leaseIndex.resolve("wrk_a.L7")).toBeUndefined(); + expect(directory.catalogRefreshCalls).toEqual(["wrk_a"]); + + // why: a worker that no longer knows the lease drops the index entry; one with no live link answers WORKER_UNREACHABLE and keeps it. + const gatewayLease = (id: string, deviceId: string) => ({ + ...leaseFixture(id, deviceId), + ownerId: "alice-principal", + requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, + }); + workers.refresh("wrk_a", { leases: [gatewayLease("L8", "dev_2")] }); + workers.refresh("wrk_b", { leases: [gatewayLease("L9", "dev_3")] }); + expect(leaseIndex.resolve("wrk_a.L8")).toBeDefined(); + const unknown = new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { + leaseId: "L8", + }); + if (clients["wrk_a"] !== undefined) clients["wrk_a"].failWith = unknown; + await expect( + dispatcher.dispatch("baseline.save", { leaseId: "wrk_a.L8", name: "x" }, session()), + ).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); + expect(leaseIndex.resolve("wrk_a.L8")).toBeUndefined(); + + directory.clients.delete("wrk_b"); + await expect( + dispatcher.dispatch("baseline.save", { leaseId: "wrk_b.L9", name: "x" }, session()), + ).rejects.toMatchObject({ code: "WORKER_UNREACHABLE" }); + expect(leaseIndex.resolve("wrk_b.L9")).toBeDefined(); + }); + + it("lists every worker's baselines, each with its workerId, in worker id order, a disconnected worker's last ones too", async () => { + const { dispatcher, workers } = workersWith({ + wrk_b: [entry("smoke", { platform: "ios" })], + wrk_a: [entry("checkout", { version: 2 }), entry("login")], + wrk_c: [], + }); + workers.disconnected("wrk_b"); + + const listed = await dispatcher.dispatch("baseline.list", {}, session({ role: "agent" })); + + expect(listed.baselines.map(({ name, workerId }) => [workerId, name])).toEqual([ + ["wrk_a", "checkout"], + ["wrk_a", "login"], + ["wrk_b", "smoke"], + ]); + }); + + it("removes a baseline from the worker it names without sending the workerId, and refuses an unknown, a disconnected or no worker, and relays the worker's own refusals", async () => { + const { clients, directory, dispatcher, workers } = workersWith({ + wrk_a: [entry("smoke")], + wrk_b: [entry("smoke")], + }); + + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), + ).resolves.toMatchObject({ name: "smoke" }); + expect(clients["wrk_a"]?.removedBaselines).toEqual([{ name: "smoke" }]); + expect(directory.catalogRefreshCalls).toEqual(["wrk_a"]); + + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_x" }, session()), + ).rejects.toMatchObject({ code: "UNKNOWN_WORKER", message: "Unknown worker: wrk_x" }); + workers.disconnected("wrk_b"); + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_b" }, session()), + ).rejects.toMatchObject({ + code: "WORKER_NOT_CONNECTED", + message: "worker wrk_b is not connected", + }); + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke" }, session()), + ).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: + "baseline.remove on a gateway needs the worker that holds the baseline: pass --worker ", + }); + expect(clients["wrk_a"]?.removedBaselines).toHaveLength(1); + expect(clients["wrk_b"]?.removedBaselines).toEqual([]); + + // why: a worker's own refusal reaches the caller with its code and words. + for (const failure of [ + new SimlockError("BASELINE_IN_USE", "domain", "worker's words", { + leaseId: "L1", + name: "smoke", + }), + new SimlockError("UNKNOWN_BASELINE", "domain", "worker's words", { name: "smoke" }), + ]) { + const client = clients["wrk_a"]; + if (client !== undefined) client.failWith = failure; + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), + ).rejects.toMatchObject({ code: failure.code, message: "worker's words" }); + } + + // why: an uplink that drops under the call is the gateway's WORKER_NOT_CONNECTED, not the client's transport code. + const client = clients["wrk_a"]; + if (client !== undefined) { + client.failWith = new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + "Daemon connection closed", + {}, + ); + } + await expect( + dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), + ).rejects.toMatchObject({ + code: "WORKER_NOT_CONNECTED", + message: "worker wrk_a is not connected", + }); + }); + }); +}); diff --git a/src/gateway/dispatcher-events-tokens.test.ts b/src/gateway/dispatcher-events-tokens.test.ts new file mode 100644 index 000000000..43920bc57 --- /dev/null +++ b/src/gateway/dispatcher-events-tokens.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest"; +import { type EventEnvelope } from "../bus/index.js"; +import { fixedEventHistory, usageRig } from "../history/testing.js"; +import { harness, session } from "./test-dispatcher.js"; + +describe("GatewayDispatcher", () => { + it("replays and subscribes to its own bus, which carries the fleet's events", async () => { + const { dispatcher, eventBus } = harness(); + eventBus.emit( + "lease.granted", + { + deviceId: "dev_1", + leaseId: "l1", + platform: "ios", + requester: "a", + requestId: "req_1", + source: "warm", + }, + "worker", + ); + + await expect(dispatcher.dispatch("events.replay", {}, session())).resolves.toEqual([ + expect.objectContaining({ event: "lease.granted" }), + ]); + await expect(dispatcher.dispatch("events.subscribe", {}, session())).resolves.toEqual({ + subscribed: true, + subscriptionId: "sub_1", + }); + }); + + it("events.replay with sinceTs returns what the event history returns", async () => { + const fromHistory: EventEnvelope[] = [ + { + id: "evt_7", + seq: 7, + timestamp: 50, + event: "daemon.stopping", + payload: { reason: "x" }, + module: "d", + }, + ]; + const asked: unknown[] = []; + const { dispatcher } = harness({ + eventHistory: { + replay: async (input) => { + asked.push(input); + return fromHistory; + }, + replayPage: async () => ({ events: [], kind: "events" }), + status: () => ({ opened: true, writing: true }), + }, + }); + + await expect(dispatcher.dispatch("events.replay", { sinceTs: 10 }, session())).resolves.toEqual( + fromHistory, + ); + expect(asked).toEqual([{ sinceTs: 10 }]); + }); + + it("events.replay with beforeId returns a worker's relayed event with payload.workerId", async () => { + const rig = await usageRig({ fleet: true }); + rig.relay("w-1", "daemon.stopping", { reason: "from-worker" }); + rig.emit("daemon.stopping", { reason: "local" }); + const newest = rig.bus.replay().at(-1) as { id: string }; + const { dispatcher } = harness({ eventHistory: rig.history }); + + const page = await dispatcher.dispatch( + "events.replay", + { beforeId: newest.id, limit: 5 }, + session(), + ); + + expect(page).toEqual([ + expect.objectContaining({ payload: expect.objectContaining({ workerId: "w-1" }) }), + ]); + }); + + it("events.replay with limit alone returns the newest n events, oldest first, through the gateway dispatcher", async () => { + const rig = await usageRig({ fleet: true }); + for (const reason of ["a", "b", "c"]) rig.emit("daemon.stopping", { reason }); + const { dispatcher } = harness({ eventHistory: rig.history }); + + const page = await dispatcher.dispatch("events.replay", { limit: 2 }, session()); + + expect(page.map((event) => (event.payload as { reason: string }).reason)).toEqual(["b", "c"]); + }); + + it("events.replay with beforeId and no limit returns at most 100 events through the gateway dispatcher", async () => { + const rig = await usageRig({ fleet: true }); + for (let n = 0; n < 130; n += 1) rig.emit("daemon.stopping", { reason: `r${n}` }); + const newest = rig.bus.replay().at(-1) as { id: string }; + const { dispatcher } = harness({ eventHistory: rig.history }); + + const page = await dispatcher.dispatch("events.replay", { beforeId: newest.id }, session()); + + expect(page).toHaveLength(100); + }); + + it.each([ + { + health: "running" as const, + history: { opened: true, setAsideAt: 1_760_605_920_000, writing: false }, + }, + { health: "starting" as const, history: { opened: false, writing: false } }, + { health: "running" as const, history: { opened: true, writing: true } }, + { + health: "running" as const, + history: { opened: true, rebuiltAt: 1_760_605_920_000, writing: true }, + }, + { + health: "running" as const, + history: { opened: true, rebuildFailed: true as const, writing: true }, + }, + ])( + "status.get carries the gateway's own history opened, writing, setAsideAt, rebuiltAt and rebuildFailed, in the starting answer too ($health, $history)", + async ({ health, history }) => { + const { dispatcher } = harness({ eventHistory: fixedEventHistory(history), health }); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + expect((status as { history?: unknown }).history).toStrictEqual(history); + }, + ); + + it("mints its own tokens, worker join tokens included", async () => { + const { dispatcher, tokens } = harness(); + + await expect( + dispatcher.dispatch("token.create", { role: "worker", label: "mac-mini-1" }, session()), + ).resolves.toMatchObject({ secret: "slk_secret", token: { role: "worker" } }); + expect(tokens.created).toEqual(["worker"]); + }); + + it("closes the uplink a revoked token authorized, not merely the store entry (C-2)", async () => { + const { closedUplinkTokens, dispatcher } = harness(); + + await expect(dispatcher.dispatch("token.revoke", { id: "tok_1" }, session())).resolves.toEqual({ + revoked: true, + }); + + expect(closedUplinkTokens).toEqual(["tok_1"]); + }); + + it("does not try to close an uplink for a token id that was never real (C-2)", async () => { + const { closedUplinkTokens, dispatcher } = harness(); + + await expect( + dispatcher.dispatch("token.revoke", { id: "tok_unknown" }, session()), + ).resolves.toEqual({ revoked: false }); + + expect(closedUplinkTokens).toEqual([]); + }); +}); diff --git a/src/gateway/dispatcher-lease-lifecycle.test.ts b/src/gateway/dispatcher-lease-lifecycle.test.ts new file mode 100644 index 000000000..998add901 --- /dev/null +++ b/src/gateway/dispatcher-lease-lifecycle.test.ts @@ -0,0 +1,526 @@ +import { describe, expect, it } from "vitest"; +import { + catalogFixture, + grantFixture, + leaseFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { GATEWAY_REQUESTER_PREFIX, harness, session } from "./test-dispatcher.js"; + +describe("GatewayDispatcher", () => { + describe("lease lifecycle and device.exec (#118)", () => { + it("forwards lease.request through the fleet coordinator and answers UNKNOWN_LEASE for renew/release/exec against an id it never issued", async () => { + const { clock, dispatcher } = harness(); + + await expect( + dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", noWait: true, platform: "ios" }, + session({ role: "agent" }), + ), + ).rejects.toMatchObject({ code: "NO_CAPACITY" }); + + const ghostCalls = [ + ["lease.renew", { leaseId: "wrk_ghost.lease_1" }], + ["lease.release", { leaseId: "wrk_ghost.lease_1" }], + ["device.exec", { args: [], leaseId: "wrk_ghost.lease_1", tool: "adb" }], + ] as const; + // why: inside the gateway's first minute a generated id may belong to a worker yet to report, which answers WORKER_UNREACHABLE. + clock.advance(59_999); + for (const [operation, input] of ghostCalls) { + await expect( + dispatcher.dispatch(operation, input, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "WORKER_UNREACHABLE" }); + } + clock.advance(1); + for (const [operation, input] of ghostCalls) { + await expect( + dispatcher.dispatch(operation, input, session({ role: "agent" })), + ).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); + } + }); + + it("refuses a lease.request that names no requesterId when the session principal has a control character", async () => { + const { dispatcher } = harness(); + + await expect( + dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", noWait: true, platform: "ios" }, + session({ principal: "a\u001b[2Jb", role: "agent" }), + ), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + }); + + it.each([ + ["with noWait: true", { noWait: true }], + ["without noWait", {}], + ])( + "refuses a lease.request carrying fleetRequestId with FORBIDDEN from every session, even the uplink, %s", + async (_label, extra) => { + const { dispatcher } = harness(); + const input = { fleetRequestId: "req_gw1", model: "iPhone 17", platform: "ios", ...extra }; + + for (const refused of [ + session({ role: "agent" }), + session({ role: "admin" }), + session({ isGatewayUplink: true, role: "admin" }), + ]) { + await expect(dispatcher.dispatch("lease.request", input, refused)).rejects.toMatchObject({ + code: "FORBIDDEN", + message: expect.stringContaining("fleetRequestId"), + }); + } + }, + ); + + it("answers not-found for lease.cancel and an empty result for lease.release-all with nothing queued or leased", async () => { + const { dispatcher } = harness(); + + await expect( + dispatcher.dispatch("lease.cancel", {}, session({ role: "agent" })), + ).resolves.toEqual({ result: "not-found" }); + await expect(dispatcher.dispatch("lease.release-all", {}, session())).resolves.toEqual({ + leaseIds: [], + }); + }); + + // why: ADR §27a's whole point: the owner a rebuilt lease authorizes against has to be the real + // principal the worker round-tripped back, not this gateway's own uplink principal -- and it + // has to work on an index built purely by `rebuildFromWorker` (a reconnect), never through + // `coordinator.request` in this process. This is the one place a naive "derive ownerId from + // the connection" implementation and a correct one visibly disagree. + it("authorizes lease.renew for the original requester and forbids everyone else, on a lease rebuilt after reconnect", async () => { + const { dispatcher, directory, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { + leases: [ + { + ...leaseFixture("lease_1", "dev_1"), + ownerId: "alice-principal", + requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, + }, + ], + }); + const gatewayLeaseId = "wrk_1.lease_1"; + client.renewLeaseQueue.push({ + kind: "ok", + record: { + ...leaseFixture("lease_1", "dev_1"), + ownerId: "alice-principal", + ttlMs: 1_800_000, + }, + }); + + await expect( + dispatcher.dispatch( + "lease.renew", + { leaseId: gatewayLeaseId }, + session({ principal: "alice-principal", role: "agent" }), + ), + ).resolves.toMatchObject({ id: gatewayLeaseId, requesterId: "alice" }); + + await expect( + dispatcher.dispatch( + "lease.renew", + { leaseId: gatewayLeaseId }, + session({ principal: "mallory-principal", role: "agent" }), + ), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it.each([ + ["mode slim", { mode: "slim" as const }], + ["no mode", {}], + ])("forwards a lease.request with %s to the worker as it arrived", async (_label, mode) => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", noWait: true, platform: "ios", ...mode }, + session({ role: "agent" }), + ); + + if ("mode" in mode) expect(client.lastRequestLeaseInput).toMatchObject(mode); + else expect(client.lastRequestLeaseInput).not.toHaveProperty("mode"); + }); + + it.each([ + ["a class", { class: "phone", platform: "ios" }], + ["neither a model nor a class", { platform: "ios" }], + ["a class and an OS range", { class: "phone", osVersion: ">=18", platform: "ios" }], + ["a model and an OS range", { model: "iPhone 17", osVersion: ">=18", platform: "ios" }], + [ + "a model and an exact osVersion", + { model: "iPhone 17", osVersion: "26.0", platform: "ios" }, + ], + ] as const)( + "serves a lease.request naming %s through a worker that fits, forwarding it as it arrived", + async (_label, input) => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([ + { + modelClasses: { "iPhone 17": "phone" } as const, + models: ["iPhone 17"], + platform: "ios", + runtimes: ["26.0"], + }, + ]).platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + await dispatcher.dispatch( + "lease.request", + { ...input, noWait: true }, + session({ role: "agent" }), + ); + + expect(client.lastRequestLeaseInput).toMatchObject(input); + if (!("model" in input)) expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); + }, + ); + + it("forwards a lease.request's image tag and baseline to the worker unchanged", async () => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + baselines: [ + { + imageTag: "google_apis_playstore", + mode: "full", + model: "Pixel 8", + name: "checkout", + osVersion: "34", + platform: "android", + savedAt: "2026-11-02T13:10:00Z", + version: 1, + }, + ], + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([ + { + images: [{ abi: "arm64-v8a", runtime: "34", tag: "google_apis_playstore" }], + models: ["Pixel 8"], + platform: "android", + runtimes: ["34"], + }, + ]).platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + await dispatcher.dispatch( + "lease.request", + { + baseline: "checkout", + imageTag: "google_apis_playstore", + model: "Pixel 8", + noWait: true, + platform: "android", + }, + session({ role: "agent" }), + ); + + expect(client.lastRequestLeaseInput).toMatchObject({ + baseline: "checkout", + imageTag: "google_apis_playstore", + }); + }); + + it("forwards a lease.request's leaseId to the worker, and answers the grant under that ID with no worker in front of it", async () => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { ...grantFixture().lease, id: "ad-7f3a", idChosenByRequester: true }, + }), + kind: "grant", + }); + + const grant = await dispatcher.dispatch( + "lease.request", + { leaseId: "ad-7f3a", model: "iPhone 17", noWait: true, platform: "ios" }, + session({ role: "agent" }), + ); + + expect(client.lastRequestLeaseInput).toMatchObject({ leaseId: "ad-7f3a" }); + expect(grant.lease).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); + }); + + it("forwards device.exec to the worker that holds the lease, gated on the caller owning it", async () => { + const { coordinator, directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await coordinator.request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, + ); + client.execQueue.push({ exitCode: 3, kind: "ok" }); + + await expect( + dispatcher.dispatch( + "device.exec", + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + session({ principal: "agent-1", role: "agent" }), + ), + ).resolves.toEqual({ exitCode: 3 }); + + await expect( + dispatcher.dispatch( + "device.exec", + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + session({ principal: "someone-else", role: "agent" }), + ), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }); + + it("refuses a device.exec whose stdin is longer than 8,388,608 characters with BAD_REQUEST and forwards nothing to the worker, and forwards one at 8,388,608", async () => { + const { coordinator, directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, + ); + const exec = (stdin: string) => + dispatcher.dispatch( + "device.exec", + { args: ["spawn", "booted", "untar"], leaseId: grant.lease.id, stdin, tool: "simctl" }, + session({ principal: "agent-1", role: "agent" }), + ); + const forwarded = () => client.calls.filter((call) => call.startsWith("device.exec:")); + + client.execQueue.push({ exitCode: 0, kind: "ok" }); + await expect(exec("A".repeat(8_388_608))).resolves.toEqual({ exitCode: 0 }); + expect(forwarded()).toHaveLength(1); + + await expect(exec("A".repeat(8_388_609))).rejects.toMatchObject({ code: "BAD_REQUEST" }); + expect(forwarded()).toHaveLength(1); + }); + + it("forwards a device.exec's stdinFile to the lease's worker under the same name, and nothing when it is omitted", async () => { + const { coordinator, directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, + ); + const exec = (extra: { stdinFile?: string }) => { + client.execQueue.push({ exitCode: 0, kind: "ok" }); + return dispatcher.dispatch( + "device.exec", + { args: ["spawn", "booted", "untar"], leaseId: grant.lease.id, tool: "simctl", ...extra }, + session({ principal: "agent-1", role: "agent" }), + ); + }; + + await exec({ stdinFile: "fixture.tar" }); + await exec({}); + + expect(client.execInputs.map((forwarded) => forwarded.stdinFile)).toEqual([ + "fixture.tar", + undefined, + ]); + expect("stdinFile" in (client.execInputs[1] ?? {})).toBe(false); + }); + + it("calls onStarted for a silent device.exec from the worker's own started push (ADR §19a/§19b/§19e)", async () => { + const { coordinator, directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, + ); + // why: §19b's own worked example: `simctl install ` never writes anything while it runs, + // and the worker here never answers at all -- but it does say the process exists, which is + // the fact a transport needs to commit its 200. No clock is advanced below: this asserts + // the relay, not a timer. + client.execQueue.push({ kind: "hang", started: true }); + + let started = false; + void dispatcher.dispatch( + "device.exec", + { args: ["install", "/path/to.app"], leaseId: grant.lease.id, tool: "simctl" }, + session({ + onStarted: () => { + started = true; + }, + principal: "agent-1", + role: "agent", + }), + ); + await expect.poll(() => started).toBe(true); + }); + + it.each([ + { owner: "someone-else", principal: "agent-1", role: "agent" as const }, + { owner: "agent-7", principal: "operator-1", role: "admin" as const }, + ])( + "lease.request: rejects a $role session naming owner with FORBIDDEN, never silently ignoring it (ADR §27a, H3, H7)", + async ({ owner, principal, role }) => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + await expect( + dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", noWait: true, owner, platform: "ios" }, + session({ principal, role }), + ), + ).rejects.toMatchObject({ code: "FORBIDDEN" }); + }, + ); + + it("lease.request: forwards owner as `ownerId` end to end when the session is the gateway's own uplink (ADR §27a, H3) -- exercising the same field the worker's own dispatcher gates, on this dispatcher's shared handler shape", async () => { + const { directory, dispatcher, leaseIndex, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", noWait: true, owner: "agent-7", platform: "ios" }, + session({ isGatewayUplink: true, principal: "gw:instance-1", role: "admin" }), + ); + + const gatewayLeaseId = (grant as { lease: { id: string } }).lease.id; + expect((grant as { lease: { ownerId: string } }).lease.ownerId).toBe("agent-7"); + expect(leaseIndex.ownerId(gatewayLeaseId)).toBe("agent-7"); + }); + + it("lease.request: names the stored request to the session and replays it under the same idempotency key, forwarding nothing twice", async () => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + health: "running", + queueDepth: 0, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + downloads: { policy: "on-request" }, + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const admitted: [string, boolean][] = []; + const input = { idempotencyKey: "key-1", model: "iPhone 17", platform: "ios" } as const; + const track = session({ + onRequestAdmitted: (id, replayed) => admitted.push([id, replayed]), + principal: "agent-1", + role: "agent", + }); + + const first = await dispatcher.dispatch("lease.request", input, track); + const repeat = await dispatcher.dispatch("lease.request", input, track); + + expect((repeat as { lease: { id: string } }).lease.id).toBe( + (first as { lease: { id: string } }).lease.id, + ); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + expect(admitted).toEqual([ + [expect.stringMatching(/^req_/), false], + [admitted[0]?.[0], true], + ]); + }); + }); +}); diff --git a/src/gateway/dispatcher-leases.test.ts b/src/gateway/dispatcher-leases.test.ts new file mode 100644 index 000000000..bc7fd6ab9 --- /dev/null +++ b/src/gateway/dispatcher-leases.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, it } from "vitest"; +import { + catalogFixture, + deviceFixture, + grantFixture, + leaseFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { GATEWAY_REQUESTER_PREFIX, GATEWAY_HOST, harness, session } from "./test-dispatcher.js"; + +describe("GatewayDispatcher", () => { + it("lists the fleet's leases and devices, each naming its worker", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { + devices: [deviceFixture("dev_1", "leased")], + leases: [leaseFixture("lease_1", "dev_1")], + }); + + await expect(dispatcher.dispatch("lease.list", {}, session())).resolves.toEqual({ + leases: [expect.objectContaining({ id: "lease_1", workerId: "wrk_1" })], + }); + await expect(dispatcher.dispatch("list.get", { kind: "devices" }, session())).resolves.toEqual([ + expect.objectContaining({ id: "dev_1", workerId: "wrk_1" }), + ]); + // why: Cleanup rules are a machine's own configuration, and a gateway runs no reaper. + await expect(dispatcher.dispatch("list.get", { kind: "rules" }, session())).resolves.toEqual( + [], + ); + }); + + it("lease.get reads one lease by its gateway id for its owner or an admin, a worker's own local lease for an admin only, and answers UNKNOWN_LEASE otherwise", async () => { + const { dispatcher, leaseIndex, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", undefined); + const issued = { + ...leaseFixture("lease_1", "dev_1"), + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + }; + const local = { ...leaseFixture("local_1", "dev_2"), ownerId: "worker-agent" }; + workers.refresh("wrk_1", { leases: [issued, local] }); + const owner = session({ principal: "agent-1", role: "agent" }); + const stranger = session({ principal: "agent-2", role: "agent" }); + const get = (leaseId: string, who: ReturnType) => + dispatcher.dispatch("lease.get", { leaseId }, who); + + await expect(get("wrk_1.lease_1", owner)).resolves.toEqual({ + lease: expect.objectContaining({ id: "wrk_1.lease_1", workerId: "wrk_1" }), + }); + await expect(get("wrk_1.lease_1", session())).resolves.toMatchObject({ + lease: { id: "wrk_1.lease_1" }, + }); + await expect(get("local_1", session())).resolves.toMatchObject({ + lease: { id: "local_1", workerId: "wrk_1" }, + }); + const unknown = { code: "UNKNOWN_LEASE", details: { leaseId: expect.any(String) } }; + await expect(get("wrk_1.lease_1", stranger)).rejects.toMatchObject(unknown); + await expect(get("local_1", stranger)).rejects.toMatchObject(unknown); + await expect(get("wrk_1.lease_nope", session())).rejects.toMatchObject(unknown); + await expect(get("lease_1", session())).rejects.toMatchObject(unknown); + // why: an index entry can outlive its worker view, or the view its lease. + for (const [workerId, workerLeaseId] of [ + ["wrk_gone", "lease_9"], + ["wrk_1", "lease_gone"], + ] as const) { + leaseIndex.add({ + gatewayLeaseId: `${workerId}.${workerLeaseId}`, + grantedAt: 1, + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + workerId, + workerLeaseId, + }); + await expect(get(`${workerId}.${workerLeaseId}`, session())).rejects.toMatchObject(unknown); + await expect(get(`${workerId}.${workerLeaseId}`, owner)).rejects.toMatchObject(unknown); + } + }); + + it("names the worker's own lease id as workerLeaseId on the leases of lease.list, lease.get and list.get kind=leases, never on status.get", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", undefined); + const issued = (id: string, chosen: boolean) => ({ + ...leaseFixture(id, `dev_${id}`), + idChosenByRequester: chosen, + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + }); + workers.refresh("wrk_1", { + leases: [ + issued("lse_42", false), + issued("my-lease", true), + leaseFixture("local_1", "dev_local"), + ], + }); + const expected = [ + { id: "wrk_1.lse_42", workerLeaseId: "lse_42" }, + { id: "my-lease", workerLeaseId: "my-lease" }, + { id: "local_1", workerLeaseId: "local_1" }, + ]; + const pick = (leases: readonly { id: string; workerLeaseId?: string | undefined }[]) => + leases.map(({ id, workerLeaseId }) => ({ id, workerLeaseId })); + + const listed = await dispatcher.dispatch("lease.list", {}, session()); + expect(pick(listed.leases)).toEqual(expected); + const fleet = await dispatcher.dispatch("list.get", { kind: "leases" }, session()); + expect(pick(fleet as never)).toEqual(expected); + const owned = await dispatcher.dispatch( + "lease.list", + {}, + session({ principal: "agent-1", role: "agent" }), + ); + expect(pick(owned.leases)).toEqual(expected.slice(0, 2)); + for (const { id, workerLeaseId } of expected) { + const { lease } = await dispatcher.dispatch("lease.get", { leaseId: id }, session()); + expect(lease).toMatchObject({ id, workerLeaseId }); + } + const status = await dispatcher.dispatch("status.get", {}, session()); + expect((status.leases ?? []).map((lease) => Object.keys(lease))).toEqual( + expected.map(() => expect.not.arrayContaining(["workerLeaseId"])), + ); + }); + + it.each([ + ["an agent", session({ principal: "agent-1", role: "agent" })], + ["an admin", session({ principal: "claude-code:1", role: "admin" })], + ])( + "lease.list lists the lease to %s that asked for it as soon as lease.request answers, before the worker's own lease event has refreshed the view", + async (_who, asker) => { + const { directory, dispatcher, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + workers.connected("wrk_1", undefined, "0.3.0"); + workers.refresh("wrk_1", { + capacity: statusFixture().capacity, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + devices: [deviceFixture("dev_1")], + downloads: { policy: "on-request" }, + health: "running", + leases: [], + queueDepth: 0, + }); + const workerLease = { + ...grantFixture().lease, + ownerId: asker.principal, + requesterId: `${GATEWAY_REQUESTER_PREFIX}${asker.principal}`, + }; + client.requestLeaseQueue.push({ grant: grantFixture({ lease: workerLease }), kind: "grant" }); + // why: the worker lists the lease once the gateway reads its view again; the read its `lease.granted` push starts has not landed. + directory.onRefresh.set("wrk_1", async () => { + await new Promise((resolve) => setImmediate(resolve)); + workers.refresh("wrk_1", { + devices: [deviceFixture("dev_1", "leased")], + leases: [workerLease], + }); + }); + + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17", platform: "ios" }, + asker, + ); + const { leases } = await dispatcher.dispatch("lease.list", {}, asker); + + expect((grant as { lease: { id: string } }).lease.id).toBe("wrk_1.lse_1"); + expect(leases.map(({ id, requesterId }) => ({ id, requesterId }))).toEqual([ + { id: "wrk_1.lse_1", requesterId: asker.principal }, + ]); + }, + ); + + it("lists no device or lease for a worker that is starting, whether the session is an admin or the owner of a lease the gateway indexed on it, and answers worker.list with the view as it is", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", undefined); + const lease = { + ...leaseFixture("lease_1", "dev_1"), + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + }; + workers.refresh("wrk_1", { + devices: [deviceFixture("dev_1", "leased")], + health: "running", + leases: [lease], + }); + const owner = session({ principal: "agent-1", role: "agent" }); + await expect(dispatcher.dispatch("lease.list", {}, owner)).resolves.toEqual({ + leases: [expect.objectContaining({ id: "wrk_1.lease_1" })], + }); + + workers.refresh("wrk_1", { health: "starting", host: GATEWAY_HOST }); + + await expect(dispatcher.dispatch("lease.list", {}, session())).resolves.toEqual({ + leases: [], + }); + await expect(dispatcher.dispatch("lease.list", {}, owner)).resolves.toEqual({ leases: [] }); + await expect(dispatcher.dispatch("list.get", { kind: "devices" }, session())).resolves.toEqual( + [], + ); + const { workers: views } = await dispatcher.dispatch("worker.list", {}, session()); + expect(views).toHaveLength(1); + expect(views[0]).toMatchObject({ health: "starting", host: GATEWAY_HOST, id: "wrk_1" }); + expect(Object.keys(views[0] ?? {})).not.toContain("leases"); + }); + + it("inbox.get answers the items the fleet's views, leases, waiting requests and tokens make", async () => { + const holders = async (options: Parameters[0]) => { + const { clock, dispatcher, workers } = harness(options); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + workers.refresh("wrk_1", { + devices: [deviceFixture("dev_1", "leased")], + health: "running", + leases: [{ ...leaseFixture("lease_1", "dev_1"), ttlDeadline: 121_000 }], + waiting: [ + { + createdAt: 500, + id: "req_1", + requesterId: "agent-2", + spec: { model: "iPhone 17", platform: "ios" }, + stage: "starting", + }, + ], + }); + workers.connected("wrk_2", "mac-mini-2", "0.3.0"); + workers.disconnected("wrk_2"); + clock.advance(60_000); + const { items } = await dispatcher.dispatch("inbox.get", {}, session()); + return items; + }; + + const named = await holders({ tokenRecords: [{ id: "agent-1", label: "ci-runner" }] }); + + expect(named).toEqual([ + expect.objectContaining({ condition: "disconnected", key: "wrk_2/disconnected" }), + expect.objectContaining({ condition: "waiting", key: "request/req_1/waiting" }), + expect.objectContaining({ + condition: "ending", + key: "lease/lease_1/ending", + lease: expect.objectContaining({ holder: "ci-runner", model: "iPhone 17" }), + }), + ]); + const unnamed = await holders({ withoutTokens: true }); + expect(unnamed).toContainEqual( + expect.objectContaining({ lease: expect.objectContaining({ holder: "agent-1" }) }), + ); + }); + + it("lists each of an owner's gateway-issued leases under its own record, and none whose worker has no view", async () => { + const { dispatcher, leaseIndex, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", undefined); + const owned = (id: string, deviceId: string) => ({ + ...leaseFixture(id, deviceId), + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + ttlMs: id === "lease_1" ? 111 : 222, + }); + workers.refresh("wrk_1", { leases: [owned("lease_1", "dev_1"), owned("lease_2", "dev_2")] }); + // why: An entry for a worker the gateway holds no view of, as a race with a removal leaves it. + leaseIndex.add({ + gatewayLeaseId: "wrk_gone.lease_9", + grantedAt: 1, + ownerId: "agent-1", + requesterId: "agent-1", + workerId: "wrk_gone", + workerLeaseId: "lease_9", + }); + + const { leases } = await dispatcher.dispatch( + "lease.list", + {}, + session({ principal: "agent-1", role: "agent" }), + ); + + expect(leases.map(({ id, ttlMs }) => ({ id, ttlMs }))).toEqual([ + { id: "wrk_1.lease_1", ttlMs: 111 }, + { id: "wrk_1.lease_2", ttlMs: 222 }, + ]); + }); + + it("shows a non-admin session none of a worker's own local leases", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); + + await expect( + dispatcher.dispatch("lease.list", {}, session({ principal: "someone", role: "agent" })), + ).resolves.toEqual({ leases: [] }); + }); + + it("does not let a gateway session's raw principal match a worker's own local lease owner of the same name", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", undefined, undefined); + workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); + + await expect( + dispatcher.dispatch("lease.list", {}, session({ principal: "agent-1", role: "agent" })), + ).resolves.toEqual({ leases: [] }); + }); + + it("sees a gateway-issued lease rebuilt from a worker's view, rewritten to its gateway id", async () => { + const { dispatcher, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", undefined); + const lease = { + ...leaseFixture("lease_1", "dev_1"), + ownerId: "agent-1", + requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, + }; + workers.refresh("wrk_1", { leases: [lease] }); + + await expect( + dispatcher.dispatch("lease.list", {}, session({ principal: "agent-1", role: "agent" })), + ).resolves.toEqual({ + leases: [ + expect.objectContaining({ + id: "wrk_1.lease_1", + requesterId: "agent-1", + worker: { id: "wrk_1", label: "mac-mini-1" }, + }), + ], + }); + }); + + it("status.get: an agent sees only the fleet leases and fleet requests it owns, one that owns none sees empty lists, admin sees all", async () => { + const { coordinator, directory, dispatcher, workers } = harness(); + directory.add("wrk_1", new ScriptedWorkerClient()); + const capacity = statusFixture().capacity; + const issued = (id: string, deviceId: string, ownerId: string) => ({ + ...leaseFixture(id, deviceId), + ownerId, + requesterId: `${GATEWAY_REQUESTER_PREFIX}${ownerId}`, + }); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + workers.refresh("wrk_1", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 0 } }, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + health: "running", + // why: lease_3 is the worker's own local lease, owned by a local "agent-1" this gateway never issued to. + leases: [ + issued("lease_1", "dev_1", "agent-1"), + issued("lease_2", "dev_2", "agent-2"), + leaseFixture("lease_3", "dev_3"), + ], + queueDepth: 0, + }); + const wait = (ownerId: string, requesterId: string) => + void coordinator + .request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: false, ownerId, requesterId }, + ) + .catch(() => undefined); + wait("agent-1", "fleet-a"); + wait("agent-3", "fleet-b"); + await expect.poll(() => coordinator.queueDepth).toBe(2); + + const cases = [ + [ + "admin", + { principal: "operator-1", role: "admin" }, + ["wrk_1.lease_1", "wrk_1.lease_2", "lease_3"], + ["fleet-a", "fleet-b"], + ], + [ + "an agent with a lease and a waiting request", + { principal: "agent-1", role: "agent" }, + ["wrk_1.lease_1"], + ["fleet-a"], + ], + [ + "an agent with a lease only", + { principal: "agent-2", role: "agent" }, + ["wrk_1.lease_2"], + [], + ], + [ + "an agent with a waiting request only", + { principal: "agent-3", role: "agent" }, + [], + ["fleet-b"], + ], + ["an agent that owns nothing", { principal: "nobody", role: "agent" }, [], []], + ] as const; + const seen = await Promise.all( + cases.map(async ([name, who]) => { + const status = await dispatcher.dispatch("status.get", {}, session(who)); + return { + leases: status.leases?.map((lease) => lease.id), + name, + waiting: status.waiting?.map((request) => request.requesterId), + }; + }), + ); + + expect(seen).toEqual(cases.map(([name, , leases, waiting]) => ({ leases, name, waiting }))); + + // why: `workers[]` repeats each worker's own leases and waiting requests, so an agent must not read them there. + const held = (status: Awaited>>) => + (status.workers ?? []).flatMap((worker) => [ + ...(worker.leases ?? []).map((lease) => lease.id), + ...(worker.waiting ?? []).map((request) => request.requesterId), + ]); + expect( + held( + await dispatcher.dispatch( + "status.get", + {}, + session({ principal: "operator-1", role: "admin" }), + ), + ), + ).not.toEqual([]); + for (const principal of ["agent-1", "agent-2", "nobody"]) { + expect( + held(await dispatcher.dispatch("status.get", {}, session({ principal, role: "agent" }))), + principal, + ).toEqual([]); + } + }); +}); diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 37b3cff0c..ea8e6c398 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -1,35 +1,14 @@ import { describe, expect, it, vi } from "vitest"; -import { createHmac } from "node:crypto"; -import { createServer } from "node:net"; -import type { EventEnvelope } from "../bus/index.js"; -import type { AlertDestination } from "../core/index.js"; -import { fixedEventHistory, usageRig } from "../history/testing.js"; -import { - type BaselineEntry, - OPERATIONS, - SimlockError, - type OperationName, -} from "../contract/index.js"; +import { OPERATIONS, SimlockError, type OperationName } from "../contract/index.js"; import { DispatchError } from "../daemon/dispatch.js"; -import { FetchWebhookPoster, type WebhookPoster } from "../ports/index.js"; import { - catalogFixture, deviceFixture, - grantFixture, hostFixture, leaseFixture, ScriptedWorkerClient, statusFixture, } from "./test-support.js"; -import { - gatewayConfig, - GATEWAY_REQUESTER_PREFIX, - GATEWAY_HOST, - harness, - session, -} from "./test-dispatcher.js"; - -type SentRequest = Parameters[0]; +import { gatewayConfig, GATEWAY_HOST, harness, session } from "./test-dispatcher.js"; const EVERY_OPERATION = (Object.keys(OPERATIONS) as OperationName[]).filter( (name) => name !== "daemon.stop", @@ -503,1972 +482,6 @@ describe("GatewayDispatcher", () => { ).rejects.toMatchObject({ code: "FORBIDDEN" }); }); - it("lists the fleet's leases and devices, each naming its worker", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { - devices: [deviceFixture("dev_1", "leased")], - leases: [leaseFixture("lease_1", "dev_1")], - }); - - await expect(dispatcher.dispatch("lease.list", {}, session())).resolves.toEqual({ - leases: [expect.objectContaining({ id: "lease_1", workerId: "wrk_1" })], - }); - await expect(dispatcher.dispatch("list.get", { kind: "devices" }, session())).resolves.toEqual([ - expect.objectContaining({ id: "dev_1", workerId: "wrk_1" }), - ]); - // why: Cleanup rules are a machine's own configuration, and a gateway runs no reaper. - await expect(dispatcher.dispatch("list.get", { kind: "rules" }, session())).resolves.toEqual( - [], - ); - }); - - it("lease.get reads one lease by its gateway id for its owner or an admin, a worker's own local lease for an admin only, and answers UNKNOWN_LEASE otherwise", async () => { - const { dispatcher, leaseIndex, workers } = harness(); - workers.connected("wrk_1", "mac-mini-1", undefined); - const issued = { - ...leaseFixture("lease_1", "dev_1"), - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - }; - const local = { ...leaseFixture("local_1", "dev_2"), ownerId: "worker-agent" }; - workers.refresh("wrk_1", { leases: [issued, local] }); - const owner = session({ principal: "agent-1", role: "agent" }); - const stranger = session({ principal: "agent-2", role: "agent" }); - const get = (leaseId: string, who: ReturnType) => - dispatcher.dispatch("lease.get", { leaseId }, who); - - await expect(get("wrk_1.lease_1", owner)).resolves.toEqual({ - lease: expect.objectContaining({ id: "wrk_1.lease_1", workerId: "wrk_1" }), - }); - await expect(get("wrk_1.lease_1", session())).resolves.toMatchObject({ - lease: { id: "wrk_1.lease_1" }, - }); - await expect(get("local_1", session())).resolves.toMatchObject({ - lease: { id: "local_1", workerId: "wrk_1" }, - }); - const unknown = { code: "UNKNOWN_LEASE", details: { leaseId: expect.any(String) } }; - await expect(get("wrk_1.lease_1", stranger)).rejects.toMatchObject(unknown); - await expect(get("local_1", stranger)).rejects.toMatchObject(unknown); - await expect(get("wrk_1.lease_nope", session())).rejects.toMatchObject(unknown); - await expect(get("lease_1", session())).rejects.toMatchObject(unknown); - // why: an index entry can outlive its worker view, or the view its lease. - for (const [workerId, workerLeaseId] of [ - ["wrk_gone", "lease_9"], - ["wrk_1", "lease_gone"], - ] as const) { - leaseIndex.add({ - gatewayLeaseId: `${workerId}.${workerLeaseId}`, - grantedAt: 1, - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - workerId, - workerLeaseId, - }); - await expect(get(`${workerId}.${workerLeaseId}`, session())).rejects.toMatchObject(unknown); - await expect(get(`${workerId}.${workerLeaseId}`, owner)).rejects.toMatchObject(unknown); - } - }); - - it("names the worker's own lease id as workerLeaseId on the leases of lease.list, lease.get and list.get kind=leases, never on status.get", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", "mac-mini-1", undefined); - const issued = (id: string, chosen: boolean) => ({ - ...leaseFixture(id, `dev_${id}`), - idChosenByRequester: chosen, - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - }); - workers.refresh("wrk_1", { - leases: [ - issued("lse_42", false), - issued("my-lease", true), - leaseFixture("local_1", "dev_local"), - ], - }); - const expected = [ - { id: "wrk_1.lse_42", workerLeaseId: "lse_42" }, - { id: "my-lease", workerLeaseId: "my-lease" }, - { id: "local_1", workerLeaseId: "local_1" }, - ]; - const pick = (leases: readonly { id: string; workerLeaseId?: string | undefined }[]) => - leases.map(({ id, workerLeaseId }) => ({ id, workerLeaseId })); - - const listed = await dispatcher.dispatch("lease.list", {}, session()); - expect(pick(listed.leases)).toEqual(expected); - const fleet = await dispatcher.dispatch("list.get", { kind: "leases" }, session()); - expect(pick(fleet as never)).toEqual(expected); - const owned = await dispatcher.dispatch( - "lease.list", - {}, - session({ principal: "agent-1", role: "agent" }), - ); - expect(pick(owned.leases)).toEqual(expected.slice(0, 2)); - for (const { id, workerLeaseId } of expected) { - const { lease } = await dispatcher.dispatch("lease.get", { leaseId: id }, session()); - expect(lease).toMatchObject({ id, workerLeaseId }); - } - const status = await dispatcher.dispatch("status.get", {}, session()); - expect((status.leases ?? []).map((lease) => Object.keys(lease))).toEqual( - expected.map(() => expect.not.arrayContaining(["workerLeaseId"])), - ); - }); - - it.each([ - ["an agent", session({ principal: "agent-1", role: "agent" })], - ["an admin", session({ principal: "claude-code:1", role: "admin" })], - ])( - "lease.list lists the lease to %s that asked for it as soon as lease.request answers, before the worker's own lease event has refreshed the view", - async (_who, asker) => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - devices: [deviceFixture("dev_1")], - downloads: { policy: "on-request" }, - health: "running", - leases: [], - queueDepth: 0, - }); - const workerLease = { - ...grantFixture().lease, - ownerId: asker.principal, - requesterId: `${GATEWAY_REQUESTER_PREFIX}${asker.principal}`, - }; - client.requestLeaseQueue.push({ grant: grantFixture({ lease: workerLease }), kind: "grant" }); - // why: the worker lists the lease once the gateway reads its view again; the read its `lease.granted` push starts has not landed. - directory.onRefresh.set("wrk_1", async () => { - await new Promise((resolve) => setImmediate(resolve)); - workers.refresh("wrk_1", { - devices: [deviceFixture("dev_1", "leased")], - leases: [workerLease], - }); - }); - - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", platform: "ios" }, - asker, - ); - const { leases } = await dispatcher.dispatch("lease.list", {}, asker); - - expect((grant as { lease: { id: string } }).lease.id).toBe("wrk_1.lse_1"); - expect(leases.map(({ id, requesterId }) => ({ id, requesterId }))).toEqual([ - { id: "wrk_1.lse_1", requesterId: asker.principal }, - ]); - }, - ); - - it("lists no device or lease for a worker that is starting, whether the session is an admin or the owner of a lease the gateway indexed on it, and answers worker.list with the view as it is", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", "mac-mini-1", undefined); - const lease = { - ...leaseFixture("lease_1", "dev_1"), - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - }; - workers.refresh("wrk_1", { - devices: [deviceFixture("dev_1", "leased")], - health: "running", - leases: [lease], - }); - const owner = session({ principal: "agent-1", role: "agent" }); - await expect(dispatcher.dispatch("lease.list", {}, owner)).resolves.toEqual({ - leases: [expect.objectContaining({ id: "wrk_1.lease_1" })], - }); - - workers.refresh("wrk_1", { health: "starting", host: GATEWAY_HOST }); - - await expect(dispatcher.dispatch("lease.list", {}, session())).resolves.toEqual({ - leases: [], - }); - await expect(dispatcher.dispatch("lease.list", {}, owner)).resolves.toEqual({ leases: [] }); - await expect(dispatcher.dispatch("list.get", { kind: "devices" }, session())).resolves.toEqual( - [], - ); - const { workers: views } = await dispatcher.dispatch("worker.list", {}, session()); - expect(views).toHaveLength(1); - expect(views[0]).toMatchObject({ health: "starting", host: GATEWAY_HOST, id: "wrk_1" }); - expect(Object.keys(views[0] ?? {})).not.toContain("leases"); - }); - - it("inbox.get answers the items the fleet's views, leases, waiting requests and tokens make", async () => { - const holders = async (options: Parameters[0]) => { - const { clock, dispatcher, workers } = harness(options); - workers.connected("wrk_1", "mac-mini-1", "0.3.0"); - workers.refresh("wrk_1", { - devices: [deviceFixture("dev_1", "leased")], - health: "running", - leases: [{ ...leaseFixture("lease_1", "dev_1"), ttlDeadline: 121_000 }], - waiting: [ - { - createdAt: 500, - id: "req_1", - requesterId: "agent-2", - spec: { model: "iPhone 17", platform: "ios" }, - stage: "starting", - }, - ], - }); - workers.connected("wrk_2", "mac-mini-2", "0.3.0"); - workers.disconnected("wrk_2"); - clock.advance(60_000); - const { items } = await dispatcher.dispatch("inbox.get", {}, session()); - return items; - }; - - const named = await holders({ tokenRecords: [{ id: "agent-1", label: "ci-runner" }] }); - - expect(named).toEqual([ - expect.objectContaining({ condition: "disconnected", key: "wrk_2/disconnected" }), - expect.objectContaining({ condition: "waiting", key: "request/req_1/waiting" }), - expect.objectContaining({ - condition: "ending", - key: "lease/lease_1/ending", - lease: expect.objectContaining({ holder: "ci-runner", model: "iPhone 17" }), - }), - ]); - const unnamed = await holders({ withoutTokens: true }); - expect(unnamed).toContainEqual( - expect.objectContaining({ lease: expect.objectContaining({ holder: "agent-1" }) }), - ); - }); - - it("lists each of an owner's gateway-issued leases under its own record, and none whose worker has no view", async () => { - const { dispatcher, leaseIndex, workers } = harness(); - workers.connected("wrk_1", "mac-mini-1", undefined); - const owned = (id: string, deviceId: string) => ({ - ...leaseFixture(id, deviceId), - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - ttlMs: id === "lease_1" ? 111 : 222, - }); - workers.refresh("wrk_1", { leases: [owned("lease_1", "dev_1"), owned("lease_2", "dev_2")] }); - // why: An entry for a worker the gateway holds no view of, as a race with a removal leaves it. - leaseIndex.add({ - gatewayLeaseId: "wrk_gone.lease_9", - grantedAt: 1, - ownerId: "agent-1", - requesterId: "agent-1", - workerId: "wrk_gone", - workerLeaseId: "lease_9", - }); - - const { leases } = await dispatcher.dispatch( - "lease.list", - {}, - session({ principal: "agent-1", role: "agent" }), - ); - - expect(leases.map(({ id, ttlMs }) => ({ id, ttlMs }))).toEqual([ - { id: "wrk_1.lease_1", ttlMs: 111 }, - { id: "wrk_1.lease_2", ttlMs: 222 }, - ]); - }); - - it("shows a non-admin session none of a worker's own local leases", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); - - await expect( - dispatcher.dispatch("lease.list", {}, session({ principal: "someone", role: "agent" })), - ).resolves.toEqual({ leases: [] }); - }); - - it("does not let a gateway session's raw principal match a worker's own local lease owner of the same name", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { leases: [leaseFixture("lease_1", "dev_1")] }); - - await expect( - dispatcher.dispatch("lease.list", {}, session({ principal: "agent-1", role: "agent" })), - ).resolves.toEqual({ leases: [] }); - }); - - it("sees a gateway-issued lease rebuilt from a worker's view, rewritten to its gateway id", async () => { - const { dispatcher, workers } = harness(); - workers.connected("wrk_1", "mac-mini-1", undefined); - const lease = { - ...leaseFixture("lease_1", "dev_1"), - ownerId: "agent-1", - requesterId: `${GATEWAY_REQUESTER_PREFIX}agent-1`, - }; - workers.refresh("wrk_1", { leases: [lease] }); - - await expect( - dispatcher.dispatch("lease.list", {}, session({ principal: "agent-1", role: "agent" })), - ).resolves.toEqual({ - leases: [ - expect.objectContaining({ - id: "wrk_1.lease_1", - requesterId: "agent-1", - worker: { id: "wrk_1", label: "mac-mini-1" }, - }), - ], - }); - }); - - it("status.get: an agent sees only the fleet leases and fleet requests it owns, one that owns none sees empty lists, admin sees all", async () => { - const { coordinator, directory, dispatcher, workers } = harness(); - directory.add("wrk_1", new ScriptedWorkerClient()); - const capacity = statusFixture().capacity; - const issued = (id: string, deviceId: string, ownerId: string) => ({ - ...leaseFixture(id, deviceId), - ownerId, - requesterId: `${GATEWAY_REQUESTER_PREFIX}${ownerId}`, - }); - workers.connected("wrk_1", "mac-mini-1", "0.3.0"); - workers.refresh("wrk_1", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 0 } }, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - health: "running", - // why: lease_3 is the worker's own local lease, owned by a local "agent-1" this gateway never issued to. - leases: [ - issued("lease_1", "dev_1", "agent-1"), - issued("lease_2", "dev_2", "agent-2"), - leaseFixture("lease_3", "dev_3"), - ], - queueDepth: 0, - }); - const wait = (ownerId: string, requesterId: string) => - void coordinator - .request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: false, ownerId, requesterId }, - ) - .catch(() => undefined); - wait("agent-1", "fleet-a"); - wait("agent-3", "fleet-b"); - await expect.poll(() => coordinator.queueDepth).toBe(2); - - const cases = [ - [ - "admin", - { principal: "operator-1", role: "admin" }, - ["wrk_1.lease_1", "wrk_1.lease_2", "lease_3"], - ["fleet-a", "fleet-b"], - ], - [ - "an agent with a lease and a waiting request", - { principal: "agent-1", role: "agent" }, - ["wrk_1.lease_1"], - ["fleet-a"], - ], - [ - "an agent with a lease only", - { principal: "agent-2", role: "agent" }, - ["wrk_1.lease_2"], - [], - ], - [ - "an agent with a waiting request only", - { principal: "agent-3", role: "agent" }, - [], - ["fleet-b"], - ], - ["an agent that owns nothing", { principal: "nobody", role: "agent" }, [], []], - ] as const; - const seen = await Promise.all( - cases.map(async ([name, who]) => { - const status = await dispatcher.dispatch("status.get", {}, session(who)); - return { - leases: status.leases?.map((lease) => lease.id), - name, - waiting: status.waiting?.map((request) => request.requesterId), - }; - }), - ); - - expect(seen).toEqual(cases.map(([name, , leases, waiting]) => ({ leases, name, waiting }))); - - // why: `workers[]` repeats each worker's own leases and waiting requests, so an agent must not read them there. - const held = (status: Awaited>>) => - (status.workers ?? []).flatMap((worker) => [ - ...(worker.leases ?? []).map((lease) => lease.id), - ...(worker.waiting ?? []).map((request) => request.requesterId), - ]); - expect( - held( - await dispatcher.dispatch( - "status.get", - {}, - session({ principal: "operator-1", role: "admin" }), - ), - ), - ).not.toEqual([]); - for (const principal of ["agent-1", "agent-2", "nobody"]) { - expect( - held(await dispatcher.dispatch("status.get", {}, session({ principal, role: "agent" }))), - principal, - ).toEqual([]); - } - }); - - it("replays and subscribes to its own bus, which carries the fleet's events", async () => { - const { dispatcher, eventBus } = harness(); - eventBus.emit( - "lease.granted", - { - deviceId: "dev_1", - leaseId: "l1", - platform: "ios", - requester: "a", - requestId: "req_1", - source: "warm", - }, - "worker", - ); - - await expect(dispatcher.dispatch("events.replay", {}, session())).resolves.toEqual([ - expect.objectContaining({ event: "lease.granted" }), - ]); - await expect(dispatcher.dispatch("events.subscribe", {}, session())).resolves.toEqual({ - subscribed: true, - subscriptionId: "sub_1", - }); - }); - - it("events.replay with sinceTs returns what the event history returns", async () => { - const fromHistory: EventEnvelope[] = [ - { - id: "evt_7", - seq: 7, - timestamp: 50, - event: "daemon.stopping", - payload: { reason: "x" }, - module: "d", - }, - ]; - const asked: unknown[] = []; - const { dispatcher } = harness({ - eventHistory: { - replay: async (input) => { - asked.push(input); - return fromHistory; - }, - replayPage: async () => ({ events: [], kind: "events" }), - status: () => ({ opened: true, writing: true }), - }, - }); - - await expect(dispatcher.dispatch("events.replay", { sinceTs: 10 }, session())).resolves.toEqual( - fromHistory, - ); - expect(asked).toEqual([{ sinceTs: 10 }]); - }); - - it("events.replay with beforeId returns a worker's relayed event with payload.workerId", async () => { - const rig = await usageRig({ fleet: true }); - rig.relay("w-1", "daemon.stopping", { reason: "from-worker" }); - rig.emit("daemon.stopping", { reason: "local" }); - const newest = rig.bus.replay().at(-1) as { id: string }; - const { dispatcher } = harness({ eventHistory: rig.history }); - - const page = await dispatcher.dispatch( - "events.replay", - { beforeId: newest.id, limit: 5 }, - session(), - ); - - expect(page).toEqual([ - expect.objectContaining({ payload: expect.objectContaining({ workerId: "w-1" }) }), - ]); - }); - - it("events.replay with limit alone returns the newest n events, oldest first, through the gateway dispatcher", async () => { - const rig = await usageRig({ fleet: true }); - for (const reason of ["a", "b", "c"]) rig.emit("daemon.stopping", { reason }); - const { dispatcher } = harness({ eventHistory: rig.history }); - - const page = await dispatcher.dispatch("events.replay", { limit: 2 }, session()); - - expect(page.map((event) => (event.payload as { reason: string }).reason)).toEqual(["b", "c"]); - }); - - it("events.replay with beforeId and no limit returns at most 100 events through the gateway dispatcher", async () => { - const rig = await usageRig({ fleet: true }); - for (let n = 0; n < 130; n += 1) rig.emit("daemon.stopping", { reason: `r${n}` }); - const newest = rig.bus.replay().at(-1) as { id: string }; - const { dispatcher } = harness({ eventHistory: rig.history }); - - const page = await dispatcher.dispatch("events.replay", { beforeId: newest.id }, session()); - - expect(page).toHaveLength(100); - }); - - it.each([ - { - health: "running" as const, - history: { opened: true, setAsideAt: 1_760_605_920_000, writing: false }, - }, - { health: "starting" as const, history: { opened: false, writing: false } }, - { health: "running" as const, history: { opened: true, writing: true } }, - { - health: "running" as const, - history: { opened: true, rebuiltAt: 1_760_605_920_000, writing: true }, - }, - { - health: "running" as const, - history: { opened: true, rebuildFailed: true as const, writing: true }, - }, - ])( - "status.get carries the gateway's own history opened, writing, setAsideAt, rebuiltAt and rebuildFailed, in the starting answer too ($health, $history)", - async ({ health, history }) => { - const { dispatcher } = harness({ eventHistory: fixedEventHistory(history), health }); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - expect((status as { history?: unknown }).history).toStrictEqual(history); - }, - ); - - it("mints its own tokens, worker join tokens included", async () => { - const { dispatcher, tokens } = harness(); - - await expect( - dispatcher.dispatch("token.create", { role: "worker", label: "mac-mini-1" }, session()), - ).resolves.toMatchObject({ secret: "slk_secret", token: { role: "worker" } }); - expect(tokens.created).toEqual(["worker"]); - }); - - it("closes the uplink a revoked token authorized, not merely the store entry (C-2)", async () => { - const { closedUplinkTokens, dispatcher } = harness(); - - await expect(dispatcher.dispatch("token.revoke", { id: "tok_1" }, session())).resolves.toEqual({ - revoked: true, - }); - - expect(closedUplinkTokens).toEqual(["tok_1"]); - }); - - it("does not try to close an uplink for a token id that was never real (C-2)", async () => { - const { closedUplinkTokens, dispatcher } = harness(); - - await expect( - dispatcher.dispatch("token.revoke", { id: "tok_unknown" }, session()), - ).resolves.toEqual({ revoked: false }); - - expect(closedUplinkTokens).toEqual([]); - }); - - describe("alerts.test", () => { - const generic: AlertDestination = { - name: "ops", - format: "generic", - url: "https://hooks.example.com/ops-token", - events: ["worker.connected"], - secret: "s3cr3t", - }; - const slack: AlertDestination = { - name: "chat", - format: "slack", - url: "https://hooks.slack.example.com/chat-token", - events: ["worker.connected"], - }; - - it("alerts.test posts one sample to the named destination and answers its status", async () => { - const { dispatcher, poster } = harness({ alertDestinations: [generic, slack] }); - - await expect( - dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), - ).resolves.toEqual({ destination: "ops", status: 200 }); - await expect( - dispatcher.dispatch("alerts.test", { destination: "chat" }, session()), - ).resolves.toEqual({ destination: "chat", status: 200 }); - - expect(poster.requests).toHaveLength(2); - const [first, second] = poster.requests as [SentRequest, SentRequest]; - expect(first.url).toBe("https://hooks.example.com/ops-token"); - expect(JSON.parse(first.body)).toEqual({ - id: "evt_sample-id", - seq: 0, - timestamp: 1_000, - event: "alerts.test-sent", - payload: { destination: "ops" }, - module: "alerts", - }); - expect(first.headers).toEqual({ - "Content-Type": "application/json", - "User-Agent": "simlock/9.9.9", - "X-Simlock-Event": "alerts.test-sent", - "X-Simlock-Delivery": "evt_sample-id", - "X-Simlock-Signature": `sha256=${createHmac("sha256", "s3cr3t") - .update(first.body) - .digest("hex")}`, - }); - expect(second.url).toBe("https://hooks.slack.example.com/chat-token"); - expect(JSON.parse(second.body)).toEqual({ - text: "🔔 Test message from the simlock gateway to destination chat.", - }); - expect(second.headers).toEqual({ - "Content-Type": "application/json", - "User-Agent": "simlock/9.9.9", - }); - - const other = harness({ alertDestinations: [generic] }); - other.poster.respond = () => Promise.resolve({ status: 204 }); - await expect( - other.dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), - ).resolves.toEqual({ destination: "ops", status: 204 }); - }); - - it("alerts.test fails with ALERT_SEND_FAILED on a non-2xx status, a rejection and no answer within 10 s, and tries once", async () => { - const refused = Object.assign(new TypeError("fetch failed"), { - cause: { code: "ECONNREFUSED" }, - }); - const cases = [ - [() => Promise.resolve({ status: 404 }), "ops answered HTTP 404", { status: 404 }], - [() => Promise.resolve({ status: 199 }), "ops answered HTTP 199", { status: 199 }], - [() => Promise.resolve({ status: 301 }), "ops answered HTTP 301", { status: 301 }], - [() => Promise.resolve({ status: 503 }), "ops answered HTTP 503", { status: 503 }], - [ - () => Promise.reject(refused), - "ops could not be reached: ECONNREFUSED", - { error: "ECONNREFUSED" }, - ], - [ - () => Promise.reject(new TypeError("fetch failed")), - "ops could not be reached: network-error", - { error: "network-error" }, - ], - ] as const; - for (const [respond, message, details] of cases) { - const { clock, dispatcher, poster } = harness({ alertDestinations: [generic] }); - poster.respond = respond; - - await expect( - dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), - ).rejects.toMatchObject({ - code: "ALERT_SEND_FAILED", - message, - details: { destination: "ops", ...details }, - }); - clock.advance(60_000); - expect(poster.requests).toHaveLength(1); - } - - const { clock, dispatcher, poster } = harness({ alertDestinations: [generic] }); - poster.respond = () => new Promise(() => {}); - let outcome: unknown = "pending"; - void dispatcher.dispatch("alerts.test", { destination: "ops" }, session()).then( - (value) => (outcome = value), - (error: unknown) => (outcome = error), - ); - await vi.waitFor(() => expect(poster.requests).toHaveLength(1)); - clock.advance(9_999); - await new Promise((resolve) => setImmediate(resolve)); - expect(outcome).toBe("pending"); - expect(poster.requests[0]?.signal.aborted).toBe(false); - - clock.advance(1); - await new Promise((resolve) => setImmediate(resolve)); - expect(outcome).toMatchObject({ - code: "ALERT_SEND_FAILED", - message: "ops did not answer within 10 s", - details: { destination: "ops", error: "timeout" }, - }); - expect(poster.requests[0]?.signal.aborted).toBe(true); - clock.advance(60_000); - expect(poster.requests).toHaveLength(1); - }); - - it("alerts.test answers UNKNOWN_ALERT_DESTINATION for a name not configured and on a gateway with no destinations", async () => { - const unknown = { - code: "UNKNOWN_ALERT_DESTINATION", - message: - 'No alert destination named "nope". Check gateway.alerts.destinations in the gateway\'s config.', - details: { destination: "nope" }, - }; - const configured = harness({ alertDestinations: [generic] }); - await expect( - configured.dispatcher.dispatch("alerts.test", { destination: "ops" }, session()), - ).resolves.toEqual({ destination: "ops", status: 200 }); - - await expect( - configured.dispatcher.dispatch("alerts.test", { destination: "nope" }, session()), - ).rejects.toMatchObject(unknown); - await expect( - harness().dispatcher.dispatch("alerts.test", { destination: "nope" }, session()), - ).rejects.toMatchObject(unknown); - await expect( - configured.dispatcher.dispatch("alerts.test", { destination: "o" }, session()), - ).rejects.toMatchObject({ code: "UNKNOWN_ALERT_DESTINATION", details: { destination: "o" } }); - - expect(configured.poster.requests).toHaveLength(1); - }); - - it("a refused connection answers could not be reached with the network code, and neither the answer nor the failed call's operation log line holds the URL", async () => { - const closedPort = await new Promise((resolve) => { - const server = createServer(); - server.listen(0, "127.0.0.1", () => { - const { port } = server.address() as { port: number }; - server.close(() => resolve(port)); - }); - }); - const url = `http://127.0.0.1:${closedPort}/hook-token-xyz`; - const { dispatcher, logSink } = harness({ - alertDestinations: [{ ...generic, url }], - webhookPoster: new FetchWebhookPoster(), - }); - - const failure = await dispatcher - .dispatch("alerts.test", { destination: "ops" }, session()) - .then( - () => undefined, - (error: unknown) => error as { code: string; message: string; details: unknown }, - ); - - expect(failure).toMatchObject({ - code: "ALERT_SEND_FAILED", - message: "ops could not be reached: ECONNREFUSED", - details: { destination: "ops", error: "ECONNREFUSED" }, - }); - expect( - logSink.records - .filter((record) => record.message === "operation") - .map((record) => record.fields), - ).toMatchObject([ - { - operation: "alerts.test", - code: "ALERT_SEND_FAILED", - message: "ops could not be reached: ECONNREFUSED", - }, - ]); - for (const text of [JSON.stringify(failure), JSON.stringify(logSink.records)]) { - expect(text).not.toContain("hook-token-xyz"); - expect(text).not.toContain(String(closedPort)); - } - }); - - it("alerts.test answers BAD_REQUEST for a destination of no characters or more than 32, and looks up one of 32", async () => { - const { dispatcher, poster } = harness({ alertDestinations: [generic] }); - const longest = "n".repeat(32); - - for (const destination of ["", "n".repeat(33)]) { - await expect( - dispatcher.dispatch("alerts.test", { destination }, session()), - ).rejects.toMatchObject({ code: "BAD_REQUEST" }); - } - await expect( - dispatcher.dispatch("alerts.test", { destination: longest }, session()), - ).rejects.toMatchObject({ code: "UNKNOWN_ALERT_DESTINATION" }); - expect(poster.requests).toEqual([]); - }); - - it("stop during a test answers DAEMON_STOPPING", async () => { - const { alerts, dispatcher, poster } = harness({ alertDestinations: [generic] }); - poster.respond = () => new Promise(() => {}); - let outcome: unknown = "pending"; - void dispatcher.dispatch("alerts.test", { destination: "ops" }, session()).then( - (value) => (outcome = value), - (error: unknown) => (outcome = error), - ); - await vi.waitFor(() => expect(poster.requests).toHaveLength(1)); - expect(outcome).toBe("pending"); - - alerts?.stop(); - await new Promise((resolve) => setImmediate(resolve)); - - expect(outcome).toMatchObject({ code: "DAEMON_STOPPING" }); - }); - }); - - describe("worker.install-component (ADR 0010 §7)", () => { - function withWorker() { - const fixture = harness(); - const worker = new ScriptedWorkerClient(); - fixture.workers.connected("wrk_1", undefined, "0.3.0"); - fixture.workers.refresh("wrk_1", { - downloads: { policy: "on-request", timeoutMs: 60_000 }, - }); - fixture.directory.add("wrk_1", worker); - return { ...fixture, worker }; - } - - it("refuses an agent session with FORBIDDEN and asks no worker", async () => { - const { dispatcher, worker } = withWorker(); - - await expect( - dispatcher.dispatch( - "worker.install-component", - { platform: "android", version: "35", workers: "all" }, - session({ role: "agent" }), - ), - ).rejects.toMatchObject({ code: "FORBIDDEN" }); - expect(worker.calls.filter((call) => call.startsWith("component.install"))).toEqual([]); - }); - - it("opens the session's stream before asking a worker, and hands it each update with the worker's id", async () => { - const { dispatcher, worker } = withWorker(); - const seen: unknown[] = []; - worker.installComponentHandler = async (input, options) => { - seen.push("asked"); - options.onProgress?.({ stage: "waiting" }); - return { - component: input.version, - outcome: "installed", - platform: input.platform, - version: input.version, - }; - }; - - const result = await dispatcher.dispatch( - "worker.install-component", - { platform: "android", version: "35", workers: ["wrk_1"] }, - session({ - onComponentProgress: (progress, workerId) => seen.push({ progress, workerId }), - onStarted: () => seen.push("started"), - }), - ); - - expect(seen).toEqual([ - "started", - "asked", - { progress: { stage: "waiting" }, workerId: "wrk_1" }, - ]); - expect(result).toEqual({ - results: [{ outcome: "installed", version: "35", workerId: "wrk_1" }], - }); - }); - }); - - describe("UNSUPPORTED_IN_GATEWAY_MODE", () => { - it.each([ - "nuke.run", - "cleanup.run", - "doctor.run", - "driver.passthrough", - "warm.forget", - ] as const)("%s stays per-worker, permanently", async (operation) => { - const { dispatcher } = harness(); - const input = - operation === "driver.passthrough" - ? { args: ["devices"], tool: "adb" } - : operation === "warm.forget" - ? { model: "Pixel", platform: "android" } - : { fix: true }; - - await expect(dispatcher.dispatch(operation, input, session())).rejects.toMatchObject({ - code: "UNSUPPORTED_IN_GATEWAY_MODE", - details: { operation }, - }); - }); - - it.each([ - [ - "component.install", - { platform: "android", version: "35" }, - "admin", - "component.install installs on one machine; on a gateway, name the workers to install on " + - "(simlock component install --worker or --all-workers)", - ], - [ - "component.list", - {}, - "agent", - "component.list lists one machine's components; run it against a worker", - ], - [ - "component.remove", - { platform: "ios", version: "26.4" }, - "admin", - "component.remove removes a component from one machine; run it against a worker", - ], - [ - "device.add", - { id: "00008110-001A2B3C4D5E", platform: "ios" }, - "admin", - "device.add enrols a device plugged into one machine; run it on the worker the device is plugged into", - ], - [ - "device.remove", - { id: "00008110-001A2B3C4D5E" }, - "admin", - "device.remove unenrols a device plugged into one machine; run it on the worker the device is plugged into", - ], - ] as const)( - "%s answers UNSUPPORTED_IN_GATEWAY_MODE with a message pointing at the worker", - async (operation, input, role, message) => { - const { dispatcher } = harness(); - - await expect( - dispatcher.dispatch(operation, input, session({ role })), - ).rejects.toMatchObject({ - code: "UNSUPPORTED_IN_GATEWAY_MODE", - details: { operation }, - message, - }); - }, - ); - }); - - describe("daemon.update", () => { - it("asks the gateway's own update coordinator and answers with its outcome, the gateway's version and its health, and status.get carries the update and its last failure", async () => { - const { dispatcher, update } = harness(); - const run = { - forced: true, - from: "0.0.0-test", - stage: "installing", - startedAt: 1_000, - to: "0.4.0", - } as const; - const failure = { - at: 500, - from: "0.0.0-test", - message: "npm failed", - stage: "installing", - to: "0.3.0", - } as const; - update.progress = { lastUpdateFailure: failure, latestVersion: "0.4.0", update: run }; - - await expect( - dispatcher.dispatch("daemon.update", { force: true }, session({ role: "admin" })), - ).resolves.toEqual({ - health: "running", - lastUpdateFailure: failure, - latestVersion: "0.4.0", - resolved: "0.4.0", - started: true, - update: run, - upToDate: false, - version: "0.0.0-test", - }); - expect(update.requests).toEqual([{ force: true }]); - - const status = await dispatcher.dispatch("status.get", {}, session({ role: "admin" })); - expect(status.daemon).toMatchObject({ lastUpdateFailure: failure, update: run }); - }); - }); - - describe("a lease request naming a baseline (#1208)", () => { - const CATALOG = catalogFixture([ - { - modelClasses: { "Pixel 9": "phone" } as const, - models: ["Pixel 9"], - platform: "android", - runtimes: ["36"], - }, - { - modelClasses: { "iPhone 17": "phone" } as const, - models: ["iPhone 17"], - platform: "ios", - runtimes: ["27.0"], - }, - ]).platforms; - const checkout: BaselineEntry = { - mode: "full", - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android", - savedAt: "2026-11-02T13:10:00Z", - version: 2, - }; - const smoke: BaselineEntry = { - ...checkout, - mode: "slim", - model: "iPhone 17", - name: "smoke", - osVersion: "27.0", - platform: "ios", - version: 1, - }; - const stale = (entry: BaselineEntry) => ({ - ...entry, - staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - }); - - function fleet(baselinesByWorker: Record) { - const rig = harness(); - const clients: Record = {}; - for (const [workerId, baselines] of Object.entries(baselinesByWorker)) { - const client = new ScriptedWorkerClient(); - clients[workerId] = client; - rig.directory.add(workerId, client); - rig.workers.connected(workerId, undefined, "0.3.0"); - rig.workers.refresh(workerId, { - baselines: [...baselines], - capacity: statusFixture().capacity, - catalog: CATALOG, - devices: [], - downloads: { policy: "on-request" }, - health: "running", - leases: [], - queueDepth: 0, - }); - } - return { ...rig, clients }; - } - - type Rejected = { readonly code: string; readonly message: string }; - const cases: readonly [ - string, - Record, - Record, - { readonly grantedBy: string } | { readonly rejected: Rejected }, - string[]?, - ][] = [ - [ - "one worker holds it, another has none", - { wrk_a: [], wrk_b: [checkout] }, - { baseline: "checkout", platform: "android" }, - { grantedBy: "wrk_b" }, - ], - [ - "an OS range that holds the baseline's OS", - { wrk_a: [], wrk_b: [checkout] }, - { baseline: "checkout", osVersion: ">=35", platform: "android" }, - { grantedBy: "wrk_b" }, - ], - [ - "no worker holds a baseline of the name", - { wrk_a: [checkout], wrk_b: [] }, - { baseline: "nothere", platform: "android" }, - { - rejected: { - code: "UNKNOWN_BASELINE", - message: 'no worker holds a baseline named "nothere"', - }, - }, - ], - [ - "every holder differs from the request: the first holder's message", - { wrk_a: [checkout], wrk_b: [{ ...checkout, osVersion: "35" }] }, - { baseline: "checkout", osVersion: "34", platform: "android" }, - { - rejected: { - code: "BASELINE_MISMATCH", - message: "baseline checkout is Android 36, the request asks for 34", - }, - }, - ], - [ - "every holder is stale: the first holder's reason", - { wrk_a: [stale(smoke)], wrk_b: [{ ...smoke, staleReason: "image is gone" }] }, - { baseline: "smoke", platform: "ios" }, - { - rejected: { - code: "BASELINE_STALE", - message: - "baseline smoke is stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - }, - }, - ], - [ - "every holder is stale and differs: the mismatch is checked first", - { wrk_a: [stale(smoke)] }, - { baseline: "smoke", mode: "full", platform: "ios" }, - { - rejected: { - code: "BASELINE_MISMATCH", - message: "baseline smoke is slim, the request asks for full", - }, - }, - ], - [ - "the only holder is disconnected", - { wrk_a: [checkout], wrk_b: [] }, - { baseline: "checkout", platform: "android" }, - { - rejected: { - code: "NO_CAPACITY", - message: "No worker in the fleet can currently serve this request", - }, - }, - ["wrk_a"], - ], - [ - "the only fresh holder is disconnected and the other holder is stale", - { wrk_a: [checkout], wrk_b: [stale(checkout)] }, - { baseline: "checkout", platform: "android" }, - { - rejected: { - code: "NO_CAPACITY", - message: "No worker in the fleet can currently serve this request", - }, - }, - ["wrk_a"], - ], - ]; - - it.each(cases)( - "sends a request naming a baseline only to a worker that holds it, fresh: %s", - async (_label, baselines, ask, outcome, disconnected = []) => { - const { clients, dispatcher, workers } = fleet(baselines); - for (const workerId of disconnected) workers.disconnected(workerId); - for (const client of Object.values(clients)) { - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - } - - // why: a rejection must come at once, so the request does not opt out of waiting and the answer is read after the queue is flushed; a request that waited instead would have none. - const answer = dispatcher.dispatch( - "lease.request", - ("rejected" in outcome ? ask : { ...ask, noWait: true }) as never, - session({ role: "agent" }), - ); - - if ("rejected" in outcome) { - let settled: unknown; - void answer.catch((error: unknown) => { - settled = error; - }); - await new Promise((resolve) => setImmediate(resolve)); - expect(settled).toMatchObject(outcome.rejected); - for (const client of Object.values(clients)) { - expect(client.lastRequestLeaseInput).toBeUndefined(); - } - return; - } - const grant = await answer; - expect(grant.lease.id).toMatch(new RegExp(`^${outcome.grantedBy}\\.`)); - for (const [workerId, client] of Object.entries(clients)) { - if (workerId === outcome.grantedBy) { - expect(client.lastRequestLeaseInput).toMatchObject({ baseline: ask["baseline"] }); - } else { - expect(client.lastRequestLeaseInput).toBeUndefined(); - } - } - }, - ); - - it.each([ - ["UNKNOWN_BASELINE", "no-baseline"], - ["BASELINE_MISMATCH", "baseline-mismatch"], - ["BASELINE_STALE", "baseline-stale"], - ] as const)( - "leaves out a holder that answers %s, asks the next holder, ends with that answer when none is left, and names %s for it while the request waits", - async (code, reason) => { - const refusal = () => - new SimlockError(code, "domain", `${code} from the worker`, {} as never); - const ask = { baseline: "checkout", noWait: true, platform: "android" } as const; - const next = fleet({ wrk_a: [checkout], wrk_b: [checkout] }); - next.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); - next.clients["wrk_b"]?.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await next.dispatcher.dispatch( - "lease.request", - ask, - session({ role: "agent" }), - ); - - expect(grant.lease.id).toMatch(/^wrk_b\./); - expect(next.directory.catalogRefreshCalls).toContain("wrk_a"); - - const alone = fleet({ wrk_a: [checkout] }); - alone.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); - - await expect( - alone.dispatcher.dispatch("lease.request", ask, session({ role: "agent" })), - ).rejects.toMatchObject({ code, message: `${code} from the worker` }); - - const busy = fleet({ wrk_a: [checkout], wrk_b: [checkout] }); - busy.workers.refresh("wrk_b", { queueDepth: 1 }); - busy.clients["wrk_a"]?.requestLeaseQueue.push({ error: refusal(), kind: "error" }); - void busy.dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform: "android" }, - session({ role: "agent" }), - ); - await vi.waitFor(async () => { - const status = await busy.dispatcher.dispatch("status.get", {}, session()); - expect(status.waiting?.[0]?.workers).toContainEqual({ reason, workerId: "wrk_a" }); - }); - }, - ); - }); - - describe("a request naming a baseline that waits (#1208)", () => { - it("waits while its only holder is busy, and status.get names why each other worker is passed over", async () => { - const entry: BaselineEntry = { - mode: "full", - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android", - savedAt: "2026-11-02T13:10:00Z", - version: 2, - }; - const held: Record = { - wrk_a: { baselines: [entry], queueDepth: 1 }, - wrk_b: { baselines: [], queueDepth: 0 }, - wrk_c: { baselines: [{ ...entry, staleReason: "image is gone" }], queueDepth: 0 }, - wrk_d: { baselines: [{ ...entry, mode: "slim" }], queueDepth: 0 }, - }; - const { directory, dispatcher, workers } = harness(); - for (const [workerId, view] of Object.entries(held)) { - directory.add(workerId, new ScriptedWorkerClient()); - workers.connected(workerId, undefined, "0.3.0"); - workers.refresh(workerId, { - ...view, - capacity: statusFixture().capacity, - catalog: catalogFixture([ - { - modelClasses: { "Pixel 9": "phone" } as const, - models: ["Pixel 9"], - platform: "android", - runtimes: ["36"], - }, - ]).platforms, - devices: [], - downloads: { policy: "on-request" }, - health: "running", - leases: [], - }); - } - - void dispatcher.dispatch( - "lease.request", - { baseline: "checkout", mode: "full", platform: "android" }, - session({ role: "agent" }), - ); - await vi.waitFor(async () => { - const status = await dispatcher.dispatch("status.get", {}, session()); - expect(status.waiting?.[0]?.workers).toEqual([ - { reason: "busy", workerId: "wrk_a" }, - { reason: "no-baseline", workerId: "wrk_b" }, - { reason: "baseline-stale", workerId: "wrk_c" }, - { reason: "baseline-mismatch", workerId: "wrk_d" }, - ]); - }); - }); - }); - - describe("a request naming a baseline held by a starting worker (#1208)", () => { - it("keeps the starting worker's last baselines for routing, so the request waits instead of failing, and shows none in its view", async () => { - const { directory, dispatcher, workers } = harness(); - directory.add("wrk_a", new ScriptedWorkerClient()); - workers.connected("wrk_a", undefined, "0.3.0"); - workers.refresh("wrk_a", { - baselines: [ - { - mode: "full", - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android", - savedAt: "2026-11-02T13:10:00Z", - version: 1, - }, - ], - capacity: statusFixture().capacity, - catalog: catalogFixture([ - { - modelClasses: { "Pixel 9": "phone" } as const, - models: ["Pixel 9"], - platform: "android", - runtimes: ["36"], - }, - ]).platforms, - devices: [], - health: "running", - leases: [], - queueDepth: 0, - }); - workers.refresh("wrk_a", { health: "starting" }); - - void dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform: "android" }, - session({ role: "agent" }), - ); - - await vi.waitFor(async () => { - const status = await dispatcher.dispatch("status.get", {}, session()); - expect(status.waiting).toHaveLength(1); - }); - expect(workers.view("wrk_a")).not.toHaveProperty("baselines"); - }); - }); - - describe("baselines through a gateway (#1208)", () => { - const entry = (name: string, overrides: Partial = {}): BaselineEntry => ({ - mode: "full", - model: "Pixel 9", - name, - osVersion: "36", - platform: "android", - savedAt: "2026-11-02T13:10:00Z", - version: 1, - ...overrides, - }); - - function workersWith(baselinesByWorker: Record) { - const rig = harness(); - const clients: Record = {}; - for (const [workerId, baselines] of Object.entries(baselinesByWorker)) { - const client = new ScriptedWorkerClient(); - client.baselines = baselines; - clients[workerId] = client; - rig.directory.add(workerId, client); - rig.workers.connected(workerId, undefined, "0.3.0"); - rig.workers.refresh(workerId, { - baselines, - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - }); - } - return { ...rig, clients }; - } - - it("saves a baseline on the worker that holds the lease, with the worker's lease id, and drops the lease from its index, or keeps it when the worker has no live link", async () => { - const { clients, directory, dispatcher, leaseIndex, workers } = workersWith({ - wrk_a: [], - wrk_b: [], - }); - workers.refresh("wrk_a", { - leases: [ - { - ...leaseFixture("L7", "dev_1"), - ownerId: "alice-principal", - requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, - }, - ], - }); - - const saved = await dispatcher.dispatch( - "baseline.save", - { leaseId: "wrk_a.L7", name: "checkout", note: "app 4.3" }, - session(), - ); - - expect(clients["wrk_a"]?.savedBaselines).toEqual([ - { leaseId: "L7", name: "checkout", note: "app 4.3" }, - ]); - expect(clients["wrk_b"]?.savedBaselines).toEqual([]); - expect(saved).toMatchObject({ baseline: { name: "checkout" }, leaseId: "wrk_a.L7" }); - expect(leaseIndex.resolve("wrk_a.L7")).toBeUndefined(); - expect(directory.catalogRefreshCalls).toEqual(["wrk_a"]); - - // why: a worker that no longer knows the lease drops the index entry; one with no live link answers WORKER_UNREACHABLE and keeps it. - const gatewayLease = (id: string, deviceId: string) => ({ - ...leaseFixture(id, deviceId), - ownerId: "alice-principal", - requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, - }); - workers.refresh("wrk_a", { leases: [gatewayLease("L8", "dev_2")] }); - workers.refresh("wrk_b", { leases: [gatewayLease("L9", "dev_3")] }); - expect(leaseIndex.resolve("wrk_a.L8")).toBeDefined(); - const unknown = new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { - leaseId: "L8", - }); - if (clients["wrk_a"] !== undefined) clients["wrk_a"].failWith = unknown; - await expect( - dispatcher.dispatch("baseline.save", { leaseId: "wrk_a.L8", name: "x" }, session()), - ).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); - expect(leaseIndex.resolve("wrk_a.L8")).toBeUndefined(); - - directory.clients.delete("wrk_b"); - await expect( - dispatcher.dispatch("baseline.save", { leaseId: "wrk_b.L9", name: "x" }, session()), - ).rejects.toMatchObject({ code: "WORKER_UNREACHABLE" }); - expect(leaseIndex.resolve("wrk_b.L9")).toBeDefined(); - }); - - it("lists every worker's baselines, each with its workerId, in worker id order, a disconnected worker's last ones too", async () => { - const { dispatcher, workers } = workersWith({ - wrk_b: [entry("smoke", { platform: "ios" })], - wrk_a: [entry("checkout", { version: 2 }), entry("login")], - wrk_c: [], - }); - workers.disconnected("wrk_b"); - - const listed = await dispatcher.dispatch("baseline.list", {}, session({ role: "agent" })); - - expect(listed.baselines.map(({ name, workerId }) => [workerId, name])).toEqual([ - ["wrk_a", "checkout"], - ["wrk_a", "login"], - ["wrk_b", "smoke"], - ]); - }); - - it("removes a baseline from the worker it names without sending the workerId, and refuses an unknown, a disconnected or no worker, and relays the worker's own refusals", async () => { - const { clients, directory, dispatcher, workers } = workersWith({ - wrk_a: [entry("smoke")], - wrk_b: [entry("smoke")], - }); - - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), - ).resolves.toMatchObject({ name: "smoke" }); - expect(clients["wrk_a"]?.removedBaselines).toEqual([{ name: "smoke" }]); - expect(directory.catalogRefreshCalls).toEqual(["wrk_a"]); - - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_x" }, session()), - ).rejects.toMatchObject({ code: "UNKNOWN_WORKER", message: "Unknown worker: wrk_x" }); - workers.disconnected("wrk_b"); - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_b" }, session()), - ).rejects.toMatchObject({ - code: "WORKER_NOT_CONNECTED", - message: "worker wrk_b is not connected", - }); - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke" }, session()), - ).rejects.toMatchObject({ - code: "BAD_REQUEST", - message: - "baseline.remove on a gateway needs the worker that holds the baseline: pass --worker ", - }); - expect(clients["wrk_a"]?.removedBaselines).toHaveLength(1); - expect(clients["wrk_b"]?.removedBaselines).toEqual([]); - - // why: a worker's own refusal reaches the caller with its code and words. - for (const failure of [ - new SimlockError("BASELINE_IN_USE", "domain", "worker's words", { - leaseId: "L1", - name: "smoke", - }), - new SimlockError("UNKNOWN_BASELINE", "domain", "worker's words", { name: "smoke" }), - ]) { - const client = clients["wrk_a"]; - if (client !== undefined) client.failWith = failure; - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), - ).rejects.toMatchObject({ code: failure.code, message: "worker's words" }); - } - - // why: an uplink that drops under the call is the gateway's WORKER_NOT_CONNECTED, not the client's transport code. - const client = clients["wrk_a"]; - if (client !== undefined) { - client.failWith = new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - "Daemon connection closed", - {}, - ); - } - await expect( - dispatcher.dispatch("baseline.remove", { name: "smoke", workerId: "wrk_a" }, session()), - ).rejects.toMatchObject({ - code: "WORKER_NOT_CONNECTED", - message: "worker wrk_a is not connected", - }); - }); - }); - - describe("lease lifecycle and device.exec (#118)", () => { - it("forwards lease.request through the fleet coordinator and answers UNKNOWN_LEASE for renew/release/exec against an id it never issued", async () => { - const { clock, dispatcher } = harness(); - - await expect( - dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", noWait: true, platform: "ios" }, - session({ role: "agent" }), - ), - ).rejects.toMatchObject({ code: "NO_CAPACITY" }); - - const ghostCalls = [ - ["lease.renew", { leaseId: "wrk_ghost.lease_1" }], - ["lease.release", { leaseId: "wrk_ghost.lease_1" }], - ["device.exec", { args: [], leaseId: "wrk_ghost.lease_1", tool: "adb" }], - ] as const; - // why: inside the gateway's first minute a generated id may belong to a worker yet to report, which answers WORKER_UNREACHABLE. - clock.advance(59_999); - for (const [operation, input] of ghostCalls) { - await expect( - dispatcher.dispatch(operation, input, session({ role: "agent" })), - ).rejects.toMatchObject({ code: "WORKER_UNREACHABLE" }); - } - clock.advance(1); - for (const [operation, input] of ghostCalls) { - await expect( - dispatcher.dispatch(operation, input, session({ role: "agent" })), - ).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); - } - }); - - it("refuses a lease.request that names no requesterId when the session principal has a control character", async () => { - const { dispatcher } = harness(); - - await expect( - dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", noWait: true, platform: "ios" }, - session({ principal: "a\u001b[2Jb", role: "agent" }), - ), - ).rejects.toMatchObject({ code: "BAD_REQUEST" }); - }); - - it.each([ - ["with noWait: true", { noWait: true }], - ["without noWait", {}], - ])( - "refuses a lease.request carrying fleetRequestId with FORBIDDEN from every session, even the uplink, %s", - async (_label, extra) => { - const { dispatcher } = harness(); - const input = { fleetRequestId: "req_gw1", model: "iPhone 17", platform: "ios", ...extra }; - - for (const refused of [ - session({ role: "agent" }), - session({ role: "admin" }), - session({ isGatewayUplink: true, role: "admin" }), - ]) { - await expect(dispatcher.dispatch("lease.request", input, refused)).rejects.toMatchObject({ - code: "FORBIDDEN", - message: expect.stringContaining("fleetRequestId"), - }); - } - }, - ); - - it("answers not-found for lease.cancel and an empty result for lease.release-all with nothing queued or leased", async () => { - const { dispatcher } = harness(); - - await expect( - dispatcher.dispatch("lease.cancel", {}, session({ role: "agent" })), - ).resolves.toEqual({ result: "not-found" }); - await expect(dispatcher.dispatch("lease.release-all", {}, session())).resolves.toEqual({ - leaseIds: [], - }); - }); - - // why: ADR §27a's whole point: the owner a rebuilt lease authorizes against has to be the real - // principal the worker round-tripped back, not this gateway's own uplink principal -- and it - // has to work on an index built purely by `rebuildFromWorker` (a reconnect), never through - // `coordinator.request` in this process. This is the one place a naive "derive ownerId from - // the connection" implementation and a correct one visibly disagree. - it("authorizes lease.renew for the original requester and forbids everyone else, on a lease rebuilt after reconnect", async () => { - const { dispatcher, directory, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, undefined); - workers.refresh("wrk_1", { - leases: [ - { - ...leaseFixture("lease_1", "dev_1"), - ownerId: "alice-principal", - requesterId: `${GATEWAY_REQUESTER_PREFIX}alice`, - }, - ], - }); - const gatewayLeaseId = "wrk_1.lease_1"; - client.renewLeaseQueue.push({ - kind: "ok", - record: { - ...leaseFixture("lease_1", "dev_1"), - ownerId: "alice-principal", - ttlMs: 1_800_000, - }, - }); - - await expect( - dispatcher.dispatch( - "lease.renew", - { leaseId: gatewayLeaseId }, - session({ principal: "alice-principal", role: "agent" }), - ), - ).resolves.toMatchObject({ id: gatewayLeaseId, requesterId: "alice" }); - - await expect( - dispatcher.dispatch( - "lease.renew", - { leaseId: gatewayLeaseId }, - session({ principal: "mallory-principal", role: "agent" }), - ), - ).rejects.toMatchObject({ code: "FORBIDDEN" }); - }); - - it.each([ - ["mode slim", { mode: "slim" as const }], - ["no mode", {}], - ])("forwards a lease.request with %s to the worker as it arrived", async (_label, mode) => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", noWait: true, platform: "ios", ...mode }, - session({ role: "agent" }), - ); - - if ("mode" in mode) expect(client.lastRequestLeaseInput).toMatchObject(mode); - else expect(client.lastRequestLeaseInput).not.toHaveProperty("mode"); - }); - - it.each([ - ["a class", { class: "phone", platform: "ios" }], - ["neither a model nor a class", { platform: "ios" }], - ["a class and an OS range", { class: "phone", osVersion: ">=18", platform: "ios" }], - ["a model and an OS range", { model: "iPhone 17", osVersion: ">=18", platform: "ios" }], - [ - "a model and an exact osVersion", - { model: "iPhone 17", osVersion: "26.0", platform: "ios" }, - ], - ] as const)( - "serves a lease.request naming %s through a worker that fits, forwarding it as it arrived", - async (_label, input) => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([ - { - modelClasses: { "iPhone 17": "phone" } as const, - models: ["iPhone 17"], - platform: "ios", - runtimes: ["26.0"], - }, - ]).platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - await dispatcher.dispatch( - "lease.request", - { ...input, noWait: true }, - session({ role: "agent" }), - ); - - expect(client.lastRequestLeaseInput).toMatchObject(input); - if (!("model" in input)) expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); - }, - ); - - it("forwards a lease.request's image tag and baseline to the worker unchanged", async () => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - baselines: [ - { - imageTag: "google_apis_playstore", - mode: "full", - model: "Pixel 8", - name: "checkout", - osVersion: "34", - platform: "android", - savedAt: "2026-11-02T13:10:00Z", - version: 1, - }, - ], - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([ - { - images: [{ abi: "arm64-v8a", runtime: "34", tag: "google_apis_playstore" }], - models: ["Pixel 8"], - platform: "android", - runtimes: ["34"], - }, - ]).platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - await dispatcher.dispatch( - "lease.request", - { - baseline: "checkout", - imageTag: "google_apis_playstore", - model: "Pixel 8", - noWait: true, - platform: "android", - }, - session({ role: "agent" }), - ); - - expect(client.lastRequestLeaseInput).toMatchObject({ - baseline: "checkout", - imageTag: "google_apis_playstore", - }); - }); - - it("forwards a lease.request's leaseId to the worker, and answers the grant under that ID with no worker in front of it", async () => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { ...grantFixture().lease, id: "ad-7f3a", idChosenByRequester: true }, - }), - kind: "grant", - }); - - const grant = await dispatcher.dispatch( - "lease.request", - { leaseId: "ad-7f3a", model: "iPhone 17", noWait: true, platform: "ios" }, - session({ role: "agent" }), - ); - - expect(client.lastRequestLeaseInput).toMatchObject({ leaseId: "ad-7f3a" }); - expect(grant.lease).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); - }); - - it("forwards device.exec to the worker that holds the lease, gated on the caller owning it", async () => { - const { coordinator, directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await coordinator.request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, - ); - client.execQueue.push({ exitCode: 3, kind: "ok" }); - - await expect( - dispatcher.dispatch( - "device.exec", - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - session({ principal: "agent-1", role: "agent" }), - ), - ).resolves.toEqual({ exitCode: 3 }); - - await expect( - dispatcher.dispatch( - "device.exec", - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - session({ principal: "someone-else", role: "agent" }), - ), - ).rejects.toMatchObject({ code: "FORBIDDEN" }); - }); - - it("refuses a device.exec whose stdin is longer than 8,388,608 characters with BAD_REQUEST and forwards nothing to the worker, and forwards one at 8,388,608", async () => { - const { coordinator, directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, - ); - const exec = (stdin: string) => - dispatcher.dispatch( - "device.exec", - { args: ["spawn", "booted", "untar"], leaseId: grant.lease.id, stdin, tool: "simctl" }, - session({ principal: "agent-1", role: "agent" }), - ); - const forwarded = () => client.calls.filter((call) => call.startsWith("device.exec:")); - - client.execQueue.push({ exitCode: 0, kind: "ok" }); - await expect(exec("A".repeat(8_388_608))).resolves.toEqual({ exitCode: 0 }); - expect(forwarded()).toHaveLength(1); - - await expect(exec("A".repeat(8_388_609))).rejects.toMatchObject({ code: "BAD_REQUEST" }); - expect(forwarded()).toHaveLength(1); - }); - - it("forwards a device.exec's stdinFile to the lease's worker under the same name, and nothing when it is omitted", async () => { - const { coordinator, directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, - ); - const exec = (extra: { stdinFile?: string }) => { - client.execQueue.push({ exitCode: 0, kind: "ok" }); - return dispatcher.dispatch( - "device.exec", - { args: ["spawn", "booted", "untar"], leaseId: grant.lease.id, tool: "simctl", ...extra }, - session({ principal: "agent-1", role: "agent" }), - ); - }; - - await exec({ stdinFile: "fixture.tar" }); - await exec({}); - - expect(client.execInputs.map((forwarded) => forwarded.stdinFile)).toEqual([ - "fixture.tar", - undefined, - ]); - expect("stdinFile" in (client.execInputs[1] ?? {})).toBe(false); - }); - - it("calls onStarted for a silent device.exec from the worker's own started push (ADR §19a/§19b/§19e)", async () => { - const { coordinator, directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: true, ownerId: "agent-1", requesterId: "agent-1" }, - ); - // why: §19b's own worked example: `simctl install ` never writes anything while it runs, - // and the worker here never answers at all -- but it does say the process exists, which is - // the fact a transport needs to commit its 200. No clock is advanced below: this asserts - // the relay, not a timer. - client.execQueue.push({ kind: "hang", started: true }); - - let started = false; - void dispatcher.dispatch( - "device.exec", - { args: ["install", "/path/to.app"], leaseId: grant.lease.id, tool: "simctl" }, - session({ - onStarted: () => { - started = true; - }, - principal: "agent-1", - role: "agent", - }), - ); - await expect.poll(() => started).toBe(true); - }); - - it.each([ - { owner: "someone-else", principal: "agent-1", role: "agent" as const }, - { owner: "agent-7", principal: "operator-1", role: "admin" as const }, - ])( - "lease.request: rejects a $role session naming owner with FORBIDDEN, never silently ignoring it (ADR §27a, H3, H7)", - async ({ owner, principal, role }) => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - await expect( - dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", noWait: true, owner, platform: "ios" }, - session({ principal, role }), - ), - ).rejects.toMatchObject({ code: "FORBIDDEN" }); - }, - ); - - it("lease.request: forwards owner as `ownerId` end to end when the session is the gateway's own uplink (ADR §27a, H3) -- exercising the same field the worker's own dispatcher gates, on this dispatcher's shared handler shape", async () => { - const { directory, dispatcher, leaseIndex, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17", noWait: true, owner: "agent-7", platform: "ios" }, - session({ isGatewayUplink: true, principal: "gw:instance-1", role: "admin" }), - ); - - const gatewayLeaseId = (grant as { lease: { id: string } }).lease.id; - expect((grant as { lease: { ownerId: string } }).lease.ownerId).toBe("agent-7"); - expect(leaseIndex.ownerId(gatewayLeaseId)).toBe("agent-7"); - }); - - it("lease.request: names the stored request to the session and replays it under the same idempotency key, forwarding nothing twice", async () => { - const { directory, dispatcher, workers } = harness(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_1", client); - workers.connected("wrk_1", undefined, "0.3.0"); - workers.refresh("wrk_1", { - capacity: statusFixture().capacity, - health: "running", - queueDepth: 0, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - downloads: { policy: "on-request" }, - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const admitted: [string, boolean][] = []; - const input = { idempotencyKey: "key-1", model: "iPhone 17", platform: "ios" } as const; - const track = session({ - onRequestAdmitted: (id, replayed) => admitted.push([id, replayed]), - principal: "agent-1", - role: "agent", - }); - - const first = await dispatcher.dispatch("lease.request", input, track); - const repeat = await dispatcher.dispatch("lease.request", input, track); - - expect((repeat as { lease: { id: string } }).lease.id).toBe( - (first as { lease: { id: string } }).lease.id, - ); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - expect(admitted).toEqual([ - [expect.stringMatching(/^req_/), false], - [admitted[0]?.[0], true], - ]); - }); - }); - it("has an answer for every operation the contract declares", async () => { const inputs: Partial> = { "driver.passthrough": { args: [], tool: "adb" }, From a81eff99c27ead494fb3c5122ef8090600ac6b1d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:25:54 +0100 Subject: [PATCH 28/83] test(gateway): split fleet-routing.test.ts by topic --- .../fleet-routing-ranges-and-except.test.ts | 521 ++++++++ ...leet-routing-starting-and-physical.test.ts | 600 +++++++++ src/gateway/fleet-routing.test.ts | 1080 +---------------- 3 files changed, 1122 insertions(+), 1079 deletions(-) create mode 100644 src/gateway/fleet-routing-ranges-and-except.test.ts create mode 100644 src/gateway/fleet-routing-starting-and-physical.test.ts diff --git a/src/gateway/fleet-routing-ranges-and-except.test.ts b/src/gateway/fleet-routing-ranges-and-except.test.ts new file mode 100644 index 000000000..470e7cd6e --- /dev/null +++ b/src/gateway/fleet-routing-ranges-and-except.test.ts @@ -0,0 +1,521 @@ +import { describe, expect, it } from "vitest"; +import { fromWireError, SimlockError } from "../contract/index.js"; +import { + catalogFixture, + deviceFixture, + grantFixture, + hostFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import type { WorkerRegistry } from "./worker-registry.js"; +import { + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + requestOptions, +} from "./test-fleet.js"; + +// why: the coordinator's tests, split by subject; they assert the queue's `RequesterAlreadyLeasedError`, which index.ts does not export. +describe("FleetLeaseCoordinator routes a class or range request (ADR 0015 §8)", () => { + const phones = { + modelClasses: { "iPhone 16": "phone", "iPhone 17": "phone" } as const, + models: ["iPhone 16", "iPhone 17"], + platform: "ios" as const, + runtimes: ["18.0", "26.0"], + }; + + function connectWith( + workers: WorkerRegistry, + workerId: string, + entry: Parameters[0][number], + options: { + readonly capacity?: ReturnType["capacity"]; + readonly devices?: ReturnType[number]["devices"]; + } = {}, + ): void { + workers.connected(workerId, undefined, "0.3.0"); + workers.refresh(workerId, { + capacity: options.capacity ?? statusFixture().capacity, + catalog: catalogFixture([entry]).platforms, + devices: options.devices ?? [], + downloads: { policy: "on-request" }, + health: "running", + leases: [], + queueDepth: 0, + }); + } + + function leaseRequests(client: ScriptedWorkerClient): string[] { + return client.calls.filter((call) => call.startsWith("lease.request")); + } + + function single() { + const fleet = buildFleet(); + const client = new ScriptedWorkerClient(); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + fleet.directory.add("wrk_a", client); + connectWith(fleet.workers, "wrk_a", phones); + return { ...fleet, client }; + } + + it("forwards the class and the range as they arrived, with no model", async () => { + const { client, coordinator } = single(); + + await coordinator.request( + { class: "phone", osVersion: ">=18 <26", platform: "ios" }, + requestOptions({ noWait: true }), + ); + + expect(client.lastRequestLeaseInput).toMatchObject({ + class: "phone", + osVersion: ">=18 <26", + platform: "ios", + }); + expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); + }); + + it("forwards a request naming neither model nor class with neither", async () => { + const { client, coordinator } = single(); + + await coordinator.request({ platform: "ios" }, requestOptions({ noWait: true })); + + expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); + expect(client.lastRequestLeaseInput).not.toHaveProperty("class"); + }); + + it("still forwards an exact model under the worker's own name, with no class", async () => { + const { client, coordinator } = single(); + + await coordinator.request( + { model: "iphone 17", platform: "ios" }, + requestOptions({ noWait: true }), + ); + + expect(client.lastRequestLeaseInput?.model).toBe("iPhone 17"); + expect(client.lastRequestLeaseInput).not.toHaveProperty("class"); + }); + + it("emits request.dispatched with class and no model for a class request, and model and no class for an exact one", async () => { + const byClass = single(); + const dispatchedByClass: object[] = []; + byClass.eventBus.subscribe("request.dispatched", (envelope) => + dispatchedByClass.push(envelope.payload), + ); + await byClass.coordinator.request( + { class: "phone", platform: "ios" }, + requestOptions({ noWait: true }), + ); + const exact = single(); + const dispatchedExact: object[] = []; + exact.eventBus.subscribe("request.dispatched", (envelope) => + dispatchedExact.push(envelope.payload), + ); + await exact.coordinator.request( + { model: "iPhone 17", platform: "ios" }, + requestOptions({ noWait: true }), + ); + + expect(dispatchedByClass).toEqual([expect.objectContaining({ class: "phone" })]); + expect(dispatchedByClass[0]).not.toHaveProperty("model"); + expect(dispatchedExact).toEqual([expect.objectContaining({ model: "iPhone 17" })]); + expect(dispatchedExact[0]).not.toHaveProperty("class"); + }); + + it("sends a request for a class only one worker lists a model of to that worker", async () => { + const { coordinator, directory, workers } = buildFleet(); + const phonesOnly = new ScriptedWorkerClient(); + const withTablet = new ScriptedWorkerClient(); + directory.add("wrk_a", phonesOnly); + directory.add("wrk_b", withTablet); + withTablet.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWith(workers, "wrk_a", phones); + connectWith(workers, "wrk_b", { + ...phones, + modelClasses: { ...phones.modelClasses, "iPad Pro": "tablet" }, + models: [...phones.models, "iPad Pro"], + }); + + await coordinator.request( + { class: "tablet", platform: "ios" }, + requestOptions({ noWait: true }), + ); + + expect(leaseRequests(phonesOnly)).toEqual([]); + expect(withTablet.lastRequestLeaseInput).toMatchObject({ class: "tablet" }); + }); + + it("retries a worker's UNKNOWN_MODEL for a class request, before any progress, on another worker", async () => { + const { coordinator, directory, workers } = buildFleet(); + const a = new ScriptedWorkerClient(); + const b = new ScriptedWorkerClient(); + directory.add("wrk_a", a); + directory.add("wrk_b", b); + connectWith(workers, "wrk_a", phones); + connectWith(workers, "wrk_b", phones); + a.requestLeaseQueue.push({ + error: new SimlockError("UNKNOWN_MODEL", "domain", "no model of class phone", { + class: "phone", + platform: "ios", + }), + kind: "error", + }); + b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await coordinator.request( + { class: "phone", platform: "ios" }, + requestOptions({ noWait: true }), + ); + + expect(leaseRequests(a)).toHaveLength(1); + expect(leaseRequests(b)).toHaveLength(1); + expect(grant.lease.worker?.id).toBe("wrk_b"); + }); + + it("dispatches an Android phone request to the worker with a warm Pixel over one with free capacity", async () => { + const { coordinator, directory, workers } = buildFleet(); + const free = new ScriptedWorkerClient(); + const warm = new ScriptedWorkerClient(); + directory.add("wrk_a", free); + directory.add("wrk_b", warm); + warm.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const pixels = { + modelClasses: { "Pixel 8": "phone" } as const, + models: ["Pixel 8"], + platform: "android" as const, + runtimes: ["35"], + }; + const ready = deviceFixture("dev_1", "ready"); + connectWith(workers, "wrk_a", pixels); + connectWith(workers, "wrk_b", pixels, { + devices: [ + { + ...ready, + spec: { + physical: false, + model: "Pixel 8", + osVersion: "35", + platform: "android" as const, + }, + }, + ], + }); + + await coordinator.request( + { class: "phone", platform: "android" }, + requestOptions({ noWait: true }), + ); + + expect(leaseRequests(warm)).toHaveLength(1); + expect(leaseRequests(free)).toEqual([]); + }); +}); + +describe("FleetLeaseCoordinator: requires", () => { + const available = [{ name: "agent-device", state: "available" }] as const; + const off = [{ name: "agent-device", state: "off" }] as const; + const missing = () => + fromWireError("CAPABILITY_MISSING", "This worker cannot serve agent-device", { + capabilities: ["agent-device"], + }); + + it("sends requires to the worker with the request", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + workers.refresh("wrk_a", { capabilities: [...available] }); + + await coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })); + + expect(client.lastRequestLeaseInput).toMatchObject({ requires: ["agent-device"] }); + }); + + it("leaves a worker whose own check answers CAPABILITY_MISSING out for this request and tries the next, and gives the caller that error when no worker is left", async () => { + const { coordinator, directory, workers } = buildFleet(); + const stale = new ScriptedWorkerClient(); + const next = new ScriptedWorkerClient(); + stale.requestLeaseQueue.push({ error: missing(), kind: "error" }); + next.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + directory.add("wrk_a", stale); + directory.add("wrk_b", next); + for (const id of ["wrk_a", "wrk_b"]) { + connectWorker(workers, id); + workers.refresh(id, { capabilities: [...available] }); + } + + await expect( + coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })), + ).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); + expect(stale.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + + stale.requestLeaseQueue.push({ error: missing(), kind: "error" }); + next.requestLeaseQueue.push({ error: missing(), kind: "error" }); + await expect( + coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-2", requesterId: "agent-2", requires: ["agent-device"] }), + ), + ).rejects.toMatchObject({ + code: "CAPABILITY_MISSING", + message: "This worker cannot serve agent-device", + }); + }); + + it("fails a request waiting only on the capable worker with CAPABILITY_MISSING when that worker's view turns starting", async () => { + const { coordinator, directory, workers } = buildFleet(); + directory.add("wrk_a", new ScriptedWorkerClient()); + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + workers.refresh("wrk_a", { capabilities: [...available] }); + const waiting = coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })); + const outcome = { settled: false }; + waiting.then( + () => (outcome.settled = true), + () => (outcome.settled = true), + ); + await tick(); + expect(coordinator.queueDepth).toBe(1); + expect(outcome.settled).toBe(false); + + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + await tick(); + + expect(outcome.settled).toBe(true); + await expect(waiting).rejects.toMatchObject({ code: "CAPABILITY_MISSING" }); + }); + + it("keeps a request with requires waiting for a busy capable worker while a free worker lists the capability off", async () => { + const { coordinator, directory, workers } = buildFleet(); + directory.add("wrk_a", new ScriptedWorkerClient()); + const free = new ScriptedWorkerClient(); + directory.add("wrk_b", free); + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + workers.refresh("wrk_a", { capabilities: [...available] }); + connectWorker(workers, "wrk_b"); + workers.refresh("wrk_b", { capabilities: [...off] }); + + void coordinator + .request(REQUEST, requestOptions({ requires: ["agent-device"] })) + .catch(() => undefined); + await tick(); + + expect(coordinator.queueDepth).toBe(1); + expect(free.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + }); +}); + +describe("FleetLeaseCoordinator routes an except request by each worker's slim set and default mode (ADR 0066 §5)", () => { + const every = ["widgets", "store", "mdm", "other"]; + + function fleet( + slims: Readonly< + Record< + string, + { + categories: string[]; + defaultMode: "slim" | "full"; + baseline?: "slim" | "full"; + busy?: true; + drained?: true; + models?: string[]; + } + > + >, + ) { + const made = buildFleet(); + const clients: Record = {}; + for (const [id, slim] of Object.entries(slims)) { + const client = new ScriptedWorkerClient(); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + clients[id] = client; + made.directory.add(id, client); + connectWorker(made.workers, id, slim.busy === true ? { capacity: saturatedIos() } : {}); + if (slim.drained === true) made.workers.setDrained(id, true); + made.workers.refresh(id, { + catalog: catalogFixture([ + { + defaultMode: slim.defaultMode, + models: slim.models ?? ["iPhone 17"], + platform: "ios", + runtimes: ["26.0"], + slimCategories: slim.categories, + }, + ]).platforms, + ...(slim.baseline === undefined + ? {} + : { + baselines: [ + { + mode: slim.baseline, + model: "iPhone 17", + name: "b", + osVersion: "26.0", + platform: "ios" as const, + savedAt: "2026-11-02T13:10:00Z", + version: 1, + }, + ], + }), + }); + } + return { ...made, clients }; + } + + it("routes an except request by each worker's slim set and default mode, and gives the chosen worker the except list", async () => { + const cases: { + label: string; + request: { baseline?: string; except: string[]; mode?: "slim" | "full" }; + routedTo?: string; + rejected?: string; + rejectedCode?: "BAD_REQUEST" | "NO_CAPACITY"; + workers: Parameters[0]; + }[] = [ + { + label: "one worker slims the name", + request: { except: ["mdm"], mode: "slim" as const }, + routedTo: "wrk_b", + workers: { + wrk_a: { categories: ["store"], defaultMode: "full" as const }, + wrk_b: { categories: every, defaultMode: "full" as const }, + }, + }, + { + label: "two names, one worker slims only the first", + request: { except: ["store", "mdm"], mode: "slim" as const }, + routedTo: "wrk_b", + workers: { + wrk_a: { categories: ["store"], defaultMode: "slim" as const }, + wrk_b: { categories: every, defaultMode: "slim" as const }, + }, + }, + { + label: "no mode, mixed default modes", + request: { except: ["store"] }, + routedTo: "wrk_b", + workers: { + wrk_a: { categories: every, defaultMode: "full" as const }, + wrk_b: { categories: every, defaultMode: "slim" as const }, + }, + }, + { + label: "a slim baseline on a worker whose default mode is full", + request: { baseline: "b", except: ["store"] }, + routedTo: "wrk_a", + workers: { + wrk_a: { baseline: "slim" as const, categories: every, defaultMode: "full" as const }, + wrk_b: { categories: every, defaultMode: "slim" as const }, + }, + }, + { + label: "a full baseline on a worker whose default mode is slim", + request: { baseline: "b", except: ["store"] }, + rejected: 'no worker can keep slim categories store (baseline "b" is full)', + workers: { + wrk_a: { baseline: "full" as const, categories: every, defaultMode: "slim" as const }, + }, + }, + { + label: "no worker slims the name", + request: { except: ["other"], mode: "slim" as const }, + rejected: "no worker slims every category in except: other", + workers: { + wrk_a: { categories: ["store"], defaultMode: "slim" as const }, + wrk_b: { categories: ["store"], defaultMode: "slim" as const }, + }, + }, + { + label: "no mode, every worker defaults to full", + request: { except: ["store"] }, + rejected: + "no worker can keep slim categories store (default mode is full on every worker that slims them)", + workers: { + wrk_a: { categories: every, defaultMode: "full" as const }, + wrk_b: { categories: every, defaultMode: "full" as const }, + }, + }, + { + label: "the workers that would give a slim device do not serve the model", + request: { except: ["store"] }, + rejected: + "no worker can keep slim categories store (no worker that slims them in slim mode serves this model and runtime)", + workers: { + wrk_a: { categories: every, defaultMode: "slim" as const, models: ["iPhone 16"] }, + wrk_b: { categories: every, defaultMode: "full" as const }, + }, + }, + { + label: + "the one worker that keeps the names is drained, another serves the model without them", + request: { except: ["mdm"], mode: "slim" as const }, + rejectedCode: "NO_CAPACITY" as const, + rejected: "No worker in the fleet can currently serve this request", + workers: { + wrk_a: { categories: every, defaultMode: "slim" as const, drained: true as const }, + wrk_b: { categories: ["store"], defaultMode: "slim" as const }, + }, + }, + { + label: + "the only worker that serves the model is drained and does not keep the names, a taker that keeps them does not serve it", + request: { except: ["mdm"], mode: "slim" as const }, + rejected: + "no worker can keep slim categories mdm (no worker that slims them in slim mode serves this model and runtime)", + workers: { + wrk_a: { categories: ["store"], defaultMode: "slim" as const, drained: true as const }, + wrk_b: { categories: every, defaultMode: "slim" as const, models: ["iPhone 16"] }, + }, + }, + ]; + + for (const { label, request, routedTo, rejected, rejectedCode, workers: slims } of cases) { + const { clients, coordinator } = fleet(slims); + + if (rejected !== undefined || routedTo === undefined) { + // why: no `noWait`, so a request the coordinator would only queue shows as "waiting", not as a rejection. + const outcome = await Promise.race([ + coordinator.request({ ...REQUEST, ...request }, requestOptions()).then( + () => "granted", + (error: unknown) => error, + ), + tick().then(() => "waiting"), + ]); + expect(outcome, label).toMatchObject({ + code: rejectedCode ?? "BAD_REQUEST", + message: rejected, + }); + for (const client of Object.values(clients)) { + expect(client.lastRequestLeaseInput, label).toBeUndefined(); + } + continue; + } + await coordinator.request({ ...REQUEST, ...request }, requestOptions()); + + const received = Object.entries(clients) + .filter(([, client]) => client.lastRequestLeaseInput !== undefined) + .map(([id]) => id); + expect(received, label).toEqual([routedTo]); + expect(clients[routedTo]?.lastRequestLeaseInput, label).toMatchObject({ + except: request.except, + }); + } + + const waiting = fleet({ + wrk_a: { busy: true, categories: every, defaultMode: "slim" }, + wrk_b: { categories: ["store"], defaultMode: "slim" }, + }); + void waiting.coordinator + .request({ ...REQUEST, except: ["mdm"] }, requestOptions()) + .catch(() => undefined); + await tick(); + + expect( + (waiting.coordinator.waitingRequests()[0] as { workers?: unknown } | undefined)?.workers, + ).toEqual([ + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_a" }, + { reason: "cannot-keep-except", workerId: "wrk_b" }, + ]); + }); +}); diff --git a/src/gateway/fleet-routing-starting-and-physical.test.ts b/src/gateway/fleet-routing-starting-and-physical.test.ts new file mode 100644 index 000000000..51c571f32 --- /dev/null +++ b/src/gateway/fleet-routing-starting-and-physical.test.ts @@ -0,0 +1,600 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../contract/index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; +import { RequesterAlreadyLeasedError } from "./queue.js"; +import { + catalogFixture, + deviceFixture, + grantFixture, + hostFixture, + noCapacityError, + physicalDeviceFixture, + type PhysicalPresenceFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import type { WorkerRegistry } from "./worker-registry.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + roomierIos, + requestOptions, +} from "./test-fleet.js"; + +// why: the coordinator's tests, split by subject; they assert the queue's `RequesterAlreadyLeasedError`, which index.ts does not export. +describe("FleetLeaseCoordinator: a worker that is still starting", () => { + const gatewayLease = { + deviceId: "dev_1", + grantedAt: 1, + id: "lse_1", + lastRenewedAt: 1, + idChosenByRequester: false, + ownerId: "agent-9", + requesterId: `${GATEWAY_PREFIX}agent-9`, + ttlDeadline: 900_001, + ttlMs: 900_000, + }; + const running = (leases: (typeof gatewayLease)[]) => ({ + capacity: statusFixture().capacity, + devices: [deviceFixture("dev_1", "leased")], + health: "running" as const, + host: hostFixture(), + leases, + queueDepth: 0, + }); + + async function requestByAgent9(coordinator: FleetLeaseCoordinator) { + return coordinator + .request( + REQUEST, + requestOptions({ noWait: true, ownerId: "agent-9", requesterId: "agent-9" }), + ) + .catch((error: unknown) => error); + } + + it("gives a starting worker a view with health and host and no devices, leases or capacity, indexes none of its leases, then indexes them once the worker answers running", async () => { + const { coordinator, directory, workers } = buildFleet(); + const ready = new ScriptedWorkerClient(); + directory.add("wrk_b", ready); + ready.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_b"); + workers.connected("wrk_a", undefined, "0.3.0"); + + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + + const starting = workers.view("wrk_a"); + expect(starting?.health).toBe("starting"); + expect(starting?.host).toEqual(hostFixture()); + expect(Object.keys(starting ?? {})).not.toContain("devices"); + expect(Object.keys(starting ?? {})).not.toContain("leases"); + expect(Object.keys(starting ?? {})).not.toContain("capacity"); + const granted = await requestByAgent9(coordinator); + expect(granted).toMatchObject({ lease: { worker: { id: "wrk_b" } } }); + await coordinator.release((granted as { lease: { id: string } }).lease.id); + + workers.refresh("wrk_a", running([gatewayLease])); + + const refusal = await requestByAgent9(coordinator); + expect(refusal).toBeInstanceOf(RequesterAlreadyLeasedError); + expect((refusal as RequesterAlreadyLeasedError).existingLeaseId).toBe("wrk_a.lse_1"); + }); + + it("keeps the leases it indexed for a worker through every answer the worker gives while starting, until it answers running", async () => { + const { coordinator, workers } = buildFleet(); + workers.connected("wrk_a", undefined, "0.3.0"); + workers.refresh("wrk_a", running([gatewayLease])); + const refused = async () => { + const refusal = await requestByAgent9(coordinator); + expect(refusal).toBeInstanceOf(RequesterAlreadyLeasedError); + expect((refusal as RequesterAlreadyLeasedError).existingLeaseId).toBe("wrk_a.lse_1"); + }; + await refused(); + + for (let answer = 0; answer < 3; answer += 1) { + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + expect(Object.keys(workers.view("wrk_a") ?? {})).not.toContain("leases"); + await refused(); + } + + workers.refresh("wrk_a", running([gatewayLease])); + await refused(); + }); + + it("sends a request to the worker that answered running, never to one that answered starting", async () => { + const { coordinator, directory, workers } = buildFleet(); + const starting = new ScriptedWorkerClient(); + const ready = new ScriptedWorkerClient(); + directory.add("wrk_a", starting); + directory.add("wrk_b", ready); + ready.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b"); + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + + await expect(coordinator.request(REQUEST, requestOptions())).resolves.toMatchObject({ + lease: { worker: { id: "wrk_b" } }, + }); + + expect(starting.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + }); + + it("keeps dispatching a queued request to a running worker while another, already reconciled, answers starting", async () => { + const { coordinator, directory, workers } = buildFleet(); + const restarting = new ScriptedWorkerClient(); + const other = new ScriptedWorkerClient(); + directory.add("wrk_a", restarting); + directory.add("wrk_b", other); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b", { capacity: saturatedIos() }); + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + const granted = coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + workers.refresh("wrk_b", { capacity: statusFixture().capacity }); + await tick(); + + expect(coordinator.queueDepth).toBe(0); + expect(other.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + await expect(granted).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); + }); + + it("queues a request for the only worker that can serve it, which is starting, though another worker takes requests but lists no such model", async () => { + const { coordinator, directory, workers } = buildFleet(); + const other = new ScriptedWorkerClient(); + const restarting = new ScriptedWorkerClient(); + directory.add("wrk_a", other); + directory.add("wrk_b", restarting); + restarting.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_a", { models: ["iPad Pro"] }); + connectWorker(workers, "wrk_b"); + workers.refresh("wrk_b", { health: "starting", host: hostFixture() }); + + const granted = coordinator.request(REQUEST, requestOptions()); + await tick(); + + expect(coordinator.queueDepth).toBe(1); + expect(restarting.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + + workers.refresh("wrk_b", { + capacity: statusFixture().capacity, + catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) + .platforms, + devices: [], + health: "running", + leases: [], + queueDepth: 0, + }); + await expect(granted).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); + }); +}); + +describe("FleetLeaseCoordinator routes a physical request by the enrolled devices in the workers' views (ADR 0022)", () => { + const PHONE = { platform: "ios" as const, physical: true as const }; + const NOT_PRESENT = { present: false, reason: "not-present" } as const; + + function physicalRequest( + overrides: { + readonly model?: string; + readonly osVersion?: string; + readonly class?: "phone" | "tablet"; + } = {}, + ) { + return { ...PHONE, ...overrides }; + } + + function iphone( + id = "dev_p", + state: "ready" | "leased" | "absent" = "ready", + presence?: PhysicalPresenceFixture, + model?: string, + ) { + return physicalDeviceFixture(id, state, { + ...(presence === undefined ? {} : { presence }), + ...(model === undefined ? {} : { model }), + }); + } + + function leaseRequests(client: ScriptedWorkerClient): string[] { + return client.calls.filter((call) => call.startsWith("lease.request")); + } + + async function start( + coordinator: FleetLeaseCoordinator, + request: Parameters[0], + options: Parameters[0] = {}, + ) { + const outcome = coordinator.request(request, requestOptions(options)); + const ended = outcome.catch((error: unknown) => error); + const state = promiseState(outcome); + await tick(); + return { ended, outcome, state }; + } + + const owner = (name: string) => requestOptions({ ownerId: name, requesterId: name }); + + function oneWorker( + devices: ReturnType[], + capacity?: ReturnType["capacity"], + ) { + const fleetHarness = buildFleet(); + const client = new ScriptedWorkerClient(); + fleetHarness.directory.add("wrk_a", client); + connectWorker(fleetHarness.workers, "wrk_a", { + devices, + ...(capacity === undefined ? {} : { capacity }), + }); + return { client, ...fleetHarness }; + } + + function twoWorkers( + devicesA: ReturnType[] = [iphone()], + devicesB: ReturnType[] = [iphone("dev_q")], + ) { + const fleetHarness = buildFleet(); + const a = new ScriptedWorkerClient(); + const b = new ScriptedWorkerClient(); + fleetHarness.directory.add("wrk_a", a); + fleetHarness.directory.add("wrk_b", b); + connectWorker(fleetHarness.workers, "wrk_a", { devices: devicesA }); + connectWorker(fleetHarness.workers, "wrk_b", { devices: devicesB }); + return { a, b, ...fleetHarness }; + } + + it.each([ + { model: "iPhone 15 Pro", osVersion: "18.4", row: "exact model and OS" }, + { + model: "iphone 15 pro", + osVersion: ">=18 <19", + row: "model in another letter case and an OS range", + }, + { model: undefined, osVersion: undefined, row: "neither model nor OS" }, + ])( + "a physical request with an enrolled match is dispatched with physical: true and its model as typed ($row)", + async ({ model, osVersion }) => { + const { client, coordinator } = oneWorker([iphone()]); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { state } = await start( + coordinator, + physicalRequest({ + ...(model === undefined ? {} : { model }), + ...(osVersion === undefined ? {} : { osVersion }), + }), + ); + + expect(state.state).toBe("fulfilled"); + expect(leaseRequests(client)).toHaveLength(1); + expect(client.lastRequestLeaseInput).toMatchObject({ + ...(model === undefined ? {} : { model }), + ...(osVersion === undefined ? {} : { osVersion }), + physical: true, + }); + }, + ); + + it.each([ + { name: "model", request: physicalRequest({ model: "iPhone 99" }), devices: [iphone()] }, + { name: "class", request: physicalRequest({ class: "tablet" }), devices: [iphone()] }, + { name: "OS", request: physicalRequest({ osVersion: ">=17 <18" }), devices: [iphone()] }, + { + name: "state (deleted)", + request: physicalRequest(), + devices: [physicalDeviceFixture("dev_p", "deleted")], + }, + { name: "enrolled device", request: physicalRequest(), devices: [] }, + ])( + "a physical request with no enrolled match by $name fails UNKNOWN_MODEL at once, and no worker is asked", + async ({ request, devices }) => { + const { client, coordinator, eventBus } = oneWorker(devices); + const rejected: unknown[] = []; + eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + const { ended, state } = await start(coordinator, request); + + expect(state.state).toBe("rejected"); + expect(await ended).toMatchObject({ code: "UNKNOWN_MODEL" }); + expect(rejected).toEqual([expect.objectContaining({ reason: "unresolvable-spec" })]); + expect(leaseRequests(client)).toEqual([]); + }, + ); + + it.each([ + { + code: "NO_CAPACITY", + reason: "no-worker", + request: physicalRequest(), + row: "no worker takes requests", + setup: (_workers: WorkerRegistry) => {}, + }, + { + code: "NO_DRIVER", + reason: "unresolvable-spec", + request: { ...physicalRequest(), platform: "android" as const }, + row: "no known worker has its platform", + setup: (workers: WorkerRegistry) => connectWorker(workers, "wrk_a", { devices: [iphone()] }), + }, + ])( + "a physical request fails $code $reason when $row", + async ({ code, reason, request, setup }) => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + setup(workers); + const rejected: unknown[] = []; + eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + const { ended, state } = await start(coordinator, request); + + expect(state.state).toBe("rejected"); + expect(await ended).toMatchObject({ code }); + expect(rejected).toEqual([expect.objectContaining({ reason })]); + expect(leaseRequests(client)).toEqual([]); + }, + ); + + it("a physical request is offered to a worker with no free running slot that holds a ready enrolled match, and request.dispatched carries physical: true with reason and stage warm-hit", async () => { + const { client, coordinator, eventBus } = oneWorker([iphone()], saturatedIos()); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + await start(coordinator, physicalRequest()); + + expect(dispatched).toEqual([ + expect.objectContaining({ + physical: true, + reason: "warm-hit", + stage: "warm-hit", + workerId: "wrk_a", + }), + ]); + }); + + it("a physical request is sent to the worker that holds the ready enrolled match, not the one with a warm simulator", async () => { + const { a, b, coordinator } = twoWorkers( + [deviceFixture("dev_sim", "ready")], + [iphone("dev_q")], + ); + b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const { outcome, state } = await start(coordinator, physicalRequest()); + + expect(leaseRequests(a)).toEqual([]); + expect(state.state).toBe("fulfilled"); + expect((await outcome).lease.worker?.id).toBe("wrk_b"); + }); + + it("a physical request whose only enrolled match is on a restarting worker waits, and is not failed UNKNOWN_MODEL", async () => { + const { client, coordinator, workers } = oneWorker([iphone()]); + workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); + + const waiting = await start(coordinator, physicalRequest()); + + expect(waiting.state.state).toBe("pending"); + expect(leaseRequests(client)).toEqual([]); + }); + + it("a physical request waits when its only ready enrolled match was last read not present, and is sent once the view shows it present", async () => { + const { client, clock, coordinator, workers } = oneWorker([ + iphone("dev_p", "ready", NOT_PRESENT), + ]); + + const waiting = await start(coordinator, physicalRequest()); + expect(waiting.state.state).toBe("pending"); + expect(leaseRequests(client)).toEqual([]); + + clock.advance(1_000); + workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", { present: true })] }); + await tick(); + + expect(leaseRequests(client)).toHaveLength(1); + }); + + it("a physical request refused UNKNOWN_MODEL by worker 1 is offered to worker 2, and fails UNKNOWN_MODEL when no other worker has an enrolled match", async () => { + const unknownModel = (message: string) => + new SimlockError("UNKNOWN_MODEL", "domain", message, { platform: "ios" }); + const offered = twoWorkers(); + offered.a.requestLeaseQueue.push({ error: unknownModel("worker a: no"), kind: "error" }); + offered.b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const second = await start(offered.coordinator, physicalRequest()); + + expect(second.state.state).toBe("fulfilled"); + expect(leaseRequests(offered.a)).toHaveLength(1); + expect(leaseRequests(offered.b)).toHaveLength(1); + expect((await second.outcome).lease.worker?.id).toBe("wrk_b"); + + const only = twoWorkers([iphone()], []); + only.a.requestLeaseQueue.push({ error: unknownModel("worker a: no"), kind: "error" }); + + const failed = await start(only.coordinator, physicalRequest()); + + expect(failed.state.state).toBe("rejected"); + expect(await failed.ended).toMatchObject({ code: "UNKNOWN_MODEL", message: "worker a: no" }); + expect(leaseRequests(only.a)).toHaveLength(1); + expect(leaseRequests(only.b)).toEqual([]); + }); + + it("a worker that answers a physical request NO_CAPACITY is not sent it again by a refresh that leaves its view unchanged, and the request waits meanwhile with no reason of its own", async () => { + const { client, clock, coordinator, workers } = oneWorker([iphone()]); + + const waiting = await start(coordinator, physicalRequest()); + expect(leaseRequests(client)).toHaveLength(1); + + for (let refresh = 0; refresh < 3; refresh += 1) { + clock.advance(1_000); + workers.refresh("wrk_a", { devices: [iphone()] }); + await tick(); + } + + expect(waiting.state.state).toBe("pending"); + expect(leaseRequests(client)).toHaveLength(1); + expect(coordinator.waitingRequests()).toEqual([ + expect.objectContaining({ queuePosition: 1, stage: "queued" }), + ]); + expect(coordinator.waitingRequests()[0]).not.toHaveProperty("reason"); + }); + + it("after a NO_CAPACITY refusal, a refresh in which the device's presence turns not present sends the request nowhere, and the next refresh in which it turns present sends it to that worker again", async () => { + const { client, clock, coordinator, workers } = oneWorker([ + iphone("dev_p", "ready", { present: true }), + ]); + await start(coordinator, physicalRequest()); + expect(leaseRequests(client)).toHaveLength(1); + + clock.advance(1_000); + workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", NOT_PRESENT)] }); + await tick(); + expect(leaseRequests(client)).toHaveLength(1); + + clock.advance(1_000); + workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", { present: true })] }); + await tick(); + + expect(leaseRequests(client)).toHaveLength(2); + }); + + it("a noWait physical request refused NO_CAPACITY by its only candidate fails NO_CAPACITY and is sent to that worker once", async () => { + const { client, coordinator } = oneWorker([iphone()]); + + const noWait = await start(coordinator, physicalRequest(), requestOptions({ noWait: true })); + + expect(noWait.state.state).toBe("rejected"); + expect(await noWait.ended).toMatchObject({ code: "NO_CAPACITY" }); + expect(leaseRequests(client)).toHaveLength(1); + }); + + it("a physical request whose only enrolled match is on a disconnected worker fails NO_CAPACITY no-worker, and fails UNKNOWN_MODEL after pruneExpired forgets that worker", async () => { + const { clock, coordinator, directory, eventBus, workers } = buildFleet(); + directory.add("wrk_a", new ScriptedWorkerClient()); + directory.add("wrk_b", new ScriptedWorkerClient()); + connectWorker(workers, "wrk_a", { devices: [iphone()] }); + connectWorker(workers, "wrk_b"); + workers.disconnected("wrk_a"); + const rejected: { reason: string }[] = []; + eventBus.subscribe("lease.rejected", (envelope) => + rejected.push(envelope.payload as { reason: string }), + ); + + const first = await start(coordinator, physicalRequest(), owner("agent-1")); + + expect(first.state.state).toBe("rejected"); + expect(await first.ended).toMatchObject({ code: "NO_CAPACITY" }); + expect(rejected.map((payload) => payload.reason)).toEqual(["no-worker"]); + + clock.advance(24 * 60 * 60_000); + await workers.pruneExpired(); + const second = await start(coordinator, physicalRequest(), owner("agent-2")); + + expect(second.state.state).toBe("rejected"); + expect(await second.ended).toMatchObject({ code: "UNKNOWN_MODEL" }); + expect(rejected.map((payload) => payload.reason)).toEqual(["no-worker", "unresolvable-spec"]); + }); + + it("queued physical requests count on their own line: queue.changed names the line and lease.queued reports the position within it", async () => { + const { coordinator, eventBus } = oneWorker([iphone()], saturatedIos()); + const changed: unknown[] = []; + const queued: unknown[] = []; + eventBus.subscribe("queue.changed", (envelope) => changed.push(envelope.payload)); + eventBus.subscribe("lease.queued", (envelope) => queued.push(envelope.payload)); + + await start(coordinator, physicalRequest(), owner("agent-1")); + await start(coordinator, physicalRequest(), owner("agent-2")); + await start(coordinator, { ...REQUEST, physical: false }, owner("agent-3")); + + expect(changed).toStrictEqual([ + { depth: 1, physical: true }, + { depth: 2, physical: true }, + { depth: 1 }, + ]); + expect(queued.map((payload) => (payload as { queuePosition: number }).queuePosition)).toEqual([ + 1, 2, 1, + ]); + }); + + it("in one dispatch pass, a waiting virtual request and a waiting physical request are both sent to worker 1, the only worker that can take either", async () => { + const { client, clock, coordinator, workers } = oneWorker([iphone()], saturatedIos()); + client.requestLeaseQueue.push( + { error: noCapacityError(), kind: "error" }, + { kind: "hang" }, + { kind: "hang" }, + ); + await start(coordinator, REQUEST, owner("agent-1")); + await start(coordinator, physicalRequest(), owner("agent-2")); + const before = leaseRequests(client).length; + + clock.advance(1_000); + workers.refresh("wrk_a", { capacity: roomierIos() }); + await tick(); + + expect(leaseRequests(client).slice(before).sort()).toEqual([ + `lease.request:${GATEWAY_PREFIX}agent-1`, + `lease.request:${GATEWAY_PREFIX}agent-2`, + ]); + }); + + it("in one dispatch pass, two waiting physical requests for different models, whose ready matches are both on worker 1, are both sent to it", async () => { + const { client, clock, coordinator, workers } = oneWorker( + [iphone("dev_p"), iphone("dev_q", "ready", undefined, "iPhone 16 Pro")], + saturatedIos(), + ); + client.requestLeaseQueue.push( + { error: noCapacityError(), kind: "error" }, + { error: noCapacityError(), kind: "error" }, + { kind: "hang" }, + { kind: "hang" }, + ); + await start(coordinator, physicalRequest({ model: "iPhone 15 Pro" }), owner("agent-1")); + await start(coordinator, physicalRequest({ model: "iPhone 16 Pro" }), owner("agent-2")); + expect(leaseRequests(client)).toHaveLength(2); + + clock.advance(1_000); + workers.refresh("wrk_a", { capacity: roomierIos() }); + await tick(); + + expect(leaseRequests(client).slice(2).sort()).toEqual([ + `lease.request:${GATEWAY_PREFIX}agent-1`, + `lease.request:${GATEWAY_PREFIX}agent-2`, + ]); + }); + + it("in one dispatch pass, of two waiting physical requests with the same platform, model, class and OS, only the older is sent to worker 1, though it has two ready matches; the newer is sent in the next pass", async () => { + const { client, clock, coordinator, workers } = oneWorker( + [iphone("dev_p"), iphone("dev_q")], + saturatedIos(), + ); + client.requestLeaseQueue.push( + { error: noCapacityError(), kind: "error" }, + { error: noCapacityError(), kind: "error" }, + { kind: "hang" }, + { kind: "hang" }, + ); + const request = physicalRequest({ model: "iPhone 15 Pro", osVersion: "18.4" }); + await start(coordinator, request, owner("agent-1")); + await start(coordinator, request, owner("agent-2")); + expect(leaseRequests(client)).toHaveLength(2); + + clock.advance(1_000); + workers.refresh("wrk_a", { capacity: roomierIos() }); + await tick(); + expect(leaseRequests(client).slice(2)).toEqual([`lease.request:${GATEWAY_PREFIX}agent-1`]); + + clock.advance(1_000); + workers.refresh("wrk_a", { + capacity: { + ...statusFixture().capacity, + ios: { ...statusFixture().capacity.ios, maxRunning: 4 }, + }, + }); + await tick(); + + expect(leaseRequests(client).slice(3)).toEqual([`lease.request:${GATEWAY_PREFIX}agent-2`]); + }); +}); diff --git a/src/gateway/fleet-routing.test.ts b/src/gateway/fleet-routing.test.ts index 38b4878bc..ec5b6c558 100644 --- a/src/gateway/fleet-routing.test.ts +++ b/src/gateway/fleet-routing.test.ts @@ -1,25 +1,20 @@ import { describe, expect, it } from "vitest"; import type { EventBus } from "../bus/index.js"; -import { fromWireError, SimlockError } from "../contract/index.js"; +import { SimlockError } from "../contract/index.js"; import { QueueTimeoutError } from "../leasing/index.js"; import { DispatchError } from "../daemon/dispatch.js"; import { promiseState } from "../test-support/promise-state.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import { RequesterAlreadyLeasedError } from "./queue.js"; import { catalogFixture, - deviceFixture, grantFixture, hostFixture, noCapacityError, - physicalDeviceFixture, - type PhysicalPresenceFixture, ScriptedWorkerClient, statusFixture, } from "./test-support.js"; import type { WorkerRegistry } from "./worker-registry.js"; import { - GATEWAY_PREFIX, buildFleet, connectWorker, tick, @@ -866,1076 +861,3 @@ describe("FleetLeaseCoordinator retries a worker's cannot-serve refusal on anoth }); }); }); - -describe("FleetLeaseCoordinator routes a class or range request (ADR 0015 §8)", () => { - const phones = { - modelClasses: { "iPhone 16": "phone", "iPhone 17": "phone" } as const, - models: ["iPhone 16", "iPhone 17"], - platform: "ios" as const, - runtimes: ["18.0", "26.0"], - }; - - function connectWith( - workers: WorkerRegistry, - workerId: string, - entry: Parameters[0][number], - options: { - readonly capacity?: ReturnType["capacity"]; - readonly devices?: ReturnType[number]["devices"]; - } = {}, - ): void { - workers.connected(workerId, undefined, "0.3.0"); - workers.refresh(workerId, { - capacity: options.capacity ?? statusFixture().capacity, - catalog: catalogFixture([entry]).platforms, - devices: options.devices ?? [], - downloads: { policy: "on-request" }, - health: "running", - leases: [], - queueDepth: 0, - }); - } - - function leaseRequests(client: ScriptedWorkerClient): string[] { - return client.calls.filter((call) => call.startsWith("lease.request")); - } - - function single() { - const fleet = buildFleet(); - const client = new ScriptedWorkerClient(); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - fleet.directory.add("wrk_a", client); - connectWith(fleet.workers, "wrk_a", phones); - return { ...fleet, client }; - } - - it("forwards the class and the range as they arrived, with no model", async () => { - const { client, coordinator } = single(); - - await coordinator.request( - { class: "phone", osVersion: ">=18 <26", platform: "ios" }, - requestOptions({ noWait: true }), - ); - - expect(client.lastRequestLeaseInput).toMatchObject({ - class: "phone", - osVersion: ">=18 <26", - platform: "ios", - }); - expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); - }); - - it("forwards a request naming neither model nor class with neither", async () => { - const { client, coordinator } = single(); - - await coordinator.request({ platform: "ios" }, requestOptions({ noWait: true })); - - expect(client.lastRequestLeaseInput).not.toHaveProperty("model"); - expect(client.lastRequestLeaseInput).not.toHaveProperty("class"); - }); - - it("still forwards an exact model under the worker's own name, with no class", async () => { - const { client, coordinator } = single(); - - await coordinator.request( - { model: "iphone 17", platform: "ios" }, - requestOptions({ noWait: true }), - ); - - expect(client.lastRequestLeaseInput?.model).toBe("iPhone 17"); - expect(client.lastRequestLeaseInput).not.toHaveProperty("class"); - }); - - it("emits request.dispatched with class and no model for a class request, and model and no class for an exact one", async () => { - const byClass = single(); - const dispatchedByClass: object[] = []; - byClass.eventBus.subscribe("request.dispatched", (envelope) => - dispatchedByClass.push(envelope.payload), - ); - await byClass.coordinator.request( - { class: "phone", platform: "ios" }, - requestOptions({ noWait: true }), - ); - const exact = single(); - const dispatchedExact: object[] = []; - exact.eventBus.subscribe("request.dispatched", (envelope) => - dispatchedExact.push(envelope.payload), - ); - await exact.coordinator.request( - { model: "iPhone 17", platform: "ios" }, - requestOptions({ noWait: true }), - ); - - expect(dispatchedByClass).toEqual([expect.objectContaining({ class: "phone" })]); - expect(dispatchedByClass[0]).not.toHaveProperty("model"); - expect(dispatchedExact).toEqual([expect.objectContaining({ model: "iPhone 17" })]); - expect(dispatchedExact[0]).not.toHaveProperty("class"); - }); - - it("sends a request for a class only one worker lists a model of to that worker", async () => { - const { coordinator, directory, workers } = buildFleet(); - const phonesOnly = new ScriptedWorkerClient(); - const withTablet = new ScriptedWorkerClient(); - directory.add("wrk_a", phonesOnly); - directory.add("wrk_b", withTablet); - withTablet.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - connectWith(workers, "wrk_a", phones); - connectWith(workers, "wrk_b", { - ...phones, - modelClasses: { ...phones.modelClasses, "iPad Pro": "tablet" }, - models: [...phones.models, "iPad Pro"], - }); - - await coordinator.request( - { class: "tablet", platform: "ios" }, - requestOptions({ noWait: true }), - ); - - expect(leaseRequests(phonesOnly)).toEqual([]); - expect(withTablet.lastRequestLeaseInput).toMatchObject({ class: "tablet" }); - }); - - it("retries a worker's UNKNOWN_MODEL for a class request, before any progress, on another worker", async () => { - const { coordinator, directory, workers } = buildFleet(); - const a = new ScriptedWorkerClient(); - const b = new ScriptedWorkerClient(); - directory.add("wrk_a", a); - directory.add("wrk_b", b); - connectWith(workers, "wrk_a", phones); - connectWith(workers, "wrk_b", phones); - a.requestLeaseQueue.push({ - error: new SimlockError("UNKNOWN_MODEL", "domain", "no model of class phone", { - class: "phone", - platform: "ios", - }), - kind: "error", - }); - b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await coordinator.request( - { class: "phone", platform: "ios" }, - requestOptions({ noWait: true }), - ); - - expect(leaseRequests(a)).toHaveLength(1); - expect(leaseRequests(b)).toHaveLength(1); - expect(grant.lease.worker?.id).toBe("wrk_b"); - }); - - it("dispatches an Android phone request to the worker with a warm Pixel over one with free capacity", async () => { - const { coordinator, directory, workers } = buildFleet(); - const free = new ScriptedWorkerClient(); - const warm = new ScriptedWorkerClient(); - directory.add("wrk_a", free); - directory.add("wrk_b", warm); - warm.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const pixels = { - modelClasses: { "Pixel 8": "phone" } as const, - models: ["Pixel 8"], - platform: "android" as const, - runtimes: ["35"], - }; - const ready = deviceFixture("dev_1", "ready"); - connectWith(workers, "wrk_a", pixels); - connectWith(workers, "wrk_b", pixels, { - devices: [ - { - ...ready, - spec: { - physical: false, - model: "Pixel 8", - osVersion: "35", - platform: "android" as const, - }, - }, - ], - }); - - await coordinator.request( - { class: "phone", platform: "android" }, - requestOptions({ noWait: true }), - ); - - expect(leaseRequests(warm)).toHaveLength(1); - expect(leaseRequests(free)).toEqual([]); - }); -}); - -describe("FleetLeaseCoordinator: requires", () => { - const available = [{ name: "agent-device", state: "available" }] as const; - const off = [{ name: "agent-device", state: "off" }] as const; - const missing = () => - fromWireError("CAPABILITY_MISSING", "This worker cannot serve agent-device", { - capabilities: ["agent-device"], - }); - - it("sends requires to the worker with the request", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - workers.refresh("wrk_a", { capabilities: [...available] }); - - await coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })); - - expect(client.lastRequestLeaseInput).toMatchObject({ requires: ["agent-device"] }); - }); - - it("leaves a worker whose own check answers CAPABILITY_MISSING out for this request and tries the next, and gives the caller that error when no worker is left", async () => { - const { coordinator, directory, workers } = buildFleet(); - const stale = new ScriptedWorkerClient(); - const next = new ScriptedWorkerClient(); - stale.requestLeaseQueue.push({ error: missing(), kind: "error" }); - next.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - directory.add("wrk_a", stale); - directory.add("wrk_b", next); - for (const id of ["wrk_a", "wrk_b"]) { - connectWorker(workers, id); - workers.refresh(id, { capabilities: [...available] }); - } - - await expect( - coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })), - ).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); - expect(stale.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - - stale.requestLeaseQueue.push({ error: missing(), kind: "error" }); - next.requestLeaseQueue.push({ error: missing(), kind: "error" }); - await expect( - coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-2", requesterId: "agent-2", requires: ["agent-device"] }), - ), - ).rejects.toMatchObject({ - code: "CAPABILITY_MISSING", - message: "This worker cannot serve agent-device", - }); - }); - - it("fails a request waiting only on the capable worker with CAPABILITY_MISSING when that worker's view turns starting", async () => { - const { coordinator, directory, workers } = buildFleet(); - directory.add("wrk_a", new ScriptedWorkerClient()); - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - workers.refresh("wrk_a", { capabilities: [...available] }); - const waiting = coordinator.request(REQUEST, requestOptions({ requires: ["agent-device"] })); - const outcome = { settled: false }; - waiting.then( - () => (outcome.settled = true), - () => (outcome.settled = true), - ); - await tick(); - expect(coordinator.queueDepth).toBe(1); - expect(outcome.settled).toBe(false); - - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - await tick(); - - expect(outcome.settled).toBe(true); - await expect(waiting).rejects.toMatchObject({ code: "CAPABILITY_MISSING" }); - }); - - it("keeps a request with requires waiting for a busy capable worker while a free worker lists the capability off", async () => { - const { coordinator, directory, workers } = buildFleet(); - directory.add("wrk_a", new ScriptedWorkerClient()); - const free = new ScriptedWorkerClient(); - directory.add("wrk_b", free); - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - workers.refresh("wrk_a", { capabilities: [...available] }); - connectWorker(workers, "wrk_b"); - workers.refresh("wrk_b", { capabilities: [...off] }); - - void coordinator - .request(REQUEST, requestOptions({ requires: ["agent-device"] })) - .catch(() => undefined); - await tick(); - - expect(coordinator.queueDepth).toBe(1); - expect(free.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - }); -}); - -describe("FleetLeaseCoordinator routes an except request by each worker's slim set and default mode (ADR 0066 §5)", () => { - const every = ["widgets", "store", "mdm", "other"]; - - function fleet( - slims: Readonly< - Record< - string, - { - categories: string[]; - defaultMode: "slim" | "full"; - baseline?: "slim" | "full"; - busy?: true; - drained?: true; - models?: string[]; - } - > - >, - ) { - const made = buildFleet(); - const clients: Record = {}; - for (const [id, slim] of Object.entries(slims)) { - const client = new ScriptedWorkerClient(); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - clients[id] = client; - made.directory.add(id, client); - connectWorker(made.workers, id, slim.busy === true ? { capacity: saturatedIos() } : {}); - if (slim.drained === true) made.workers.setDrained(id, true); - made.workers.refresh(id, { - catalog: catalogFixture([ - { - defaultMode: slim.defaultMode, - models: slim.models ?? ["iPhone 17"], - platform: "ios", - runtimes: ["26.0"], - slimCategories: slim.categories, - }, - ]).platforms, - ...(slim.baseline === undefined - ? {} - : { - baselines: [ - { - mode: slim.baseline, - model: "iPhone 17", - name: "b", - osVersion: "26.0", - platform: "ios" as const, - savedAt: "2026-11-02T13:10:00Z", - version: 1, - }, - ], - }), - }); - } - return { ...made, clients }; - } - - it("routes an except request by each worker's slim set and default mode, and gives the chosen worker the except list", async () => { - const cases: { - label: string; - request: { baseline?: string; except: string[]; mode?: "slim" | "full" }; - routedTo?: string; - rejected?: string; - rejectedCode?: "BAD_REQUEST" | "NO_CAPACITY"; - workers: Parameters[0]; - }[] = [ - { - label: "one worker slims the name", - request: { except: ["mdm"], mode: "slim" as const }, - routedTo: "wrk_b", - workers: { - wrk_a: { categories: ["store"], defaultMode: "full" as const }, - wrk_b: { categories: every, defaultMode: "full" as const }, - }, - }, - { - label: "two names, one worker slims only the first", - request: { except: ["store", "mdm"], mode: "slim" as const }, - routedTo: "wrk_b", - workers: { - wrk_a: { categories: ["store"], defaultMode: "slim" as const }, - wrk_b: { categories: every, defaultMode: "slim" as const }, - }, - }, - { - label: "no mode, mixed default modes", - request: { except: ["store"] }, - routedTo: "wrk_b", - workers: { - wrk_a: { categories: every, defaultMode: "full" as const }, - wrk_b: { categories: every, defaultMode: "slim" as const }, - }, - }, - { - label: "a slim baseline on a worker whose default mode is full", - request: { baseline: "b", except: ["store"] }, - routedTo: "wrk_a", - workers: { - wrk_a: { baseline: "slim" as const, categories: every, defaultMode: "full" as const }, - wrk_b: { categories: every, defaultMode: "slim" as const }, - }, - }, - { - label: "a full baseline on a worker whose default mode is slim", - request: { baseline: "b", except: ["store"] }, - rejected: 'no worker can keep slim categories store (baseline "b" is full)', - workers: { - wrk_a: { baseline: "full" as const, categories: every, defaultMode: "slim" as const }, - }, - }, - { - label: "no worker slims the name", - request: { except: ["other"], mode: "slim" as const }, - rejected: "no worker slims every category in except: other", - workers: { - wrk_a: { categories: ["store"], defaultMode: "slim" as const }, - wrk_b: { categories: ["store"], defaultMode: "slim" as const }, - }, - }, - { - label: "no mode, every worker defaults to full", - request: { except: ["store"] }, - rejected: - "no worker can keep slim categories store (default mode is full on every worker that slims them)", - workers: { - wrk_a: { categories: every, defaultMode: "full" as const }, - wrk_b: { categories: every, defaultMode: "full" as const }, - }, - }, - { - label: "the workers that would give a slim device do not serve the model", - request: { except: ["store"] }, - rejected: - "no worker can keep slim categories store (no worker that slims them in slim mode serves this model and runtime)", - workers: { - wrk_a: { categories: every, defaultMode: "slim" as const, models: ["iPhone 16"] }, - wrk_b: { categories: every, defaultMode: "full" as const }, - }, - }, - { - label: - "the one worker that keeps the names is drained, another serves the model without them", - request: { except: ["mdm"], mode: "slim" as const }, - rejectedCode: "NO_CAPACITY" as const, - rejected: "No worker in the fleet can currently serve this request", - workers: { - wrk_a: { categories: every, defaultMode: "slim" as const, drained: true as const }, - wrk_b: { categories: ["store"], defaultMode: "slim" as const }, - }, - }, - { - label: - "the only worker that serves the model is drained and does not keep the names, a taker that keeps them does not serve it", - request: { except: ["mdm"], mode: "slim" as const }, - rejected: - "no worker can keep slim categories mdm (no worker that slims them in slim mode serves this model and runtime)", - workers: { - wrk_a: { categories: ["store"], defaultMode: "slim" as const, drained: true as const }, - wrk_b: { categories: every, defaultMode: "slim" as const, models: ["iPhone 16"] }, - }, - }, - ]; - - for (const { label, request, routedTo, rejected, rejectedCode, workers: slims } of cases) { - const { clients, coordinator } = fleet(slims); - - if (rejected !== undefined || routedTo === undefined) { - // why: no `noWait`, so a request the coordinator would only queue shows as "waiting", not as a rejection. - const outcome = await Promise.race([ - coordinator.request({ ...REQUEST, ...request }, requestOptions()).then( - () => "granted", - (error: unknown) => error, - ), - tick().then(() => "waiting"), - ]); - expect(outcome, label).toMatchObject({ - code: rejectedCode ?? "BAD_REQUEST", - message: rejected, - }); - for (const client of Object.values(clients)) { - expect(client.lastRequestLeaseInput, label).toBeUndefined(); - } - continue; - } - await coordinator.request({ ...REQUEST, ...request }, requestOptions()); - - const received = Object.entries(clients) - .filter(([, client]) => client.lastRequestLeaseInput !== undefined) - .map(([id]) => id); - expect(received, label).toEqual([routedTo]); - expect(clients[routedTo]?.lastRequestLeaseInput, label).toMatchObject({ - except: request.except, - }); - } - - const waiting = fleet({ - wrk_a: { busy: true, categories: every, defaultMode: "slim" }, - wrk_b: { categories: ["store"], defaultMode: "slim" }, - }); - void waiting.coordinator - .request({ ...REQUEST, except: ["mdm"] }, requestOptions()) - .catch(() => undefined); - await tick(); - - expect( - (waiting.coordinator.waitingRequests()[0] as { workers?: unknown } | undefined)?.workers, - ).toEqual([ - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_a" }, - { reason: "cannot-keep-except", workerId: "wrk_b" }, - ]); - }); -}); - -describe("FleetLeaseCoordinator: a worker that is still starting", () => { - const gatewayLease = { - deviceId: "dev_1", - grantedAt: 1, - id: "lse_1", - lastRenewedAt: 1, - idChosenByRequester: false, - ownerId: "agent-9", - requesterId: `${GATEWAY_PREFIX}agent-9`, - ttlDeadline: 900_001, - ttlMs: 900_000, - }; - const running = (leases: (typeof gatewayLease)[]) => ({ - capacity: statusFixture().capacity, - devices: [deviceFixture("dev_1", "leased")], - health: "running" as const, - host: hostFixture(), - leases, - queueDepth: 0, - }); - - async function requestByAgent9(coordinator: FleetLeaseCoordinator) { - return coordinator - .request( - REQUEST, - requestOptions({ noWait: true, ownerId: "agent-9", requesterId: "agent-9" }), - ) - .catch((error: unknown) => error); - } - - it("gives a starting worker a view with health and host and no devices, leases or capacity, indexes none of its leases, then indexes them once the worker answers running", async () => { - const { coordinator, directory, workers } = buildFleet(); - const ready = new ScriptedWorkerClient(); - directory.add("wrk_b", ready); - ready.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - connectWorker(workers, "wrk_b"); - workers.connected("wrk_a", undefined, "0.3.0"); - - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - - const starting = workers.view("wrk_a"); - expect(starting?.health).toBe("starting"); - expect(starting?.host).toEqual(hostFixture()); - expect(Object.keys(starting ?? {})).not.toContain("devices"); - expect(Object.keys(starting ?? {})).not.toContain("leases"); - expect(Object.keys(starting ?? {})).not.toContain("capacity"); - const granted = await requestByAgent9(coordinator); - expect(granted).toMatchObject({ lease: { worker: { id: "wrk_b" } } }); - await coordinator.release((granted as { lease: { id: string } }).lease.id); - - workers.refresh("wrk_a", running([gatewayLease])); - - const refusal = await requestByAgent9(coordinator); - expect(refusal).toBeInstanceOf(RequesterAlreadyLeasedError); - expect((refusal as RequesterAlreadyLeasedError).existingLeaseId).toBe("wrk_a.lse_1"); - }); - - it("keeps the leases it indexed for a worker through every answer the worker gives while starting, until it answers running", async () => { - const { coordinator, workers } = buildFleet(); - workers.connected("wrk_a", undefined, "0.3.0"); - workers.refresh("wrk_a", running([gatewayLease])); - const refused = async () => { - const refusal = await requestByAgent9(coordinator); - expect(refusal).toBeInstanceOf(RequesterAlreadyLeasedError); - expect((refusal as RequesterAlreadyLeasedError).existingLeaseId).toBe("wrk_a.lse_1"); - }; - await refused(); - - for (let answer = 0; answer < 3; answer += 1) { - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - expect(Object.keys(workers.view("wrk_a") ?? {})).not.toContain("leases"); - await refused(); - } - - workers.refresh("wrk_a", running([gatewayLease])); - await refused(); - }); - - it("sends a request to the worker that answered running, never to one that answered starting", async () => { - const { coordinator, directory, workers } = buildFleet(); - const starting = new ScriptedWorkerClient(); - const ready = new ScriptedWorkerClient(); - directory.add("wrk_a", starting); - directory.add("wrk_b", ready); - ready.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - connectWorker(workers, "wrk_a"); - connectWorker(workers, "wrk_b"); - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - - await expect(coordinator.request(REQUEST, requestOptions())).resolves.toMatchObject({ - lease: { worker: { id: "wrk_b" } }, - }); - - expect(starting.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - }); - - it("keeps dispatching a queued request to a running worker while another, already reconciled, answers starting", async () => { - const { coordinator, directory, workers } = buildFleet(); - const restarting = new ScriptedWorkerClient(); - const other = new ScriptedWorkerClient(); - directory.add("wrk_a", restarting); - directory.add("wrk_b", other); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - connectWorker(workers, "wrk_a"); - connectWorker(workers, "wrk_b", { capacity: saturatedIos() }); - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - const granted = coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - workers.refresh("wrk_b", { capacity: statusFixture().capacity }); - await tick(); - - expect(coordinator.queueDepth).toBe(0); - expect(other.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - await expect(granted).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); - }); - - it("queues a request for the only worker that can serve it, which is starting, though another worker takes requests but lists no such model", async () => { - const { coordinator, directory, workers } = buildFleet(); - const other = new ScriptedWorkerClient(); - const restarting = new ScriptedWorkerClient(); - directory.add("wrk_a", other); - directory.add("wrk_b", restarting); - restarting.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - connectWorker(workers, "wrk_a", { models: ["iPad Pro"] }); - connectWorker(workers, "wrk_b"); - workers.refresh("wrk_b", { health: "starting", host: hostFixture() }); - - const granted = coordinator.request(REQUEST, requestOptions()); - await tick(); - - expect(coordinator.queueDepth).toBe(1); - expect(restarting.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - - workers.refresh("wrk_b", { - capacity: statusFixture().capacity, - catalog: catalogFixture([{ models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }]) - .platforms, - devices: [], - health: "running", - leases: [], - queueDepth: 0, - }); - await expect(granted).resolves.toMatchObject({ lease: { worker: { id: "wrk_b" } } }); - }); -}); - -describe("FleetLeaseCoordinator routes a physical request by the enrolled devices in the workers' views (ADR 0022)", () => { - const PHONE = { platform: "ios" as const, physical: true as const }; - const NOT_PRESENT = { present: false, reason: "not-present" } as const; - - function physicalRequest( - overrides: { - readonly model?: string; - readonly osVersion?: string; - readonly class?: "phone" | "tablet"; - } = {}, - ) { - return { ...PHONE, ...overrides }; - } - - function iphone( - id = "dev_p", - state: "ready" | "leased" | "absent" = "ready", - presence?: PhysicalPresenceFixture, - model?: string, - ) { - return physicalDeviceFixture(id, state, { - ...(presence === undefined ? {} : { presence }), - ...(model === undefined ? {} : { model }), - }); - } - - function leaseRequests(client: ScriptedWorkerClient): string[] { - return client.calls.filter((call) => call.startsWith("lease.request")); - } - - async function start( - coordinator: FleetLeaseCoordinator, - request: Parameters[0], - options: Parameters[0] = {}, - ) { - const outcome = coordinator.request(request, requestOptions(options)); - const ended = outcome.catch((error: unknown) => error); - const state = promiseState(outcome); - await tick(); - return { ended, outcome, state }; - } - - const owner = (name: string) => requestOptions({ ownerId: name, requesterId: name }); - - function oneWorker( - devices: ReturnType[], - capacity?: ReturnType["capacity"], - ) { - const fleetHarness = buildFleet(); - const client = new ScriptedWorkerClient(); - fleetHarness.directory.add("wrk_a", client); - connectWorker(fleetHarness.workers, "wrk_a", { - devices, - ...(capacity === undefined ? {} : { capacity }), - }); - return { client, ...fleetHarness }; - } - - function twoWorkers( - devicesA: ReturnType[] = [iphone()], - devicesB: ReturnType[] = [iphone("dev_q")], - ) { - const fleetHarness = buildFleet(); - const a = new ScriptedWorkerClient(); - const b = new ScriptedWorkerClient(); - fleetHarness.directory.add("wrk_a", a); - fleetHarness.directory.add("wrk_b", b); - connectWorker(fleetHarness.workers, "wrk_a", { devices: devicesA }); - connectWorker(fleetHarness.workers, "wrk_b", { devices: devicesB }); - return { a, b, ...fleetHarness }; - } - - it.each([ - { model: "iPhone 15 Pro", osVersion: "18.4", row: "exact model and OS" }, - { - model: "iphone 15 pro", - osVersion: ">=18 <19", - row: "model in another letter case and an OS range", - }, - { model: undefined, osVersion: undefined, row: "neither model nor OS" }, - ])( - "a physical request with an enrolled match is dispatched with physical: true and its model as typed ($row)", - async ({ model, osVersion }) => { - const { client, coordinator } = oneWorker([iphone()]); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { state } = await start( - coordinator, - physicalRequest({ - ...(model === undefined ? {} : { model }), - ...(osVersion === undefined ? {} : { osVersion }), - }), - ); - - expect(state.state).toBe("fulfilled"); - expect(leaseRequests(client)).toHaveLength(1); - expect(client.lastRequestLeaseInput).toMatchObject({ - ...(model === undefined ? {} : { model }), - ...(osVersion === undefined ? {} : { osVersion }), - physical: true, - }); - }, - ); - - it.each([ - { name: "model", request: physicalRequest({ model: "iPhone 99" }), devices: [iphone()] }, - { name: "class", request: physicalRequest({ class: "tablet" }), devices: [iphone()] }, - { name: "OS", request: physicalRequest({ osVersion: ">=17 <18" }), devices: [iphone()] }, - { - name: "state (deleted)", - request: physicalRequest(), - devices: [physicalDeviceFixture("dev_p", "deleted")], - }, - { name: "enrolled device", request: physicalRequest(), devices: [] }, - ])( - "a physical request with no enrolled match by $name fails UNKNOWN_MODEL at once, and no worker is asked", - async ({ request, devices }) => { - const { client, coordinator, eventBus } = oneWorker(devices); - const rejected: unknown[] = []; - eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - - const { ended, state } = await start(coordinator, request); - - expect(state.state).toBe("rejected"); - expect(await ended).toMatchObject({ code: "UNKNOWN_MODEL" }); - expect(rejected).toEqual([expect.objectContaining({ reason: "unresolvable-spec" })]); - expect(leaseRequests(client)).toEqual([]); - }, - ); - - it.each([ - { - code: "NO_CAPACITY", - reason: "no-worker", - request: physicalRequest(), - row: "no worker takes requests", - setup: (_workers: WorkerRegistry) => {}, - }, - { - code: "NO_DRIVER", - reason: "unresolvable-spec", - request: { ...physicalRequest(), platform: "android" as const }, - row: "no known worker has its platform", - setup: (workers: WorkerRegistry) => connectWorker(workers, "wrk_a", { devices: [iphone()] }), - }, - ])( - "a physical request fails $code $reason when $row", - async ({ code, reason, request, setup }) => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - setup(workers); - const rejected: unknown[] = []; - eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - - const { ended, state } = await start(coordinator, request); - - expect(state.state).toBe("rejected"); - expect(await ended).toMatchObject({ code }); - expect(rejected).toEqual([expect.objectContaining({ reason })]); - expect(leaseRequests(client)).toEqual([]); - }, - ); - - it("a physical request is offered to a worker with no free running slot that holds a ready enrolled match, and request.dispatched carries physical: true with reason and stage warm-hit", async () => { - const { client, coordinator, eventBus } = oneWorker([iphone()], saturatedIos()); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - await start(coordinator, physicalRequest()); - - expect(dispatched).toEqual([ - expect.objectContaining({ - physical: true, - reason: "warm-hit", - stage: "warm-hit", - workerId: "wrk_a", - }), - ]); - }); - - it("a physical request is sent to the worker that holds the ready enrolled match, not the one with a warm simulator", async () => { - const { a, b, coordinator } = twoWorkers( - [deviceFixture("dev_sim", "ready")], - [iphone("dev_q")], - ); - b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const { outcome, state } = await start(coordinator, physicalRequest()); - - expect(leaseRequests(a)).toEqual([]); - expect(state.state).toBe("fulfilled"); - expect((await outcome).lease.worker?.id).toBe("wrk_b"); - }); - - it("a physical request whose only enrolled match is on a restarting worker waits, and is not failed UNKNOWN_MODEL", async () => { - const { client, coordinator, workers } = oneWorker([iphone()]); - workers.refresh("wrk_a", { health: "starting", host: hostFixture() }); - - const waiting = await start(coordinator, physicalRequest()); - - expect(waiting.state.state).toBe("pending"); - expect(leaseRequests(client)).toEqual([]); - }); - - it("a physical request waits when its only ready enrolled match was last read not present, and is sent once the view shows it present", async () => { - const { client, clock, coordinator, workers } = oneWorker([ - iphone("dev_p", "ready", NOT_PRESENT), - ]); - - const waiting = await start(coordinator, physicalRequest()); - expect(waiting.state.state).toBe("pending"); - expect(leaseRequests(client)).toEqual([]); - - clock.advance(1_000); - workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", { present: true })] }); - await tick(); - - expect(leaseRequests(client)).toHaveLength(1); - }); - - it("a physical request refused UNKNOWN_MODEL by worker 1 is offered to worker 2, and fails UNKNOWN_MODEL when no other worker has an enrolled match", async () => { - const unknownModel = (message: string) => - new SimlockError("UNKNOWN_MODEL", "domain", message, { platform: "ios" }); - const offered = twoWorkers(); - offered.a.requestLeaseQueue.push({ error: unknownModel("worker a: no"), kind: "error" }); - offered.b.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const second = await start(offered.coordinator, physicalRequest()); - - expect(second.state.state).toBe("fulfilled"); - expect(leaseRequests(offered.a)).toHaveLength(1); - expect(leaseRequests(offered.b)).toHaveLength(1); - expect((await second.outcome).lease.worker?.id).toBe("wrk_b"); - - const only = twoWorkers([iphone()], []); - only.a.requestLeaseQueue.push({ error: unknownModel("worker a: no"), kind: "error" }); - - const failed = await start(only.coordinator, physicalRequest()); - - expect(failed.state.state).toBe("rejected"); - expect(await failed.ended).toMatchObject({ code: "UNKNOWN_MODEL", message: "worker a: no" }); - expect(leaseRequests(only.a)).toHaveLength(1); - expect(leaseRequests(only.b)).toEqual([]); - }); - - it("a worker that answers a physical request NO_CAPACITY is not sent it again by a refresh that leaves its view unchanged, and the request waits meanwhile with no reason of its own", async () => { - const { client, clock, coordinator, workers } = oneWorker([iphone()]); - - const waiting = await start(coordinator, physicalRequest()); - expect(leaseRequests(client)).toHaveLength(1); - - for (let refresh = 0; refresh < 3; refresh += 1) { - clock.advance(1_000); - workers.refresh("wrk_a", { devices: [iphone()] }); - await tick(); - } - - expect(waiting.state.state).toBe("pending"); - expect(leaseRequests(client)).toHaveLength(1); - expect(coordinator.waitingRequests()).toEqual([ - expect.objectContaining({ queuePosition: 1, stage: "queued" }), - ]); - expect(coordinator.waitingRequests()[0]).not.toHaveProperty("reason"); - }); - - it("after a NO_CAPACITY refusal, a refresh in which the device's presence turns not present sends the request nowhere, and the next refresh in which it turns present sends it to that worker again", async () => { - const { client, clock, coordinator, workers } = oneWorker([ - iphone("dev_p", "ready", { present: true }), - ]); - await start(coordinator, physicalRequest()); - expect(leaseRequests(client)).toHaveLength(1); - - clock.advance(1_000); - workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", NOT_PRESENT)] }); - await tick(); - expect(leaseRequests(client)).toHaveLength(1); - - clock.advance(1_000); - workers.refresh("wrk_a", { devices: [iphone("dev_p", "ready", { present: true })] }); - await tick(); - - expect(leaseRequests(client)).toHaveLength(2); - }); - - it("a noWait physical request refused NO_CAPACITY by its only candidate fails NO_CAPACITY and is sent to that worker once", async () => { - const { client, coordinator } = oneWorker([iphone()]); - - const noWait = await start(coordinator, physicalRequest(), requestOptions({ noWait: true })); - - expect(noWait.state.state).toBe("rejected"); - expect(await noWait.ended).toMatchObject({ code: "NO_CAPACITY" }); - expect(leaseRequests(client)).toHaveLength(1); - }); - - it("a physical request whose only enrolled match is on a disconnected worker fails NO_CAPACITY no-worker, and fails UNKNOWN_MODEL after pruneExpired forgets that worker", async () => { - const { clock, coordinator, directory, eventBus, workers } = buildFleet(); - directory.add("wrk_a", new ScriptedWorkerClient()); - directory.add("wrk_b", new ScriptedWorkerClient()); - connectWorker(workers, "wrk_a", { devices: [iphone()] }); - connectWorker(workers, "wrk_b"); - workers.disconnected("wrk_a"); - const rejected: { reason: string }[] = []; - eventBus.subscribe("lease.rejected", (envelope) => - rejected.push(envelope.payload as { reason: string }), - ); - - const first = await start(coordinator, physicalRequest(), owner("agent-1")); - - expect(first.state.state).toBe("rejected"); - expect(await first.ended).toMatchObject({ code: "NO_CAPACITY" }); - expect(rejected.map((payload) => payload.reason)).toEqual(["no-worker"]); - - clock.advance(24 * 60 * 60_000); - await workers.pruneExpired(); - const second = await start(coordinator, physicalRequest(), owner("agent-2")); - - expect(second.state.state).toBe("rejected"); - expect(await second.ended).toMatchObject({ code: "UNKNOWN_MODEL" }); - expect(rejected.map((payload) => payload.reason)).toEqual(["no-worker", "unresolvable-spec"]); - }); - - it("queued physical requests count on their own line: queue.changed names the line and lease.queued reports the position within it", async () => { - const { coordinator, eventBus } = oneWorker([iphone()], saturatedIos()); - const changed: unknown[] = []; - const queued: unknown[] = []; - eventBus.subscribe("queue.changed", (envelope) => changed.push(envelope.payload)); - eventBus.subscribe("lease.queued", (envelope) => queued.push(envelope.payload)); - - await start(coordinator, physicalRequest(), owner("agent-1")); - await start(coordinator, physicalRequest(), owner("agent-2")); - await start(coordinator, { ...REQUEST, physical: false }, owner("agent-3")); - - expect(changed).toStrictEqual([ - { depth: 1, physical: true }, - { depth: 2, physical: true }, - { depth: 1 }, - ]); - expect(queued.map((payload) => (payload as { queuePosition: number }).queuePosition)).toEqual([ - 1, 2, 1, - ]); - }); - - it("in one dispatch pass, a waiting virtual request and a waiting physical request are both sent to worker 1, the only worker that can take either", async () => { - const { client, clock, coordinator, workers } = oneWorker([iphone()], saturatedIos()); - client.requestLeaseQueue.push( - { error: noCapacityError(), kind: "error" }, - { kind: "hang" }, - { kind: "hang" }, - ); - await start(coordinator, REQUEST, owner("agent-1")); - await start(coordinator, physicalRequest(), owner("agent-2")); - const before = leaseRequests(client).length; - - clock.advance(1_000); - workers.refresh("wrk_a", { capacity: roomierIos() }); - await tick(); - - expect(leaseRequests(client).slice(before).sort()).toEqual([ - `lease.request:${GATEWAY_PREFIX}agent-1`, - `lease.request:${GATEWAY_PREFIX}agent-2`, - ]); - }); - - it("in one dispatch pass, two waiting physical requests for different models, whose ready matches are both on worker 1, are both sent to it", async () => { - const { client, clock, coordinator, workers } = oneWorker( - [iphone("dev_p"), iphone("dev_q", "ready", undefined, "iPhone 16 Pro")], - saturatedIos(), - ); - client.requestLeaseQueue.push( - { error: noCapacityError(), kind: "error" }, - { error: noCapacityError(), kind: "error" }, - { kind: "hang" }, - { kind: "hang" }, - ); - await start(coordinator, physicalRequest({ model: "iPhone 15 Pro" }), owner("agent-1")); - await start(coordinator, physicalRequest({ model: "iPhone 16 Pro" }), owner("agent-2")); - expect(leaseRequests(client)).toHaveLength(2); - - clock.advance(1_000); - workers.refresh("wrk_a", { capacity: roomierIos() }); - await tick(); - - expect(leaseRequests(client).slice(2).sort()).toEqual([ - `lease.request:${GATEWAY_PREFIX}agent-1`, - `lease.request:${GATEWAY_PREFIX}agent-2`, - ]); - }); - - it("in one dispatch pass, of two waiting physical requests with the same platform, model, class and OS, only the older is sent to worker 1, though it has two ready matches; the newer is sent in the next pass", async () => { - const { client, clock, coordinator, workers } = oneWorker( - [iphone("dev_p"), iphone("dev_q")], - saturatedIos(), - ); - client.requestLeaseQueue.push( - { error: noCapacityError(), kind: "error" }, - { error: noCapacityError(), kind: "error" }, - { kind: "hang" }, - { kind: "hang" }, - ); - const request = physicalRequest({ model: "iPhone 15 Pro", osVersion: "18.4" }); - await start(coordinator, request, owner("agent-1")); - await start(coordinator, request, owner("agent-2")); - expect(leaseRequests(client)).toHaveLength(2); - - clock.advance(1_000); - workers.refresh("wrk_a", { capacity: roomierIos() }); - await tick(); - expect(leaseRequests(client).slice(2)).toEqual([`lease.request:${GATEWAY_PREFIX}agent-1`]); - - clock.advance(1_000); - workers.refresh("wrk_a", { - capacity: { - ...statusFixture().capacity, - ios: { ...statusFixture().capacity.ios, maxRunning: 4 }, - }, - }); - await tick(); - - expect(leaseRequests(client).slice(3)).toEqual([`lease.request:${GATEWAY_PREFIX}agent-2`]); - }); -}); From 42757a4a014dc9723853c16411e24cc0b86e7d52 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:22:23 +0100 Subject: [PATCH 29/83] test(core): split config.test.ts by topic --- src/core/config-auth-and-proxy.test.ts | 159 ++ src/core/config-capacity.test.ts | 186 ++ src/core/config-health-downloads-http.test.ts | 197 ++ src/core/config-lease.test.ts | 284 +++ src/core/config-modes.test.ts | 541 +++++ ...onfig-platform-and-driver-settings.test.ts | 262 +++ src/core/config-redact.test.ts | 105 + src/core/config-warm-pool.test.ts | 297 +++ src/core/config.test.ts | 2091 +---------------- src/core/test-config.ts | 32 + 10 files changed, 2096 insertions(+), 2058 deletions(-) create mode 100644 src/core/config-auth-and-proxy.test.ts create mode 100644 src/core/config-capacity.test.ts create mode 100644 src/core/config-health-downloads-http.test.ts create mode 100644 src/core/config-lease.test.ts create mode 100644 src/core/config-modes.test.ts create mode 100644 src/core/config-platform-and-driver-settings.test.ts create mode 100644 src/core/config-redact.test.ts create mode 100644 src/core/config-warm-pool.test.ts create mode 100644 src/core/test-config.ts diff --git a/src/core/config-auth-and-proxy.test.ts b/src/core/config-auth-and-proxy.test.ts new file mode 100644 index 000000000..a0395154b --- /dev/null +++ b/src/core/config-auth-and-proxy.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { type AuthStrategyDefinition } from "../auth/index.js"; +import { MemoryFilesystem } from "../ports/index.js"; +import { type Config, loadConfig } from "./index.js"; +import { dottedValue, configPath, createStats } from "./test-config.js"; + +describe("loadConfig over the auth strategies' config keys", () => { + const fake: AuthStrategyDefinition = { + name: "fake", + config: { + key: "fake", + schema: z.object({ limit: z.number().int().positive() }), + defaults: { limit: 5 }, + }, + enabled: () => true, + create: () => ({ authenticate: async () => ({ kind: "not-mine", sawCredential: false }) }), + }; + + async function load(http: unknown) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http })); + return loadConfig({ + authStrategies: [fake], + configPath, + filesystem, + systemStats: createStats(), + }); + } + + it("fills a registered strategy's defaults and validates the key it owns", async () => { + expect(dottedValue(await load({}), "http.fake.limit")).toBe(5); + expect(dottedValue(await load({ fake: { limit: 9 } }), "http.fake.limit")).toBe(9); + await expect(load({ fake: { limit: "many" } })).rejects.toThrow("http.fake"); + }); +}); + +describe("loadConfig over http.proxyAuth", () => { + const complete = { + header: "cf-access-jwt-assertion", + jwksUrl: "https://team.cloudflareaccess.com/cdn-cgi/access/certs", + issuer: "https://team.cloudflareaccess.com", + audience: "4f1c0e", + }; + + async function load(proxyAuth: unknown) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify(proxyAuth === undefined ? {} : { http: { proxyAuth } }), + ); + return loadConfig({ configPath, filesystem, systemStats: createStats() }); + } + + function proxyAuthOf(config: Config): unknown { + return dottedValue(config, "http.proxyAuth"); + } + + it("starts with proxy sign-in off when http.proxyAuth is not set, and on with the four keys", async () => { + expect(proxyAuthOf(await load(undefined))).toBeUndefined(); + expect(proxyAuthOf(await load(complete))).toEqual(complete); + expect(proxyAuthOf(await load({ ...complete, signOutUrl: "http://localhost:1/out" }))).toEqual({ + ...complete, + signOutUrl: "http://localhost:1/out", + }); + }); + + it.each(["http://127.0.0.1:9000/keys", "http://localhost:9000/keys", "http://[::1]:9000/keys"])( + "accepts the key URL %s over http on the local host", + async (jwksUrl) => { + expect(proxyAuthOf(await load({ ...complete, jwksUrl }))).toEqual({ ...complete, jwksUrl }); + }, + ); + + it("refuses a key URL that is neither https nor http on 127.0.0.1, ::1 or localhost, naming the key", async () => { + await expect(load(complete)).resolves.toBeDefined(); + + for (const jwksUrl of ["http://keys.team.dev/certs", "not a url"]) { + await expect(load({ ...complete, jwksUrl }), jwksUrl).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.jwksUrl": expected an https:// URL, or http:// on 127.0.0.1, ::1 or localhost', + ); + } + }); + + it("refuses an incomplete set, naming the first missing key", async () => { + await expect(load(complete)).resolves.toBeDefined(); + + for (const missing of ["header", "jwksUrl", "issuer", "audience"] as const) { + const rest = Object.fromEntries(Object.entries(complete).filter(([key]) => key !== missing)); + await expect(load(rest), missing).rejects.toThrow( + `Invalid config value for "http.proxyAuth.${missing}": expected set with the other http.proxyAuth keys`, + ); + } + await expect(load({ header: complete.header })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.jwksUrl": expected set with the other http.proxyAuth keys', + ); + await expect(load({ signOutUrl: "https://team.dev/out" })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.header": expected set with the other http.proxyAuth keys', + ); + }); + + it("accepts a header written in any case, and refuses one that is not an HTTP header name", async () => { + expect( + proxyAuthOf(await load({ ...complete, header: "Cf-Access-Jwt-Assertion" })), + ).toMatchObject({ header: "Cf-Access-Jwt-Assertion" }); + + await expect(load({ ...complete, header: "x bad" })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.header": expected an HTTP header name', + ); + }); + + it("bounds the header name to 64 characters, the audience to 512, and refuses an empty issuer or audience", async () => { + await expect(load({ ...complete, header: "h".repeat(64) })).resolves.toBeDefined(); + await expect(load({ ...complete, header: "h".repeat(65) })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.header": expected an HTTP header name', + ); + await expect(load({ ...complete, audience: "a".repeat(512) })).resolves.toBeDefined(); + await expect(load({ ...complete, audience: "a".repeat(513) })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.audience": expected at most 512 characters', + ); + await expect(load({ ...complete, issuer: "" })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.issuer": expected a non-empty string', + ); + await expect(load({ ...complete, audience: "" })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.audience": expected a non-empty string', + ); + }); + + it("refuses a sign-out URL that is not an http or https URL", async () => { + for (const signOutUrl of ["not a url", "ftp://team.dev/out"]) { + await expect(load({ ...complete, signOutUrl }), signOutUrl).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.signOutUrl": expected an http:// or https:// URL', + ); + } + }); + + it("refuses a wrong-type header, jwksUrl or signOutUrl with our message", async () => { + const cases = [ + ["header", "an HTTP header name"], + ["jwksUrl", "an https:// URL, or http:// on 127.0.0.1, ::1 or localhost"], + ["signOutUrl", "an http:// or https:// URL"], + ] as const; + for (const [key, message] of cases) { + await expect(load({ ...complete, [key]: 7 }), key).rejects.toThrow( + `Invalid config value for "http.proxyAuth.${key}": expected ${message}`, + ); + } + }); + + it("accepts an issuer of 128 characters and refuses one of 129", async () => { + await expect(load({ ...complete, issuer: "i".repeat(128) })).resolves.toBeDefined(); + + await expect(load({ ...complete, issuer: "i".repeat(129) })).rejects.toThrow( + 'Invalid config value for "http.proxyAuth.issuer": expected at most 128 characters', + ); + }); +}); diff --git a/src/core/config-capacity.test.ts b/src/core/config-capacity.test.ts new file mode 100644 index 000000000..154f71290 --- /dev/null +++ b/src/core/config-capacity.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { loadConfig } from "./index.js"; +import { resourceOptions, configPath, gibibyte, createStats } from "./test-config.js"; + +describe("loadConfig capacity strategies", () => { + async function load( + contents: unknown, + options: { + readonly overrides?: Parameters[0]["overrides"]; + readonly warn?: (message: string) => void; + } = {}, + ) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + return loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + ...(options.overrides === undefined ? {} : { overrides: options.overrides }), + ...(options.warn === undefined ? {} : { warn: options.warn }), + }); + } + + it("defaults the fixed strategy to a machine-independent pin", async () => { + const config = await load({ capacity: { strategy: "fixed" } }); + + expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 2 } }); + }); + + it("pins concurrency from a single key", async () => { + const config = await load({ capacity: { strategy: "fixed", config: { maxRunning: 4 } } }); + + expect(config.capacity.config).toEqual({ maxRunning: 4 }); + }); + + it("lets an override switch the strategy chosen by the file", async () => { + const config = await load( + { capacity: { strategy: "resource" } }, + { overrides: { capacity: { strategy: "fixed", config: { maxRunning: 6 } } } }, + ); + + expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 6 } }); + }); + + it("starts from the selected strategy's defaults, not the default strategy's", async () => { + const config = await load({ capacity: { strategy: "fixed", config: { maxRunning: 3 } } }); + + expect(config.capacity.config).not.toHaveProperty("ramBudget"); + expect(config.capacity.config).not.toHaveProperty("limits"); + }); + + it("rejects a strategy name with no registered implementation", async () => { + await expect(load({ capacity: { strategy: "vibes" } })).rejects.toThrow("capacity.strategy"); + }); + + it("hands capacity.config to the selected strategy's own validator", async () => { + await expect( + load({ capacity: { strategy: "fixed", config: { maxRunning: 0 } } }), + ).rejects.toThrow("capacity.config.maxRunning"); + }); + + it("warns when capacity.config carries another strategy's keys", async () => { + const warn = vi.fn(); + await load({ capacity: { strategy: "fixed", config: { ramBudget: {} } } }, { warn }); + + expect(warn).toHaveBeenCalledWith('Unknown config key: "capacity.config.ramBudget"'); + }); +}); + +describe("loadConfig legacy capacity keys", () => { + async function load( + contents: unknown, + options: { + readonly overrides?: Parameters[0]["overrides"]; + readonly warn?: (message: string) => void; + } = {}, + ) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + return loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + ...(options.overrides === undefined ? {} : { overrides: options.overrides }), + ...(options.warn === undefined ? {} : { warn: options.warn }), + }); + } + + it("folds top-level limits and ramBudget into the resource strategy's options", async () => { + const warn = vi.fn(); + const config = await load( + { + limits: { ios: { maxDevices: 3, maxRunning: 3 }, maxRunning: 5 }, + ramBudget: { iosBytesPerDevice: 2 * gibibyte }, + }, + { warn }, + ); + + expect(config.capacity.strategy).toBe("resource"); + expect(resourceOptions(config).limits).toMatchObject({ + ios: { maxDevices: 3, maxRunning: 3 }, + maxRunning: 5, + }); + expect(resourceOptions(config).ramBudget.iosBytesPerDevice).toBe(2 * gibibyte); + expect(warn).not.toHaveBeenCalled(); + }); + + it("loads the slim sizes from capacity.config.ramBudget and from the legacy top-level ramBudget, absent by default", async () => { + const fromCapacity = await load({ + capacity: { + strategy: "resource", + config: { + ramBudget: { + androidSlimBytesPerDevice: 2 * gibibyte, + iosSlimBytesPerDevice: 0.75 * gibibyte, + }, + }, + }, + }); + const fromLegacy = await load({ ramBudget: { iosSlimBytesPerDevice: 0.5 * gibibyte } }); + const byDefault = await load({}); + + expect(resourceOptions(fromCapacity).ramBudget).toEqual({ + androidBytesPerDevice: 4 * gibibyte, + androidSlimBytesPerDevice: 2 * gibibyte, + iosBytesPerDevice: 1.5 * gibibyte, + iosSlimBytesPerDevice: 0.75 * gibibyte, + }); + expect(resourceOptions(fromLegacy).ramBudget.iosSlimBytesPerDevice).toBe(0.5 * gibibyte); + expect(resourceOptions(byDefault).ramBudget).not.toHaveProperty("iosSlimBytesPerDevice"); + expect(resourceOptions(byDefault).ramBudget).not.toHaveProperty("androidSlimBytesPerDevice"); + }); + + it.each([ + ["iosSlimBytesPerDevice", -1], + ["iosSlimBytesPerDevice", "1GiB"], + ["androidSlimBytesPerDevice", -1], + ["androidSlimBytesPerDevice", null], + ])("rejects a %s of %j, naming the key", async (key, value) => { + await expect( + load({ capacity: { strategy: "resource", config: { ramBudget: { [key]: value } } } }), + ).rejects.toThrow(`capacity.config.ramBudget.${key}`); + await expect(load({ ramBudget: { [key]: value } })).rejects.toThrow(`ramBudget.${key}`); + }); + + it("prefers capacity.config over the legacy spelling within one layer", async () => { + const config = await load({ + capacity: { strategy: "resource", config: { limits: { maxRunning: 9 } } }, + limits: { maxRunning: 2 }, + }); + + expect(resourceOptions(config).limits.maxRunning).toBe(9); + }); + + it("keeps layer precedence when the layers disagree about spelling", async () => { + const config = await load( + { limits: { maxRunning: 8 } }, + { overrides: { capacity: { strategy: "resource", config: { limits: { maxRunning: 4 } } } } }, + ); + + expect(resourceOptions(config).limits.maxRunning).toBe(4); + }); + + it("lets a legacy override win over a capacity.config file value", async () => { + const config = await load( + { capacity: { strategy: "resource", config: { limits: { maxRunning: 8 } } } }, + { overrides: { limits: { maxRunning: 4 } } }, + ); + + expect(resourceOptions(config).limits.maxRunning).toBe(4); + }); + + it("warns and ignores legacy keys when another strategy is selected", async () => { + const warn = vi.fn(); + const config = await load( + { capacity: { strategy: "fixed", config: { maxRunning: 3 } }, limits: { maxRunning: 8 } }, + { warn }, + ); + + expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 3 } }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("Ignoring limits")); + }); +}); diff --git a/src/core/config-health-downloads-http.test.ts b/src/core/config-health-downloads-http.test.ts new file mode 100644 index 000000000..ee287157b --- /dev/null +++ b/src/core/config-health-downloads-http.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { loadConfig } from "./index.js"; +import { configPath, createStats } from "./test-config.js"; + +describe("loadConfig", () => { + it("applies a file-level health override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ + health: { + enabled: false, + probeIntervalMs: 60_000, + stableObservations: 3, + maxRecoveryAttempts: 5, + recoveryBackoffMs: 10_000, + maxConcurrentRecoveries: 2, + }, + }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.health).toEqual({ + enabled: false, + probeIntervalMs: 60_000, + stableObservations: 3, + maxRecoveryAttempts: 5, + recoveryBackoffMs: 10_000, + maxConcurrentRecoveries: 2, + }); + }); + + it("rejects a non-boolean health.enabled", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ health: { enabled: "yes" } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("health.enabled"); + }); + + it.each([ + [{ health: { probeIntervalMs: 0 } }, "health.probeIntervalMs"], + [{ health: { recoveryBackoffMs: -1 } }, "health.recoveryBackoffMs"], + [{ health: { stableObservations: 0 } }, "health.stableObservations"], + [{ health: { stableObservations: 1.5 } }, "health.stableObservations"], + [{ health: { maxRecoveryAttempts: 0 } }, "health.maxRecoveryAttempts"], + [{ health: { maxConcurrentRecoveries: 0 } }, "health.maxConcurrentRecoveries"], + ])("rejects invalid health values in every field", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("applies a file-level downloads override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ + downloads: { policy: "always", acceptAndroidLicenses: true, timeoutMs: 60_000 }, + }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.downloads).toEqual({ + policy: "always", + acceptAndroidLicenses: true, + timeoutMs: 60_000, + }); + }); + + it("applies an override-level downloads policy over the file value", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ downloads: { policy: "never" } }), + ); + + const config = await loadConfig({ + configPath, + filesystem, + overrides: { downloads: { policy: "always" } }, + systemStats: createStats(), + }); + expect(config.downloads.policy).toBe("always"); + }); + + it("rejects a downloads.policy outside the known set", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ downloads: { policy: "sometimes" } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("downloads.policy"); + }); + + it("rejects a non-boolean downloads.acceptAndroidLicenses", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ downloads: { acceptAndroidLicenses: "yes" } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("downloads.acceptAndroidLicenses"); + }); + + it.each([ + [{ downloads: { timeoutMs: 0 } }, "downloads.timeoutMs"], + [{ downloads: { timeoutMs: -1 } }, "downloads.timeoutMs"], + [{ downloads: { timeoutMs: "1200000" } }, "downloads.timeoutMs"], + ])("rejects a non-positive or malformed downloads.timeoutMs", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("applies a file-level http override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ http: { enabled: true, host: "0.0.0.0", port: 8080 } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.http).toEqual({ enabled: true, host: "0.0.0.0", port: 8080 }); + }); + + it("applies an override-level http port over the file value", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { port: 5000 } })); + + const config = await loadConfig({ + configPath, + filesystem, + overrides: { http: { port: 6000 } }, + systemStats: createStats(), + }); + expect(config.http.port).toBe(6000); + }); + + it("rejects a non-boolean http.enabled", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { enabled: "yes" } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("http.enabled"); + }); + + it("rejects a non-string http.host", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { host: 127 } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("http.host"); + }); + + it.each([ + [{ http: { port: 0 } }, "http.port"], + [{ http: { port: 65536 } }, "http.port"], + [{ http: { port: 1.5 } }, "http.port"], + [{ http: { port: "4700" } }, "http.port"], + ])("rejects an out-of-range or malformed http.port", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); +}); diff --git a/src/core/config-lease.test.ts b/src/core/config-lease.test.ts new file mode 100644 index 000000000..856bafce2 --- /dev/null +++ b/src/core/config-lease.test.ts @@ -0,0 +1,284 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { loadConfig } from "./index.js"; +import { configPath, createStats } from "./test-config.js"; + +describe("loadConfig", () => { + it("accepts a lease TTL pair at the boundary, where the default equals the cap", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { defaultTtlMs: 40_000, maxTtlMs: 40_000 } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.lease).toMatchObject({ defaultTtlMs: 40_000, maxTtlMs: 40_000 }); + }); + + it.each([ + [{ lease: { identity: { ios: "disposable" } } }, "lease.identity.ios"], + [{ lease: { identity: { android: true } } }, "lease.identity.android"], + ])( + "rejects a lease.identity value other than reusable or fresh, naming the key (%#)", + async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }, + ); + + it("reads lease.requestRetentionMs and lease.maxRequestRecords", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { maxRequestRecords: 50, requestRetentionMs: 30_000 } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(config.lease).toMatchObject({ maxRequestRecords: 50, requestRetentionMs: 30_000 }); + }); + + it.each([ + [{ lease: { requestRetentionMs: 0 } }, "lease.requestRetentionMs"], + [{ lease: { maxRequestRecords: 1.5 } }, "lease.maxRequestRecords"], + ])("rejects an invalid lease-request limit, naming the key (%#)", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("an operator's lease.endedIdRetentionMs is checked at load: unset is 24 h, a positive number is read, and 0, a negative number or text are refused", async () => { + const load = async (written: unknown) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + if (written !== undefined) { + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { endedIdRetentionMs: written } }), + ); + } + return loadConfig({ configPath, filesystem, systemStats: createStats() }); + }; + + await expect(load(undefined)).resolves.toMatchObject({ + lease: { endedIdRetentionMs: 86_400_000 }, + }); + await expect(load(5_000)).resolves.toMatchObject({ lease: { endedIdRetentionMs: 5_000 } }); + for (const refused of [0, -1, "1d"]) { + await expect(load(refused), String(refused)).rejects.toThrow("lease.endedIdRetentionMs"); + } + }); + + it.each([ + [undefined, 1], + [2, 2], + [ + { ios: 1, android: 1 }, + { ios: 1, android: 1 }, + ], + [{ ios: 2 }, { ios: 2 }], + [{}, {}], + ])("reads an operator's lease.maxPerRequester %j as %j (absent: 1)", async (written, read) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + if (written !== undefined) { + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { maxPerRequester: written } }), + ); + } + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(config.lease.maxPerRequester).toEqual(read); + }); + + it.each([ + [0, "lease.maxPerRequester"], + [-1, "lease.maxPerRequester"], + [1.5, "lease.maxPerRequester"], + ["2", "lease.maxPerRequester"], + [{ ios: 0 }, "lease.maxPerRequester.ios"], + [{ ios: 1.5 }, "lease.maxPerRequester.ios"], + [{ ios: "1" }, "lease.maxPerRequester.ios"], + ])("rejects lease.maxPerRequester %j, naming %s", async (value, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { maxPerRequester: value } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("accepts a platform lease.maxPerRequester does not know, with a warning, and ignores it", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { maxPerRequester: { ios: 1, tvos: 1 } } }), + ); + const warn = vi.fn(); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats(), warn }); + + expect(config.lease.maxPerRequester).toEqual({ ios: 1 }); + expect(warn).toHaveBeenCalledWith('Unknown config key: "lease.maxPerRequester.tvos"'); + }); + + it("lease.endWaitMs defaults to 600000 and accepts a positive number and 2147483647", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + const unset = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: 5_000 } })); + const small = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { endWaitMs: 2_147_483_647 } }), + ); + const largest = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(unset.lease.endWaitMs).toBe(600_000); + expect(small.lease.endWaitMs).toBe(5_000); + expect(largest.lease.endWaitMs).toBe(2_147_483_647); + }); + + it.each([[0], [-1], [2_147_483_648], ["600000"]])( + "refuses lease.endWaitMs %j with a message naming the key, where 5000 loads", + async (value) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: 5_000 } })); + const accepted = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(accepted.lease.endWaitMs).toBe(5_000); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: value } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("lease.endWaitMs"); + }, + ); + + it("reads lease.identity per platform and leaves the unset platform reusable", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { identity: { ios: "fresh" } } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(config.lease.identity).toEqual({ android: "reusable", ios: "fresh" }); + }); + + it.each([ + [{ lease: { defaultTtlMs: 0 } }, "lease.defaultTtlMs"], + [{ lease: { maxTtlMs: -1 } }, "lease.maxTtlMs"], + [{ lease: { defaultTtlMs: "soon" } }, "lease.defaultTtlMs"], + ])("rejects a non-positive lease TTL, naming the offending key (%#)", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("rejects a defaultTtlMs above maxTtlMs, naming the offending key (ADR 0004)", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { defaultTtlMs: 40_001, maxTtlMs: 40_000 } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("lease.defaultTtlMs"); + }); + + it("catches the pair rule across layers, not just within one file", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ lease: { defaultTtlMs: 60_000 } }), + ); + + // The file's default is fine against the *default* cap; only the merged config is wrong. + await expect( + loadConfig({ + configPath, + filesystem, + overrides: { lease: { maxTtlMs: 30_000 } }, + systemStats: createStats(), + }), + ).rejects.toThrow("lease.defaultTtlMs"); + }); + + it.each([["detachedTtlMs"], ["heldTtlBackstopMs"], ["heartbeatIntervalMs"]])( + "warns about the retired lease.%s and ignores it, carrying no value over (ADR 0004)", + async (retired) => { + const filesystem = new MemoryFilesystem(); + const warn = vi.fn(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { [retired]: 1 } })); + + const config = await loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + warn, + }); + + expect(warn).toHaveBeenCalledWith(`Unknown config key: "lease.${retired}"`); + // Not aliased onto anything: the new keys keep their own defaults. + expect(config.lease).toEqual({ + defaultTtlMs: 15 * 60_000, + identity: { android: "reusable", ios: "reusable" }, + maxPerRequester: 1, + endWaitMs: 600_000, + maxRequestRecords: 10_000, + maxTtlMs: 4 * 60 * 60_000, + requestRetentionMs: 10 * 60_000, + endedIdRetentionMs: 24 * 60 * 60_000, + }); + }, + ); + + it.each([ + [{ limits: { web: {} } }, "limits.web"], + [{ health: { maxBoltCount: 7 } }, "health.maxBoltCount"], + [{ ios: { turboMode: true } }, "ios.turboMode"], + ])( + "warns about an unknown key, nested or not, without rejecting the file: %j", + async (contents, key) => { + const filesystem = new MemoryFilesystem(); + const warn = vi.fn(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats(), warn }), + ).resolves.toBeDefined(); + expect(warn).toHaveBeenCalledWith(`Unknown config key: "${key}"`); + }, + ); +}); diff --git a/src/core/config-modes.test.ts b/src/core/config-modes.test.ts new file mode 100644 index 000000000..b762d8ea2 --- /dev/null +++ b/src/core/config-modes.test.ts @@ -0,0 +1,541 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { loadConfig } from "./index.js"; +import { configPath, createStats } from "./test-config.js"; + +/** + * ADR 0005 §1/§2: `mode` selects what the daemon is, and a gateway reads only `mode`, `http.*`, + * `log.*`, `lease.*`, `eventBuffer.*` and `gateway.*`. + */ +describe("loadConfig modes (ADR 0005)", () => { + async function load( + contents: unknown, + options: { + readonly overrides?: Parameters[0]["overrides"]; + readonly role?: "worker" | "gateway"; + readonly warn?: (message: string) => void; + } = {}, + ) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + return loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + ...(options.role === undefined ? {} : { role: options.role }), + ...(options.overrides === undefined ? {} : { overrides: options.overrides }), + ...(options.warn === undefined ? {} : { warn: options.warn }), + }); + } + + it("defaults to worker mode, with HTTP on and a 24h disconnected-view retention", async () => { + const config = await load({}); + + expect(config.mode).toBe("worker"); + expect(config.http.enabled).toBe(true); + expect(config.gateway).toEqual({ + disconnectedRetentionMs: 24 * 60 * 60_000, + execTimeoutMs: 11 * 60_000, + leaseRequestTimeoutMs: 5 * 60_000, + routing: "warm-then-free", + alerts: { destinations: [] }, + admission: { timeoutMs: 5_000 }, + }); + }); + + it("an operator's gateway.admission.command and gateway.admission.timeoutMs are checked at load: an argv list is read, an empty list, an empty string or text is refused, the timeout defaults to 5000, and 0 or over 60000 is refused", async () => { + const admission = (written: unknown) => load({ gateway: { admission: written } }); + const argument = (length: number) => "a".repeat(length); + + const read = await admission({ command: ["bun", "/etc/simlock/policy.ts"], timeoutMs: 60_000 }); + expect(read.gateway.admission).toEqual({ + command: ["bun", "/etc/simlock/policy.ts"], + timeoutMs: 60_000, + }); + expect((await admission({ command: [argument(4096)] })).gateway.admission?.timeoutMs).toBe( + 5_000, + ); + expect((await load({})).gateway.admission).toEqual({ timeoutMs: 5_000 }); + + const refused = [ + [{ command: [] }, "gateway.admission.command"], + [{ command: ["bun", ""] }, "gateway.admission.command"], + [{ command: "bun policy.ts" }, "gateway.admission.command"], + [{ command: [argument(4097)] }, "gateway.admission.command"], + [{ command: Array.from({ length: 65 }, () => "a") }, "gateway.admission.command"], + [{ timeoutMs: 0 }, "gateway.admission.timeoutMs"], + [{ timeoutMs: 60_001 }, "gateway.admission.timeoutMs"], + [{ timeoutMs: 1.5 }, "gateway.admission.timeoutMs"], + ] as const; + for (const [written, key] of refused) { + await expect(admission(written), JSON.stringify(written)).rejects.toThrow(key); + } + }); + + it.each([ + ["file worker", { contents: { mode: "worker" } }], + ["file gateway", { contents: { mode: "gateway" } }], + ["override gateway", { contents: {}, overrides: { mode: "gateway" } }], + ] as const)( + "refuses a config that sets mode, either value, from the file or the overrides, with the message that names --gateway (%s)", + async (_name, { contents, ...options }) => { + await expect(load(contents, options)).rejects.toThrow( + '"mode" is no longer a config key. A daemon started with --gateway is a gateway; any other is a worker. Remove "mode" from config.json.', + ); + }, + ); + + it("turns HTTP on by default in gateway mode -- a gateway is the fleet's contact point", async () => { + const config = await load({}, { role: "gateway" }); + + expect(config.http.enabled).toBe(true); + expect(config.http.port).toBe(4700); + }); + + it("refuses a gateway that switches HTTP off, naming the key, and loads a worker that does", async () => { + await expect(load({ http: { enabled: false } }, { role: "gateway" })).rejects.toThrow( + "http.enabled", + ); + // HTTP is genuinely optional for a worker. + await expect(load({ http: { enabled: false } })).resolves.toMatchObject({ + http: { enabled: false }, + mode: "worker", + }); + }); + + it("keeps an explicit gateway http host/port", async () => { + const config = await load({ http: { host: "0.0.0.0", port: 4800 } }, { role: "gateway" }); + + expect(config.http).toEqual({ enabled: true, host: "0.0.0.0", port: 4800 }); + }); + + it("warns about worker-only keys in a gateway's config and ignores them", async () => { + const warn = vi.fn(); + const config = await load( + { + capacity: { strategy: "fixed", config: { maxRunning: 3 } }, + ios: { defaultMode: "slim" }, + lease: { defaultTtlMs: 60_000 }, + }, + { role: "gateway", warn }, + ); + + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "capacity"')); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "ios"')); + // Ignored, not rejected: the file still loads, and the gateway-readable keys still apply. + expect(config.lease.defaultTtlMs).toBe(60_000); + // "Ignored" means "nothing in gateway mode reads it" -- the value is still parsed and + // merged, so a config file can be shared between a worker and a gateway with only `mode` + // differing. The warning is what makes that visible rather than silent. + expect(warn).not.toHaveBeenCalledWith(expect.stringContaining('Ignoring "lease"')); + }); + + it("reads lease.maxPerRequester in gateway mode without an ignored-key warning", async () => { + const warn = vi.fn(); + + const config = await load({ lease: { maxPerRequester: 2 } }, { role: "gateway", warn }); + + expect(config.lease).toMatchObject({ maxPerRequester: 2 }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reads metrics.retention in a gateway config without a warning", async () => { + const warn = vi.fn(); + + const config = await load( + { metrics: { retention: 120 * 24 * 3_600_000 } }, + { role: "gateway", warn }, + ); + + expect(config.metrics).toEqual({ retention: 120 * 24 * 3_600_000 }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reads update.check in a gateway config without a warning", async () => { + const warn = vi.fn(); + + const config = await load({ update: { check: false } }, { role: "gateway", warn }); + + expect(config.update).toEqual({ check: false }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("raises the unknown-key warning for eventLog.rotateBytes in a gateway config", async () => { + const warn = vi.fn(); + await load({ eventLog: { rotateBytes: 1024 } }, { role: "gateway", warn }); + + expect(warn).toHaveBeenCalledWith('Unknown config key: "eventLog.rotateBytes"'); + }); + + it("warns about android.emulator on a gateway without refusing the config", async () => { + const warn = vi.fn(); + const config = await load( + { android: { defaultModels: {}, emulator: { headless: true } } }, + { role: "gateway", warn }, + ); + + expect(warn).toHaveBeenCalledWith( + 'Ignoring "android": it configures a worker, and this daemon runs in gateway mode.', + ); + // Ignored, not rejected: the gateway still starts. + expect(config.mode).toBe("gateway"); + }); + + it("warns about the worker-side gateway.* keys on a gateway, but not the gateway's own", async () => { + const warn = vi.fn(); + const config = await load( + { + gateway: { + url: "ws://127.0.0.1:4700/v1/uplink", + token: "secret", + label: "ignored", + disconnectedRetentionMs: 60_000, + }, + }, + { role: "gateway" }, + ); + + expect(config.gateway.disconnectedRetentionMs).toBe(60_000); + + const warned = await load( + { gateway: { url: "ws://127.0.0.1:4700/v1/uplink" } }, + { role: "gateway", warn }, + ); + expect(warned.mode).toBe("gateway"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "gateway.url"')); + expect(warn).not.toHaveBeenCalledWith( + expect.stringContaining('Ignoring "gateway.disconnectedRetentionMs"'), + ); + + const withHeaders = vi.fn(); + await load({ gateway: { headers: { "x-a": "1" } } }, { role: "gateway", warn: withHeaders }); + expect(withHeaders.mock.calls.map(([message]) => message)).toEqual([ + expect.stringContaining('Ignoring "gateway.headers"'), + ]); + const withoutHeaders = vi.fn(); + await load( + { gateway: { disconnectedRetentionMs: 60_000 } }, + { role: "gateway", warn: withoutHeaders }, + ); + expect(withoutHeaders).not.toHaveBeenCalled(); + }); + + it("says nothing about worker keys on a worker, which is every daemon today", async () => { + const warn = vi.fn(); + await load({ capacity: { strategy: "fixed", config: { maxRunning: 3 } } }, { warn }); + + expect(warn).not.toHaveBeenCalled(); + }); + + it("keeps a worker's gateway.url/token/label", async () => { + const config = await load({ + gateway: { url: "wss://fleet.example", token: "join-secret", label: "mac-1" }, + }); + + expect(config.gateway.url).toBe("wss://fleet.example"); + expect(config.gateway.token).toBe("join-secret"); + expect(config.gateway.label).toBe("mac-1"); + }); + + it("refuses a worker that names a gateway but no join token, and the reverse", async () => { + // Either alone is an operator half-finishing a join, and a worker that silently never + // joined looks exactly like one whose gateway is down -- so this fails the start instead. + await expect(load({ gateway: { url: "ws://fleet.example" } })).rejects.toThrow("gateway.token"); + await expect(load({ gateway: { token: "join-secret" } })).rejects.toThrow("gateway.url"); + }); + + it("stops the load on a bad gateway.headers with the message naming it, and keeps every accepted row", async () => { + const key = (what: string) => `Invalid config value for "gateway.headers": expected ${what}`; + const shape = "an object of header names to text values"; + const joined = { url: "wss://fleet.example", token: "join-secret" }; + const many = Object.fromEntries(Array.from({ length: 17 }, (_, i) => [`x-h${i}`, "v"])); + const fat = Object.fromEntries( + Array.from({ length: 3 }, (_, i) => [`x-h${i}`, "a".repeat(3000)]), + ); + const refused: readonly (readonly [string, unknown, string])[] = [ + ["an array", ["a"], shape], + ["a non-text value", { "x-ok": 1 }, shape], + ["17 entries", many, "at most 16 headers"], + [ + "a name with a space", + { "X Bad": "v" }, + "a header name of 1 to 100 letters, digits and HTTP name symbols (got X Bad)", + ], + [ + "an empty name", + { "": "x" }, + "a header name of 1 to 100 letters, digits and HTTP name symbols (got )", + ], + [ + "a 101-character name", + { ["a".repeat(101)]: "v" }, + `a header name of 1 to 100 letters, digits and HTTP name symbols (got ${"a".repeat(101)})`, + ], + ["one name twice", { "X-A": "1", "x-a": "2" }, "each header name once (got x-a twice)"], + [ + "a line break", + { "x-ok": "line\nbreak" }, + "a header value of at most 4096 plain ASCII characters (in x-ok)", + ], + [ + "a non-ASCII value", + { "x-ok": "café" }, + "a header value of at most 4096 plain ASCII characters (in x-ok)", + ], + [ + "a 4097-character value", + { "x-ok": "a".repeat(4097) }, + "a header value of at most 4096 plain ASCII characters (in x-ok)", + ], + ["over 8192 in all", fat, "at most 8192 characters of header names and values in all"], + [ + "8193 in all", + { "x-a": "a".repeat(4096), "x-b": "b".repeat(4091) }, + "at most 8192 characters of header names and values in all", + ], + ...[ + "authorization", + "x-simlock-worker-id", + "x-simlock-worker-label", + "host", + "connection", + "upgrade", + "origin", + ].map((name): readonly [string, unknown, string] => [ + `${name} in capitals`, + { [name.toUpperCase()]: "x" }, + `no header Simlock sets itself (${name.toUpperCase()})`, + ]), + ["Authorization", { Authorization: "x" }, "no header Simlock sets itself (Authorization)"], + [ + "a sec-websocket- name", + { "Sec-WebSocket-Key": "x" }, + "no header Simlock sets itself (Sec-WebSocket-Key)", + ], + ]; + for (const role of ["worker", "gateway"] as const) { + for (const [name, headers, what] of refused) { + await expect( + load({ gateway: { ...joined, headers } }, { role }), + `${role}: ${name}`, + ).rejects.toThrow(key(what)); + } + } + // why: the edge itself is accepted, so moving a cap by one fails a named row. + const sixteen = Object.fromEntries(Array.from({ length: 16 }, (_, i) => [`x-h${i}`, "v"])); + const longestName = { ["a".repeat(100)]: "v" }; + for (const headers of [sixteen, longestName]) { + await expect(load({ gateway: { ...joined, headers } })).resolves.toBeDefined(); + } + await expect(load({ gateway: { headers: { "x-a": "1" } } })).rejects.toThrow( + key("gateway.url to be set too, since gateway.headers is set"), + ); + await expect(load({ gateway: { token: "t", headers: { "x-a": "1" } } })).rejects.toThrow( + 'Invalid config value for "gateway.url": expected a gateway URL, since gateway.token is set', + ); + + const exactly8192 = { "x-a": "a".repeat(4096), "x-b": "b".repeat(4090) }; + expect((await load({ gateway: { ...joined, headers: exactly8192 } })).gateway.headers).toEqual( + exactly8192, + ); + const accepted = { "CF-Access-Client-Id": "abc.access", "x-empty": "", "x-tab": "a\tb" }; + expect((await load({ gateway: { ...joined, headers: accepted } })).gateway.headers).toEqual( + accepted, + ); + expect((await load({ gateway: { headers: {} } })).gateway.url).toBeUndefined(); + }); + + it("refuses a joined worker's own HTTP listener with the exact error, and keeps every valid and invalid neighbour as it was", async () => { + const refusal = + 'Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)'; + const joined = { url: "wss://fleet.example", token: "join-secret" }; + type Case = { + readonly name: string; + readonly contents: Record; + readonly overrides?: Parameters[0]["overrides"]; + readonly role?: "worker" | "gateway"; + readonly error?: string; + readonly http?: boolean; + }; + const cases: readonly Case[] = [ + { name: "file only", contents: { gateway: joined, http: { enabled: true } }, error: refusal }, + { + name: "overrides only", + contents: {}, + overrides: { gateway: joined, http: { enabled: true } }, + error: refusal, + }, + { + name: "membership in file, HTTP in overrides", + contents: { gateway: joined }, + overrides: { http: { enabled: true } }, + error: refusal, + }, + { + name: "HTTP in file, membership in overrides", + contents: { http: { enabled: true } }, + overrides: { gateway: joined }, + error: refusal, + }, + { + name: "override false repairs file true", + contents: { gateway: joined, http: { enabled: true } }, + overrides: { http: { enabled: false } }, + http: false, + }, + { + name: "ws URL, HTTP false", + contents: { gateway: { ...joined, url: "ws://fleet.example/x" }, http: { enabled: false } }, + http: false, + }, + { name: "wss URL, HTTP omitted", contents: { gateway: joined }, http: false }, + { name: "standalone, HTTP true", contents: { http: { enabled: true } }, http: true }, + { name: "standalone, HTTP false", contents: { http: { enabled: false } }, http: false }, + { name: "standalone, HTTP omitted", contents: {}, http: true }, + { name: "gateway, HTTP omitted", contents: {}, role: "gateway", http: true }, + { + name: "gateway, HTTP true", + contents: { http: { enabled: true } }, + role: "gateway", + http: true, + }, + { + name: "gateway, HTTP false", + contents: { http: { enabled: false } }, + role: "gateway", + error: "http.enabled", + }, + { + name: "gateway ignoring membership keys", + contents: { gateway: joined, http: { enabled: true } }, + role: "gateway", + http: true, + }, + { + name: "empty URL", + contents: { gateway: { url: "", token: "t" }, http: { enabled: true } }, + error: "a ws:// or wss:// URL", + }, + { + name: "malformed URL", + contents: { gateway: { url: "not a url", token: "t" }, http: { enabled: true } }, + error: "a ws:// or wss:// URL", + }, + { + name: "http URL", + contents: { gateway: { url: "http://fleet.example", token: "t" }, http: { enabled: true } }, + error: "a ws:// or wss:// URL", + }, + { + name: "https URL", + contents: { + gateway: { url: "https://fleet.example", token: "t" }, + http: { enabled: true }, + }, + error: "a ws:// or wss:// URL", + }, + { + name: "URL without token", + contents: { gateway: { url: "wss://fleet.example" }, http: { enabled: true } }, + error: "gateway.token", + }, + { + name: "token without URL", + contents: { gateway: { token: "t" }, http: { enabled: true } }, + error: "a gateway URL, since gateway.token is set", + }, + { + name: "removed mode key", + contents: { mode: "worker", gateway: joined, http: { enabled: true } }, + error: '"mode" is no longer a config key', + }, + ]; + + for (const { name, contents, overrides, role, error, http } of cases) { + const outcome = load(contents, { + ...(overrides === undefined ? {} : { overrides }), + ...(role === undefined ? {} : { role }), + }); + if (error === undefined) { + await expect(outcome, name).resolves.toMatchObject({ http: { enabled: http } }); + } else if (error === refusal) { + await expect(outcome, name).rejects.toMatchObject({ message: refusal }); + } else { + await expect(outcome, name).rejects.toThrow(error); + } + } + }); + + it.each(["http://fleet.example/v1/uplink", "fleet.example", "not a url"])( + "rejects a gateway.url that is not a WebSocket URL: %s", + async (url) => { + await expect(load({ gateway: { url } })).rejects.toThrow("a ws:// or wss:// URL"); + }, + ); + + it("rejects a non-positive disconnected retention", async () => { + await expect(load({ gateway: { disconnectedRetentionMs: 0 } })).rejects.toThrow( + "gateway.disconnectedRetentionMs", + ); + }); + + it("rejects a non-positive lease-request timeout", async () => { + // why: P2 (round 2 review): bounds the one forwarded uplink call that used to have none of its + // own -- see FleetForwarding.withLeaseRequestTimeout. + await expect(load({ gateway: { leaseRequestTimeoutMs: 0 } })).rejects.toThrow( + "gateway.leaseRequestTimeoutMs", + ); + }); + + it("defaults the gateway lease-request timeout well under the exec backstop", async () => { + const config = await load({}, { role: "gateway" }); + expect(config.gateway.leaseRequestTimeoutMs).toBeLessThan(config.gateway.execTimeoutMs); + }); + + it("reports the role it was loaded for as config.mode", async () => { + expect((await load({}, { role: "gateway" })).mode).toBe("gateway"); + expect((await load({}, { role: "worker" })).mode).toBe("worker"); + }); + + it("defaults the memoryPressure section, refuses a bad value naming its key, and warns a gateway it is worker-only", async () => { + expect((await load({})).memoryPressure).toEqual({ + clearAfterMs: 60_000, + enabled: true, + pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, + pauseWarming: { level: "warn", minAvailablePercent: 20 }, + }); + const accepted = await load({ + memoryPressure: { pauseWarming: { level: "off", minAvailablePercent: 0 }, clearAfterMs: 0 }, + }); + expect(accepted.memoryPressure).toMatchObject({ + clearAfterMs: 0, + pauseWarming: { level: "off", minAvailablePercent: 0 }, + }); + + const refused = [ + { memoryPressure: { pauseWarming: { level: "bogus" } } }, + { memoryPressure: { pauseNewDevices: { minAvailablePercent: 101 } } }, + { memoryPressure: { pauseNewDevices: { minAvailablePercent: -1 } } }, + { memoryPressure: { clearAfterMs: -1 } }, + ]; + const keys = [ + "memoryPressure.pauseWarming.level", + "memoryPressure.pauseNewDevices.minAvailablePercent", + "memoryPressure.pauseNewDevices.minAvailablePercent", + "memoryPressure.clearAfterMs", + ]; + for (const [index, file] of refused.entries()) { + await expect(load(file)).rejects.toThrow(keys[index]!); + } + + const warn = vi.fn(); + await load({ memoryPressure: { enabled: false } }, { role: "gateway", warn }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "memoryPressure"')); + }); + + it("HTTP defaults to port 4700 for a gateway and 4701 for a worker", async () => { + expect((await load({}, { role: "gateway" })).http.port).toBe(4700); + expect((await load({})).http.port).toBe(4701); + }); +}); diff --git a/src/core/config-platform-and-driver-settings.test.ts b/src/core/config-platform-and-driver-settings.test.ts new file mode 100644 index 000000000..c2c1950ed --- /dev/null +++ b/src/core/config-platform-and-driver-settings.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { booleanValue, driverConfig, type DriverConfigEntry, loadConfig } from "./index.js"; +import { dottedValue, configPath, createStats } from "./test-config.js"; + +describe("loadConfig", () => { + it("defaults ios.defaultMode to full", async () => { + const config = await loadConfig({ + configPath, + filesystem: new MemoryFilesystem(), + systemStats: createStats(), + }); + + expect(config.ios.defaultMode).toBe("full"); + }); + + it("applies a file-level ios.defaultMode override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultMode: "slim" } })); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.ios.defaultMode).toBe("slim"); + }); + + it("a driver's config entry adds its keys to its platform's section: defaulted, taken from the file, refused by its validator, and unknown to a loader without the entry", async () => { + const entry: DriverConfigEntry<{ readonly turbo: boolean }> = { + platform: "android", + defaults: { turbo: false }, + validators: { turbo: booleanValue }, + }; + const load = async (file: unknown, driverSections = [entry]) => { + const filesystem = new MemoryFilesystem(); + const warn = vi.fn(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(file)); + const config = await loadConfig({ + configPath, + driverSections, + filesystem, + systemStats: createStats(), + warn, + }); + return { config, section: driverConfig(config, entry), warn }; + }; + + const defaulted = await load({}); + const set = await load({ android: { turbo: true } }); + const unknown = await load({ android: { turbo: true } }, []); + + expect(defaulted.section.turbo).toBe(false); + expect([set.section.turbo, set.warn.mock.calls]).toEqual([true, []]); + await expect(load({ android: { turbo: "yes" } })).rejects.toThrow( + 'Invalid config value for "android.turbo"', + ); + expect(unknown.warn).toHaveBeenCalledWith('Unknown config key: "android.turbo"'); + expect(unknown.config.android).toEqual({ defaultModels: {} }); + expect(set.config.ios).toEqual({ defaultMode: "full", defaultModels: {} }); + }); + + it.each(["fast", "worker", true])( + "rejects an ios.defaultMode of %j, naming the key", + async (value) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultMode: value } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("ios.defaultMode"); + }, + ); + + it.each(["ios", "android"] as const)( + "%s.defaultModels defaults to no class having a list", + async (platform) => { + const config = await loadConfig({ + configPath, + filesystem: new MemoryFilesystem(), + systemStats: createStats(), + }); + + expect(config[platform].defaultModels).toEqual({}); + }, + ); + + it.each([ + ["ios", "phone", "iPhone 15"], + ["ios", "tablet", ["iPad (A16)", "iPad (10th generation)"]], + ["android", "phone", "Pixel 7"], + ["android", "tv", ["Television (4K)"]], + ] as const)( + "%s.defaultModels.%s accepts one model name or a list of them", + async (platform, deviceClass, value) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ [platform]: { defaultModels: { [deviceClass]: value } } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(config[platform].defaultModels).toStrictEqual({ + [deviceClass]: typeof value === "string" ? [value] : value, + }); + }, + ); + + it.each([ + ["ios", { phone: "" }, "ios.defaultModels.phone"], + ["ios", { phone: [] }, "ios.defaultModels.phone"], + ["ios", { phone: ["iPhone 15", ""] }, "ios.defaultModels.phone"], + ["ios", { phone: 15 }, "ios.defaultModels.phone"], + ["ios", { phone: [15] }, "ios.defaultModels.phone"], + ["ios", { phone: [["iPhone 15"]] }, "ios.defaultModels.phone"], + ["ios", { mobile: "iPhone 15" }, "ios.defaultModels.mobile"], + ["ios", { constructor: "iPhone 15" }, "ios.defaultModels.constructor"], + ["android", { phone: [] }, "android.defaultModels.phone"], + ["android", { fridge: "Pixel 8" }, "android.defaultModels.fridge"], + ])("refuses %s.defaultModels %j, naming the key", async (platform, defaultModels, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ [platform]: { defaultModels } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(`Invalid config value for "${path}"`); + }); + + it("says what ios.defaultModels expects: a device class for a key, a name or a list of names for a value", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + const load = async (defaultModels: unknown) => { + await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultModels } })); + return loadConfig({ configPath, filesystem, systemStats: createStats() }); + }; + + await expect(load({ mobile: "iPhone 15" })).rejects.toThrow( + "expected a device class (phone, tablet, watch, tv, vision, auto, desktop)", + ); + await expect(load({ phone: [] })).rejects.toThrow( + "expected a model name or a non-empty list of model names", + ); + }); + + it("applies a file-level stalledTransition override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ + stalledTransition: { thresholdMultiplier: 5, minimumThresholdMs: 120_000 }, + }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.stalledTransition).toEqual({ + thresholdMultiplier: 5, + minimumThresholdMs: 120_000, + }); + }); + + it.each([ + [{ stalledTransition: { thresholdMultiplier: 0.5 } }, "stalledTransition.thresholdMultiplier"], + [{ stalledTransition: { minimumThresholdMs: -1 } }, "stalledTransition.minimumThresholdMs"], + ])("rejects invalid stalledTransition values in every field", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + + it("defaults to no driver settings at all", async () => { + const config = await loadConfig({ + configPath, + filesystem: new MemoryFilesystem(), + systemStats: createStats(), + }); + + expect(config.drivers).toEqual({}); + }); + + it("keeps driver settings verbatim, including keys it has never heard of", async () => { + const filesystem = new MemoryFilesystem(); + const warn = vi.fn(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ + drivers: { + ios: { deviceRoot: "/Volumes/scratch/simlock-ios" }, + android: { adbServerPort: 5038, headless: true, somethingOnlyTheDriverKnows: "yes" }, + }, + }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats(), warn }); + + expect(config.drivers).toEqual({ + ios: { deviceRoot: "/Volumes/scratch/simlock-ios" }, + android: { adbServerPort: 5038, headless: true, somethingOnlyTheDriverKnows: "yes" }, + }); + // Warning about an unrecognised key here would mean the core knows which keys a + // driver has, which is the whole thing this block is not allowed to know. + expect(warn).not.toHaveBeenCalled(); + }); + + it("reads a driver setting at the dotted path `simlock config get` walks", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ drivers: { ios: { deviceRoot: "/Volumes/scratch/simlock-ios" } } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + + expect(dottedValue(config, "drivers.ios.deviceRoot")).toBe("/Volumes/scratch/simlock-ios"); + }); + + it("merges driver settings across layers key by key", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ + drivers: { ios: { deviceRoot: "/from-file" }, android: { headless: true } }, + }), + ); + + const config = await loadConfig({ + configPath, + filesystem, + systemStats: createStats(), + overrides: { drivers: { ios: { deviceRoot: "/from-override" } } }, + }); + + expect(config.drivers).toEqual({ + ios: { deviceRoot: "/from-override" }, + android: { headless: true }, + }); + }); + + it.each([ + [{ drivers: "everything" }, "drivers"], + [{ drivers: { ios: "/Volumes/scratch" } }, "drivers.ios"], + [{ drivers: { ios: { deviceRoot: { path: "/Volumes/scratch" } } } }, "drivers.ios.deviceRoot"], + [{ drivers: { ios: { deviceRoot: ["/Volumes/scratch"] } } }, "drivers.ios.deviceRoot"], + ])("rejects driver settings that are not plain scalars", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); +}); diff --git a/src/core/config-redact.test.ts b/src/core/config-redact.test.ts new file mode 100644 index 000000000..77a8110d4 --- /dev/null +++ b/src/core/config-redact.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { + type Config, + effectiveAllowDownload, + loadConfig, + REDACTED_VALUE, + redactConfig, +} from "./index.js"; +import { configPath, createStats } from "./test-config.js"; + +describe("effectiveAllowDownload", () => { + it("grants downloads for every request under the always policy", () => { + expect(effectiveAllowDownload("always", false)).toBe(true); + expect(effectiveAllowDownload("always", true)).toBe(true); + }); + + it("forbids downloads for every request under the never policy, even an explicit true", () => { + expect(effectiveAllowDownload("never", false)).toBe(false); + expect(effectiveAllowDownload("never", true)).toBe(false); + }); + + it("defers to the request's own flag under the on-request policy", () => { + expect(effectiveAllowDownload("on-request", false)).toBe(false); + expect(effectiveAllowDownload("on-request", true)).toBe(true); + }); +}); + +describe("redactConfig", () => { + async function load(contents: unknown, role?: "gateway" | "worker"): Promise { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + return loadConfig({ + configPath, + filesystem, + ...(role ? { role } : {}), + systemStats: createStats(), + }); + } + + it("replaces a set gateway.token with the marker, keeps every other key, and leaves the input untouched", async () => { + const config = await load({ + gateway: { label: "mac-1", token: "secret-join-token", url: "wss://gw.example:4700" }, + }); + + const before = structuredClone(config); + + const redacted = redactConfig(config); + + expect(redacted).toEqual({ ...before, gateway: { ...before.gateway, token: REDACTED_VALUE } }); + expect(config).toEqual(before); + expect(config.gateway.token).toBe("secret-join-token"); + }); + + it("masks every destination's url and secret, keeps name, format and events, and leaves its input untouched (a slack destination gets no secret key)", async () => { + const destinations = [ + { + name: "audit", + format: "generic", + url: "https://n8n.example.com/hook-token", + events: ["worker.connected"], + secret: "s3cr3t", + }, + { + name: "ops", + format: "generic", + url: "http://localhost:5678/x", + events: ["worker.disconnected"], + secret: "other", + }, + { + name: "chat", + format: "slack", + url: "https://hooks.slack.com/services/T0/B0/token", + events: ["worker.connected"], + }, + ]; + for (const token of ["join-token", undefined]) { + const config = await load({ gateway: { token, alerts: { destinations } } }, "gateway"); + const before = structuredClone(config); + + const redacted = redactConfig(config); + + expect(redacted.gateway.alerts.destinations).toEqual( + destinations.map((destination) => ({ + ...destination, + url: REDACTED_VALUE, + ...("secret" in destination ? { secret: REDACTED_VALUE } : {}), + })), + ); + expect(redacted.gateway.token).toBe(token === undefined ? undefined : REDACTED_VALUE); + expect(config).toEqual(before); + } + }); + + it("adds no gateway.token to a config that has none, so unset still reads as unset", async () => { + const config = await load({}); + + const redacted = redactConfig(config); + + expect(redacted).toEqual(config); + expect(Object.keys(redacted.gateway)).not.toContain("token"); + }); +}); diff --git a/src/core/config-warm-pool.test.ts b/src/core/config-warm-pool.test.ts new file mode 100644 index 000000000..09508132e --- /dev/null +++ b/src/core/config-warm-pool.test.ts @@ -0,0 +1,297 @@ +import { describe, expect, it } from "vitest"; +import { MemoryFilesystem, FakeSystemStats } from "../ports/index.js"; +import { loadConfig } from "./index.js"; +import { configPath, gibibyte, createStats } from "./test-config.js"; + +describe("loadConfig", () => { + it("applies a file-level warm-pool quarantine override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { quarantine: { maxRetries: 1, retryBackoffMs: 1_000 } } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.warmPool.quarantine).toEqual({ + maxRetries: 1, + retryBackoffMs: 1_000, + retryBackoffMultiplier: 2, + maxRetryBackoffMs: 5 * 60_000, + }); + }); + + it("keeps the warm pool on by default and applies a file-level warmPool.enabled override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + expect( + (await loadConfig({ configPath, filesystem, systemStats: createStats() })).warmPool.enabled, + ).toBe(true); + + await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool: { enabled: false } })); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.warmPool.enabled).toBe(false); + expect(config.warmPool.quarantine.maxRetries).toBe(3); + }); + + it("rejects a warmPool.enabled that is not a boolean", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool: { enabled: "no" } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("warmPool.enabled"); + }); + + it("rejects a non-positive-integer quarantine retry count", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { quarantine: { maxRetries: 0 } } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("warmPool.quarantine.maxRetries"); + }); + + it("defaults warmPool.reserveRunning to 0 on both platforms and applies a file-level value", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + expect( + (await loadConfig({ configPath, filesystem, systemStats: createStats() })).warmPool + .reserveRunning, + ).toEqual({ android: 0, ios: 0 }); + + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { reserveRunning: { ios: 4 } } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.warmPool.reserveRunning).toEqual({ android: 0, ios: 4 }); + + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { reserveRunning: { android: 0, ios: 0 } } }), + ); + const zero = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(zero.warmPool.reserveRunning).toEqual({ android: 0, ios: 0 }); + }); + + it.each(["ios", "android"] as const)( + "rejects a negative or non-integer warmPool.reserveRunning.%s naming the key", + async (platform) => { + for (const bad of [-1, 1.5, "1", null]) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { reserveRunning: { [platform]: bad } } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow( + `Invalid config value for "warmPool.reserveRunning.${platform}": expected a non-negative integer`, + ); + } + }, + ); + + describe("warmPool.targets and warmPool.maxConcurrentBoots", () => { + async function load(warmPool?: unknown, cpuCount = 8) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + if (warmPool !== undefined) { + await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool })); + } + const systemStats = new FakeSystemStats({ cpuCount, totalRamBytes: 32 * gibibyte }); + return loadConfig({ configPath, filesystem, systemStats }); + } + + it("defaults to no targets and max(1, floor(cpuCount / 4)) concurrent boots, and a file value wins", async () => { + expect((await load()).warmPool.targets).toEqual([]); + for (const [cpuCount, boots] of [ + [2, 1], + [6, 1], + [8, 2], + [10, 2], + [12, 3], + ] as const) { + expect( + (await load(undefined, cpuCount)).warmPool.maxConcurrentBoots, + `${cpuCount} CPUs`, + ).toBe(boots); + } + expect((await load({ maxConcurrentBoots: 5 }, 2)).warmPool.maxConcurrentBoots).toBe(5); + }); + + it("loads warmPool.baselineLearnAfter, 3 when unset, and rejects anything but a positive integer naming the key", async () => { + expect((await load()).warmPool.baselineLearnAfter).toBe(3); + expect((await load({ baselineLearnAfter: 1 })).warmPool.baselineLearnAfter).toBe(1); + expect((await load({ baselineLearnAfter: 5 })).warmPool.baselineLearnAfter).toBe(5); + for (const bad of [0, -1, 2.5, "3"]) { + await expect(load({ baselineLearnAfter: bad }), `${JSON.stringify(bad)}`).rejects.toThrow( + 'Invalid config value for "warmPool.baselineLearnAfter": expected a positive integer', + ); + } + }); + + it("loads a target with a model, an OS range, a mode and a count, and a boot cap of two", async () => { + const config = await load({ + maxConcurrentBoots: 2, + targets: [ + { count: 2, model: "iPhone 17", osVersion: "26.0", platform: "ios" }, + { count: 1, mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, + { count: 3, model: "Pixel 8", platform: "android" }, + ], + }); + + expect(config.warmPool.maxConcurrentBoots).toBe(2); + expect(config.warmPool.targets).toEqual([ + { count: 2, model: "iPhone 17", osVersion: "26.0", platform: "ios" }, + { count: 1, mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, + { count: 3, model: "Pixel 8", platform: "android" }, + ]); + }); + + it("rejects a target that names a class naming warmPool.targets[1].class", async () => { + await expect( + load({ + targets: [ + { count: 1, model: "iPhone 17", platform: "ios" }, + { class: "phone", count: 1, model: "iPhone 17", platform: "ios" }, + ], + }), + ).rejects.toThrow("warmPool.targets[1].class"); + }); + + it.each([ + [ + "a missing model", + { count: 1, platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].model": expected a non-empty string', + ], + [ + "an empty model", + { count: 1, model: "", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].model": expected a non-empty string', + ], + [ + "a missing platform", + { count: 1, model: "iPhone 17" }, + 'Invalid config value for "warmPool.targets[0].platform"', + ], + [ + "a count of 0", + { count: 0, model: "iPhone 17", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', + ], + [ + "an unknown platform", + { count: 1, model: "iPhone 17", platform: "tvos" }, + 'Invalid config value for "warmPool.targets[0].platform"', + ], + [ + "a missing count", + { model: "iPhone 17", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', + ], + [ + "a mode that is neither slim nor full", + { count: 1, mode: "lean", model: "iPhone 17", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].mode"', + ], + [ + "an OS range that does not parse", + { count: 1, model: "iPhone 17", osVersion: ">=", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].osVersion": expected an OS version or an OS range', + ], + [ + "an empty OS version", + { count: 1, model: "iPhone 17", osVersion: "", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].osVersion": expected a non-empty string', + ], + [ + "a count that is not an integer", + { count: 1.5, model: "iPhone 17", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', + ], + [ + "a class", + { class: "phone", count: 1, model: "iPhone 17", platform: "ios" }, + 'Invalid config value for "warmPool.targets[0].class": expected a known target key', + ], + [ + "an inherited property name as a key", + JSON.parse('{"count":1,"model":"iPhone 17","platform":"ios","constructor":1}') as object, + 'Invalid config value for "warmPool.targets[0].constructor": expected a known target key', + ], + [ + "a target that is not an object", + "iPhone 17", + 'Invalid config value for "warmPool.targets[0]": expected an object', + ], + ])("rejects a target with %s naming the key", async (_name, target, message) => { + await expect(load({ targets: [target] })).rejects.toThrow(message); + }); + + it("rejects a model or an OS version longer than status.get reports, and takes one of exactly that length", async () => { + await expect( + load({ targets: [{ count: 1, model: "m".repeat(257), platform: "ios" }] }), + ).rejects.toThrow( + 'Invalid config value for "warmPool.targets[0].model": expected a string of at most 256 characters', + ); + await expect( + load({ targets: [{ count: 1, model: "m", osVersion: "1".repeat(257), platform: "ios" }] }), + ).rejects.toThrow('Invalid config value for "warmPool.targets[0].osVersion"'); + const config = await load({ + targets: [ + { count: 1, model: "m".repeat(256), osVersion: "1".repeat(256), platform: "ios" }, + ], + }); + expect(config.warmPool.targets[0]?.model).toHaveLength(256); + }); + + it("rejects more targets than status.get reports, and takes exactly that many", async () => { + const target = { count: 1, model: "iPhone 17", platform: "ios" }; + await expect(load({ targets: Array.from({ length: 257 }, () => target) })).rejects.toThrow( + 'Invalid config value for "warmPool.targets": expected an array of at most 256 targets', + ); + const config = await load({ targets: Array.from({ length: 256 }, () => target) }); + expect(config.warmPool.targets).toHaveLength(256); + }); + + it("rejects targets that are not an array naming warmPool.targets", async () => { + await expect(load({ targets: { count: 1 } })).rejects.toThrow( + 'Invalid config value for "warmPool.targets": expected an array of targets', + ); + }); + + it.each([0, 1.5, "2", null])( + "rejects a maxConcurrentBoots of %j naming the key", + async (bad) => { + await expect(load({ maxConcurrentBoots: bad })).rejects.toThrow( + 'Invalid config value for "warmPool.maxConcurrentBoots": expected a positive integer', + ); + }, + ); + }); + + it("rejects a quarantine backoff multiplier below 1", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ warmPool: { quarantine: { retryBackoffMultiplier: 0.5 } } }), + ); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("warmPool.quarantine.retryBackoffMultiplier"); + }); +}); diff --git a/src/core/config.test.ts b/src/core/config.test.ts index 4c80d3066..a6f71ed0d 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -1,40 +1,8 @@ import { describe, expect, it, vi } from "vitest"; - -import { z } from "zod"; - -import type { AuthStrategyDefinition } from "../auth/index.js"; -import { MemoryFilesystem, FakeSystemStats } from "../ports/index.js"; -import type { ResourceStrategyOptions } from "./capacity/index.js"; +import { MemoryFilesystem } from "../ports/index.js"; import { configSchema } from "../contract/schemas.js"; -import { - booleanValue, - type Config, - driverConfig, - type DriverConfigEntry, - effectiveAllowDownload, - loadConfig, - REDACTED_VALUE, - redactConfig, -} from "./index.js"; - -/** Narrows the capacity block for assertions on resource-strategy configs. */ -function resourceOptions(config: Config): ResourceStrategyOptions { - if (config.capacity.strategy !== "resource") throw new Error("expected the resource strategy"); - return config.capacity.config; -} - -/** - * The walk `simlock config get ` performs over the daemon's config, reproduced here - * so a driver block is proven reachable by dotted key and not just present in the object. - */ -function dottedValue(config: Config, key: string): unknown { - let current: unknown = config; - for (const segment of key.split(".")) { - if (typeof current !== "object" || current === null) return undefined; - current = (current as Record)[segment]; - } - return current; -} +import { type Config, loadConfig } from "./index.js"; +import { resourceOptions, dottedValue, configPath, gibibyte, createStats } from "./test-config.js"; /** * Every leaf of a config object as a dotted key. Recursive on purpose: a first version of this @@ -50,16 +18,6 @@ function dottedLeafKeys(value: unknown, prefix = ""): string[] { ); } -const configPath = "/home/agent/.simlock/config.json"; -const gibibyte = 1024 ** 3; - -function createStats(): FakeSystemStats { - return new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }); -} - describe("loadConfig", () => { it("derives device limits from the injected machine capacity", async () => { const config = await loadConfig({ @@ -369,297 +327,6 @@ describe("loadConfig", () => { expect(equal.metrics).toEqual({ retention: 7 * DAY }); }); - it("applies a file-level warm-pool quarantine override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { quarantine: { maxRetries: 1, retryBackoffMs: 1_000 } } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.warmPool.quarantine).toEqual({ - maxRetries: 1, - retryBackoffMs: 1_000, - retryBackoffMultiplier: 2, - maxRetryBackoffMs: 5 * 60_000, - }); - }); - - it("keeps the warm pool on by default and applies a file-level warmPool.enabled override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - expect( - (await loadConfig({ configPath, filesystem, systemStats: createStats() })).warmPool.enabled, - ).toBe(true); - - await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool: { enabled: false } })); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.warmPool.enabled).toBe(false); - expect(config.warmPool.quarantine.maxRetries).toBe(3); - }); - - it("rejects a warmPool.enabled that is not a boolean", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool: { enabled: "no" } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("warmPool.enabled"); - }); - - it("rejects a non-positive-integer quarantine retry count", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { quarantine: { maxRetries: 0 } } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("warmPool.quarantine.maxRetries"); - }); - - it("defaults warmPool.reserveRunning to 0 on both platforms and applies a file-level value", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - expect( - (await loadConfig({ configPath, filesystem, systemStats: createStats() })).warmPool - .reserveRunning, - ).toEqual({ android: 0, ios: 0 }); - - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { reserveRunning: { ios: 4 } } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.warmPool.reserveRunning).toEqual({ android: 0, ios: 4 }); - - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { reserveRunning: { android: 0, ios: 0 } } }), - ); - const zero = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(zero.warmPool.reserveRunning).toEqual({ android: 0, ios: 0 }); - }); - - it.each(["ios", "android"] as const)( - "rejects a negative or non-integer warmPool.reserveRunning.%s naming the key", - async (platform) => { - for (const bad of [-1, 1.5, "1", null]) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { reserveRunning: { [platform]: bad } } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow( - `Invalid config value for "warmPool.reserveRunning.${platform}": expected a non-negative integer`, - ); - } - }, - ); - - describe("warmPool.targets and warmPool.maxConcurrentBoots", () => { - async function load(warmPool?: unknown, cpuCount = 8) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - if (warmPool !== undefined) { - await filesystem.writeFileAtomic(configPath, JSON.stringify({ warmPool })); - } - const systemStats = new FakeSystemStats({ cpuCount, totalRamBytes: 32 * gibibyte }); - return loadConfig({ configPath, filesystem, systemStats }); - } - - it("defaults to no targets and max(1, floor(cpuCount / 4)) concurrent boots, and a file value wins", async () => { - expect((await load()).warmPool.targets).toEqual([]); - for (const [cpuCount, boots] of [ - [2, 1], - [6, 1], - [8, 2], - [10, 2], - [12, 3], - ] as const) { - expect( - (await load(undefined, cpuCount)).warmPool.maxConcurrentBoots, - `${cpuCount} CPUs`, - ).toBe(boots); - } - expect((await load({ maxConcurrentBoots: 5 }, 2)).warmPool.maxConcurrentBoots).toBe(5); - }); - - it("loads warmPool.baselineLearnAfter, 3 when unset, and rejects anything but a positive integer naming the key", async () => { - expect((await load()).warmPool.baselineLearnAfter).toBe(3); - expect((await load({ baselineLearnAfter: 1 })).warmPool.baselineLearnAfter).toBe(1); - expect((await load({ baselineLearnAfter: 5 })).warmPool.baselineLearnAfter).toBe(5); - for (const bad of [0, -1, 2.5, "3"]) { - await expect(load({ baselineLearnAfter: bad }), `${JSON.stringify(bad)}`).rejects.toThrow( - 'Invalid config value for "warmPool.baselineLearnAfter": expected a positive integer', - ); - } - }); - - it("loads a target with a model, an OS range, a mode and a count, and a boot cap of two", async () => { - const config = await load({ - maxConcurrentBoots: 2, - targets: [ - { count: 2, model: "iPhone 17", osVersion: "26.0", platform: "ios" }, - { count: 1, mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, - { count: 3, model: "Pixel 8", platform: "android" }, - ], - }); - - expect(config.warmPool.maxConcurrentBoots).toBe(2); - expect(config.warmPool.targets).toEqual([ - { count: 2, model: "iPhone 17", osVersion: "26.0", platform: "ios" }, - { count: 1, mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, - { count: 3, model: "Pixel 8", platform: "android" }, - ]); - }); - - it("rejects a target that names a class naming warmPool.targets[1].class", async () => { - await expect( - load({ - targets: [ - { count: 1, model: "iPhone 17", platform: "ios" }, - { class: "phone", count: 1, model: "iPhone 17", platform: "ios" }, - ], - }), - ).rejects.toThrow("warmPool.targets[1].class"); - }); - - it.each([ - [ - "a missing model", - { count: 1, platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].model": expected a non-empty string', - ], - [ - "an empty model", - { count: 1, model: "", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].model": expected a non-empty string', - ], - [ - "a missing platform", - { count: 1, model: "iPhone 17" }, - 'Invalid config value for "warmPool.targets[0].platform"', - ], - [ - "a count of 0", - { count: 0, model: "iPhone 17", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', - ], - [ - "an unknown platform", - { count: 1, model: "iPhone 17", platform: "tvos" }, - 'Invalid config value for "warmPool.targets[0].platform"', - ], - [ - "a missing count", - { model: "iPhone 17", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', - ], - [ - "a mode that is neither slim nor full", - { count: 1, mode: "lean", model: "iPhone 17", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].mode"', - ], - [ - "an OS range that does not parse", - { count: 1, model: "iPhone 17", osVersion: ">=", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].osVersion": expected an OS version or an OS range', - ], - [ - "an empty OS version", - { count: 1, model: "iPhone 17", osVersion: "", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].osVersion": expected a non-empty string', - ], - [ - "a count that is not an integer", - { count: 1.5, model: "iPhone 17", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].count": expected a positive integer', - ], - [ - "a class", - { class: "phone", count: 1, model: "iPhone 17", platform: "ios" }, - 'Invalid config value for "warmPool.targets[0].class": expected a known target key', - ], - [ - "an inherited property name as a key", - JSON.parse('{"count":1,"model":"iPhone 17","platform":"ios","constructor":1}') as object, - 'Invalid config value for "warmPool.targets[0].constructor": expected a known target key', - ], - [ - "a target that is not an object", - "iPhone 17", - 'Invalid config value for "warmPool.targets[0]": expected an object', - ], - ])("rejects a target with %s naming the key", async (_name, target, message) => { - await expect(load({ targets: [target] })).rejects.toThrow(message); - }); - - it("rejects a model or an OS version longer than status.get reports, and takes one of exactly that length", async () => { - await expect( - load({ targets: [{ count: 1, model: "m".repeat(257), platform: "ios" }] }), - ).rejects.toThrow( - 'Invalid config value for "warmPool.targets[0].model": expected a string of at most 256 characters', - ); - await expect( - load({ targets: [{ count: 1, model: "m", osVersion: "1".repeat(257), platform: "ios" }] }), - ).rejects.toThrow('Invalid config value for "warmPool.targets[0].osVersion"'); - const config = await load({ - targets: [ - { count: 1, model: "m".repeat(256), osVersion: "1".repeat(256), platform: "ios" }, - ], - }); - expect(config.warmPool.targets[0]?.model).toHaveLength(256); - }); - - it("rejects more targets than status.get reports, and takes exactly that many", async () => { - const target = { count: 1, model: "iPhone 17", platform: "ios" }; - await expect(load({ targets: Array.from({ length: 257 }, () => target) })).rejects.toThrow( - 'Invalid config value for "warmPool.targets": expected an array of at most 256 targets', - ); - const config = await load({ targets: Array.from({ length: 256 }, () => target) }); - expect(config.warmPool.targets).toHaveLength(256); - }); - - it("rejects targets that are not an array naming warmPool.targets", async () => { - await expect(load({ targets: { count: 1 } })).rejects.toThrow( - 'Invalid config value for "warmPool.targets": expected an array of targets', - ); - }); - - it.each([0, 1.5, "2", null])( - "rejects a maxConcurrentBoots of %j naming the key", - async (bad) => { - await expect(load({ maxConcurrentBoots: bad })).rejects.toThrow( - 'Invalid config value for "warmPool.maxConcurrentBoots": expected a positive integer', - ); - }, - ); - }); - - it("rejects a quarantine backoff multiplier below 1", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ warmPool: { quarantine: { retryBackoffMultiplier: 0.5 } } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("warmPool.quarantine.retryBackoffMultiplier"); - }); - it("applies file values over defaults and explicit overrides over file values", async () => { const filesystem = new MemoryFilesystem(); await filesystem.mkdirp("/home/agent/.simlock"); @@ -738,176 +405,6 @@ describe("loadConfig", () => { ).rejects.toThrow(path); }); - it("accepts a lease TTL pair at the boundary, where the default equals the cap", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { defaultTtlMs: 40_000, maxTtlMs: 40_000 } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.lease).toMatchObject({ defaultTtlMs: 40_000, maxTtlMs: 40_000 }); - }); - - it.each([ - [{ lease: { identity: { ios: "disposable" } } }, "lease.identity.ios"], - [{ lease: { identity: { android: true } } }, "lease.identity.android"], - ])( - "rejects a lease.identity value other than reusable or fresh, naming the key (%#)", - async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }, - ); - - it("reads lease.requestRetentionMs and lease.maxRequestRecords", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { maxRequestRecords: 50, requestRetentionMs: 30_000 } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(config.lease).toMatchObject({ maxRequestRecords: 50, requestRetentionMs: 30_000 }); - }); - - it.each([ - [{ lease: { requestRetentionMs: 0 } }, "lease.requestRetentionMs"], - [{ lease: { maxRequestRecords: 1.5 } }, "lease.maxRequestRecords"], - ])("rejects an invalid lease-request limit, naming the key (%#)", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("an operator's lease.endedIdRetentionMs is checked at load: unset is 24 h, a positive number is read, and 0, a negative number or text are refused", async () => { - const load = async (written: unknown) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - if (written !== undefined) { - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { endedIdRetentionMs: written } }), - ); - } - return loadConfig({ configPath, filesystem, systemStats: createStats() }); - }; - - await expect(load(undefined)).resolves.toMatchObject({ - lease: { endedIdRetentionMs: 86_400_000 }, - }); - await expect(load(5_000)).resolves.toMatchObject({ lease: { endedIdRetentionMs: 5_000 } }); - for (const refused of [0, -1, "1d"]) { - await expect(load(refused), String(refused)).rejects.toThrow("lease.endedIdRetentionMs"); - } - }); - - it.each([ - [undefined, 1], - [2, 2], - [ - { ios: 1, android: 1 }, - { ios: 1, android: 1 }, - ], - [{ ios: 2 }, { ios: 2 }], - [{}, {}], - ])("reads an operator's lease.maxPerRequester %j as %j (absent: 1)", async (written, read) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - if (written !== undefined) { - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { maxPerRequester: written } }), - ); - } - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(config.lease.maxPerRequester).toEqual(read); - }); - - it.each([ - [0, "lease.maxPerRequester"], - [-1, "lease.maxPerRequester"], - [1.5, "lease.maxPerRequester"], - ["2", "lease.maxPerRequester"], - [{ ios: 0 }, "lease.maxPerRequester.ios"], - [{ ios: 1.5 }, "lease.maxPerRequester.ios"], - [{ ios: "1" }, "lease.maxPerRequester.ios"], - ])("rejects lease.maxPerRequester %j, naming %s", async (value, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { maxPerRequester: value } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("accepts a platform lease.maxPerRequester does not know, with a warning, and ignores it", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { maxPerRequester: { ios: 1, tvos: 1 } } }), - ); - const warn = vi.fn(); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats(), warn }); - - expect(config.lease.maxPerRequester).toEqual({ ios: 1 }); - expect(warn).toHaveBeenCalledWith('Unknown config key: "lease.maxPerRequester.tvos"'); - }); - - it("lease.endWaitMs defaults to 600000 and accepts a positive number and 2147483647", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - const unset = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: 5_000 } })); - const small = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { endWaitMs: 2_147_483_647 } }), - ); - const largest = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(unset.lease.endWaitMs).toBe(600_000); - expect(small.lease.endWaitMs).toBe(5_000); - expect(largest.lease.endWaitMs).toBe(2_147_483_647); - }); - - it.each([[0], [-1], [2_147_483_648], ["600000"]])( - "refuses lease.endWaitMs %j with a message naming the key, where 5000 loads", - async (value) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: 5_000 } })); - const accepted = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(accepted.lease.endWaitMs).toBe(5_000); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { endWaitMs: value } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("lease.endWaitMs"); - }, - ); - it("a loaded config parsed through configSchema keeps lease.endWaitMs", async () => { const config = await loadConfig({ configPath, @@ -917,1412 +414,43 @@ describe("loadConfig", () => { expect((configSchema.parse(config) as unknown as Config).lease.endWaitMs).toBe(600_000); }); +}); - it("reads lease.identity per platform and leaves the unset platform reusable", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( +describe("configSchema agrees with the config the daemon actually loads", () => { + /** + * `config.get` declares `configSchema` as its output (`contract/operations.ts`), and zod's + * object mode strips unknown keys -- so a key this schema does not list is not a type error + * anywhere, it simply vanishes on the way to the caller. + * + * Round 6 review found `gateway.leaseRequestTimeoutMs` doing exactly that: declared on + * `Config`, validated by `loadConfig`, documented in CONFIGURATION.md as inspectable via + * `simlock config get`, and silently dropped. Nothing referenced `configSchema` from any test, + * so the drift was invisible in both directions. + * + * This compares key sets per block rather than asserting one key, so the next block that gains + * a field fails here instead of shipping a `config get` that quietly omits it (testing rule 4: + * a test that enforces a rule has to see everything the rule covers). + */ + it("strips no key from any block of a freshly loaded config", async () => { + const config = await loadConfig({ configPath, - JSON.stringify({ lease: { identity: { ios: "fresh" } } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(config.lease.identity).toEqual({ android: "reusable", ios: "fresh" }); - }); + filesystem: new MemoryFilesystem(), + systemStats: createStats(), + }); - it.each([ - [{ lease: { defaultTtlMs: 0 } }, "lease.defaultTtlMs"], - [{ lease: { maxTtlMs: -1 } }, "lease.maxTtlMs"], - [{ lease: { defaultTtlMs: "soon" } }, "lease.defaultTtlMs"], - ])("rejects a non-positive lease TTL, naming the offending key (%#)", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + const parsed = configSchema.parse(config); + const declared = dottedLeafKeys(config); + const onTheWire = new Set(dottedLeafKeys(parsed)); - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); + expect(declared.filter((key) => !onTheWire.has(key))).toEqual([]); }); - it("rejects a defaultTtlMs above maxTtlMs, naming the offending key (ADR 0004)", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( + it("carries gateway.leaseRequestTimeoutMs through to what config get answers", async () => { + const config = await loadConfig({ configPath, - JSON.stringify({ lease: { defaultTtlMs: 40_001, maxTtlMs: 40_000 } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("lease.defaultTtlMs"); - }); - - it("catches the pair rule across layers, not just within one file", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ lease: { defaultTtlMs: 60_000 } }), - ); - - // The file's default is fine against the *default* cap; only the merged config is wrong. - await expect( - loadConfig({ - configPath, - filesystem, - overrides: { lease: { maxTtlMs: 30_000 } }, - systemStats: createStats(), - }), - ).rejects.toThrow("lease.defaultTtlMs"); - }); - - it.each([["detachedTtlMs"], ["heldTtlBackstopMs"], ["heartbeatIntervalMs"]])( - "warns about the retired lease.%s and ignores it, carrying no value over (ADR 0004)", - async (retired) => { - const filesystem = new MemoryFilesystem(); - const warn = vi.fn(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ lease: { [retired]: 1 } })); - - const config = await loadConfig({ - configPath, - filesystem, - systemStats: createStats(), - warn, - }); - - expect(warn).toHaveBeenCalledWith(`Unknown config key: "lease.${retired}"`); - // Not aliased onto anything: the new keys keep their own defaults. - expect(config.lease).toEqual({ - defaultTtlMs: 15 * 60_000, - identity: { android: "reusable", ios: "reusable" }, - maxPerRequester: 1, - endWaitMs: 600_000, - maxRequestRecords: 10_000, - maxTtlMs: 4 * 60 * 60_000, - requestRetentionMs: 10 * 60_000, - endedIdRetentionMs: 24 * 60 * 60_000, - }); - }, - ); - - it.each([ - [{ limits: { web: {} } }, "limits.web"], - [{ health: { maxBoltCount: 7 } }, "health.maxBoltCount"], - [{ ios: { turboMode: true } }, "ios.turboMode"], - ])( - "warns about an unknown key, nested or not, without rejecting the file: %j", - async (contents, key) => { - const filesystem = new MemoryFilesystem(); - const warn = vi.fn(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats(), warn }), - ).resolves.toBeDefined(); - expect(warn).toHaveBeenCalledWith(`Unknown config key: "${key}"`); - }, - ); - - it("applies a file-level health override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ - health: { - enabled: false, - probeIntervalMs: 60_000, - stableObservations: 3, - maxRecoveryAttempts: 5, - recoveryBackoffMs: 10_000, - maxConcurrentRecoveries: 2, - }, - }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.health).toEqual({ - enabled: false, - probeIntervalMs: 60_000, - stableObservations: 3, - maxRecoveryAttempts: 5, - recoveryBackoffMs: 10_000, - maxConcurrentRecoveries: 2, - }); - }); - - it("rejects a non-boolean health.enabled", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ health: { enabled: "yes" } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("health.enabled"); - }); - - it.each([ - [{ health: { probeIntervalMs: 0 } }, "health.probeIntervalMs"], - [{ health: { recoveryBackoffMs: -1 } }, "health.recoveryBackoffMs"], - [{ health: { stableObservations: 0 } }, "health.stableObservations"], - [{ health: { stableObservations: 1.5 } }, "health.stableObservations"], - [{ health: { maxRecoveryAttempts: 0 } }, "health.maxRecoveryAttempts"], - [{ health: { maxConcurrentRecoveries: 0 } }, "health.maxConcurrentRecoveries"], - ])("rejects invalid health values in every field", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("applies a file-level downloads override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ - downloads: { policy: "always", acceptAndroidLicenses: true, timeoutMs: 60_000 }, - }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.downloads).toEqual({ - policy: "always", - acceptAndroidLicenses: true, - timeoutMs: 60_000, - }); - }); - - it("applies an override-level downloads policy over the file value", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ downloads: { policy: "never" } }), - ); - - const config = await loadConfig({ - configPath, - filesystem, - overrides: { downloads: { policy: "always" } }, - systemStats: createStats(), - }); - expect(config.downloads.policy).toBe("always"); - }); - - it("rejects a downloads.policy outside the known set", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ downloads: { policy: "sometimes" } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("downloads.policy"); - }); - - it("rejects a non-boolean downloads.acceptAndroidLicenses", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ downloads: { acceptAndroidLicenses: "yes" } }), - ); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("downloads.acceptAndroidLicenses"); - }); - - it.each([ - [{ downloads: { timeoutMs: 0 } }, "downloads.timeoutMs"], - [{ downloads: { timeoutMs: -1 } }, "downloads.timeoutMs"], - [{ downloads: { timeoutMs: "1200000" } }, "downloads.timeoutMs"], - ])("rejects a non-positive or malformed downloads.timeoutMs", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("applies a file-level http override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ http: { enabled: true, host: "0.0.0.0", port: 8080 } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.http).toEqual({ enabled: true, host: "0.0.0.0", port: 8080 }); - }); - - it("applies an override-level http port over the file value", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { port: 5000 } })); - - const config = await loadConfig({ - configPath, - filesystem, - overrides: { http: { port: 6000 } }, - systemStats: createStats(), - }); - expect(config.http.port).toBe(6000); - }); - - it("rejects a non-boolean http.enabled", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { enabled: "yes" } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("http.enabled"); - }); - - it("rejects a non-string http.host", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { host: 127 } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("http.host"); - }); - - it.each([ - [{ http: { port: 0 } }, "http.port"], - [{ http: { port: 65536 } }, "http.port"], - [{ http: { port: 1.5 } }, "http.port"], - [{ http: { port: "4700" } }, "http.port"], - ])("rejects an out-of-range or malformed http.port", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("defaults ios.defaultMode to full", async () => { - const config = await loadConfig({ - configPath, - filesystem: new MemoryFilesystem(), - systemStats: createStats(), - }); - - expect(config.ios.defaultMode).toBe("full"); - }); - - it("applies a file-level ios.defaultMode override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultMode: "slim" } })); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.ios.defaultMode).toBe("slim"); - }); - - it("a driver's config entry adds its keys to its platform's section: defaulted, taken from the file, refused by its validator, and unknown to a loader without the entry", async () => { - const entry: DriverConfigEntry<{ readonly turbo: boolean }> = { - platform: "android", - defaults: { turbo: false }, - validators: { turbo: booleanValue }, - }; - const load = async (file: unknown, driverSections = [entry]) => { - const filesystem = new MemoryFilesystem(); - const warn = vi.fn(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(file)); - const config = await loadConfig({ - configPath, - driverSections, - filesystem, - systemStats: createStats(), - warn, - }); - return { config, section: driverConfig(config, entry), warn }; - }; - - const defaulted = await load({}); - const set = await load({ android: { turbo: true } }); - const unknown = await load({ android: { turbo: true } }, []); - - expect(defaulted.section.turbo).toBe(false); - expect([set.section.turbo, set.warn.mock.calls]).toEqual([true, []]); - await expect(load({ android: { turbo: "yes" } })).rejects.toThrow( - 'Invalid config value for "android.turbo"', - ); - expect(unknown.warn).toHaveBeenCalledWith('Unknown config key: "android.turbo"'); - expect(unknown.config.android).toEqual({ defaultModels: {} }); - expect(set.config.ios).toEqual({ defaultMode: "full", defaultModels: {} }); - }); - - it.each(["fast", "worker", true])( - "rejects an ios.defaultMode of %j, naming the key", - async (value) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultMode: value } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow("ios.defaultMode"); - }, - ); - - it.each(["ios", "android"] as const)( - "%s.defaultModels defaults to no class having a list", - async (platform) => { - const config = await loadConfig({ - configPath, - filesystem: new MemoryFilesystem(), - systemStats: createStats(), - }); - - expect(config[platform].defaultModels).toEqual({}); - }, - ); - - it.each([ - ["ios", "phone", "iPhone 15"], - ["ios", "tablet", ["iPad (A16)", "iPad (10th generation)"]], - ["android", "phone", "Pixel 7"], - ["android", "tv", ["Television (4K)"]], - ] as const)( - "%s.defaultModels.%s accepts one model name or a list of them", - async (platform, deviceClass, value) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ [platform]: { defaultModels: { [deviceClass]: value } } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(config[platform].defaultModels).toStrictEqual({ - [deviceClass]: typeof value === "string" ? [value] : value, - }); - }, - ); - - it.each([ - ["ios", { phone: "" }, "ios.defaultModels.phone"], - ["ios", { phone: [] }, "ios.defaultModels.phone"], - ["ios", { phone: ["iPhone 15", ""] }, "ios.defaultModels.phone"], - ["ios", { phone: 15 }, "ios.defaultModels.phone"], - ["ios", { phone: [15] }, "ios.defaultModels.phone"], - ["ios", { phone: [["iPhone 15"]] }, "ios.defaultModels.phone"], - ["ios", { mobile: "iPhone 15" }, "ios.defaultModels.mobile"], - ["ios", { constructor: "iPhone 15" }, "ios.defaultModels.constructor"], - ["android", { phone: [] }, "android.defaultModels.phone"], - ["android", { fridge: "Pixel 8" }, "android.defaultModels.fridge"], - ])("refuses %s.defaultModels %j, naming the key", async (platform, defaultModels, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ [platform]: { defaultModels } })); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(`Invalid config value for "${path}"`); - }); - - it("says what ios.defaultModels expects: a device class for a key, a name or a list of names for a value", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - const load = async (defaultModels: unknown) => { - await filesystem.writeFileAtomic(configPath, JSON.stringify({ ios: { defaultModels } })); - return loadConfig({ configPath, filesystem, systemStats: createStats() }); - }; - - await expect(load({ mobile: "iPhone 15" })).rejects.toThrow( - "expected a device class (phone, tablet, watch, tv, vision, auto, desktop)", - ); - await expect(load({ phone: [] })).rejects.toThrow( - "expected a model name or a non-empty list of model names", - ); - }); - - it("applies a file-level stalledTransition override", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ - stalledTransition: { thresholdMultiplier: 5, minimumThresholdMs: 120_000 }, - }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - expect(config.stalledTransition).toEqual({ - thresholdMultiplier: 5, - minimumThresholdMs: 120_000, - }); - }); - - it.each([ - [{ stalledTransition: { thresholdMultiplier: 0.5 } }, "stalledTransition.thresholdMultiplier"], - [{ stalledTransition: { minimumThresholdMs: -1 } }, "stalledTransition.minimumThresholdMs"], - ])("rejects invalid stalledTransition values in every field", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); - - it("defaults to no driver settings at all", async () => { - const config = await loadConfig({ - configPath, - filesystem: new MemoryFilesystem(), - systemStats: createStats(), - }); - - expect(config.drivers).toEqual({}); - }); - - it("keeps driver settings verbatim, including keys it has never heard of", async () => { - const filesystem = new MemoryFilesystem(); - const warn = vi.fn(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ - drivers: { - ios: { deviceRoot: "/Volumes/scratch/simlock-ios" }, - android: { adbServerPort: 5038, headless: true, somethingOnlyTheDriverKnows: "yes" }, - }, - }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats(), warn }); - - expect(config.drivers).toEqual({ - ios: { deviceRoot: "/Volumes/scratch/simlock-ios" }, - android: { adbServerPort: 5038, headless: true, somethingOnlyTheDriverKnows: "yes" }, - }); - // Warning about an unrecognised key here would mean the core knows which keys a - // driver has, which is the whole thing this block is not allowed to know. - expect(warn).not.toHaveBeenCalled(); - }); - - it("reads a driver setting at the dotted path `simlock config get` walks", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ drivers: { ios: { deviceRoot: "/Volumes/scratch/simlock-ios" } } }), - ); - - const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); - - expect(dottedValue(config, "drivers.ios.deviceRoot")).toBe("/Volumes/scratch/simlock-ios"); - }); - - it("merges driver settings across layers key by key", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify({ - drivers: { ios: { deviceRoot: "/from-file" }, android: { headless: true } }, - }), - ); - - const config = await loadConfig({ - configPath, - filesystem, - systemStats: createStats(), - overrides: { drivers: { ios: { deviceRoot: "/from-override" } } }, - }); - - expect(config.drivers).toEqual({ - ios: { deviceRoot: "/from-override" }, - android: { headless: true }, - }); - }); - - it.each([ - [{ drivers: "everything" }, "drivers"], - [{ drivers: { ios: "/Volumes/scratch" } }, "drivers.ios"], - [{ drivers: { ios: { deviceRoot: { path: "/Volumes/scratch" } } } }, "drivers.ios.deviceRoot"], - [{ drivers: { ios: { deviceRoot: ["/Volumes/scratch"] } } }, "drivers.ios.deviceRoot"], - ])("rejects driver settings that are not plain scalars", async (contents, path) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - - await expect( - loadConfig({ configPath, filesystem, systemStats: createStats() }), - ).rejects.toThrow(path); - }); -}); - -describe("loadConfig capacity strategies", () => { - async function load( - contents: unknown, - options: { - readonly overrides?: Parameters[0]["overrides"]; - readonly warn?: (message: string) => void; - } = {}, - ) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - return loadConfig({ - configPath, - filesystem, - systemStats: createStats(), - ...(options.overrides === undefined ? {} : { overrides: options.overrides }), - ...(options.warn === undefined ? {} : { warn: options.warn }), - }); - } - - it("defaults the fixed strategy to a machine-independent pin", async () => { - const config = await load({ capacity: { strategy: "fixed" } }); - - expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 2 } }); - }); - - it("pins concurrency from a single key", async () => { - const config = await load({ capacity: { strategy: "fixed", config: { maxRunning: 4 } } }); - - expect(config.capacity.config).toEqual({ maxRunning: 4 }); - }); - - it("lets an override switch the strategy chosen by the file", async () => { - const config = await load( - { capacity: { strategy: "resource" } }, - { overrides: { capacity: { strategy: "fixed", config: { maxRunning: 6 } } } }, - ); - - expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 6 } }); - }); - - it("starts from the selected strategy's defaults, not the default strategy's", async () => { - const config = await load({ capacity: { strategy: "fixed", config: { maxRunning: 3 } } }); - - expect(config.capacity.config).not.toHaveProperty("ramBudget"); - expect(config.capacity.config).not.toHaveProperty("limits"); - }); - - it("rejects a strategy name with no registered implementation", async () => { - await expect(load({ capacity: { strategy: "vibes" } })).rejects.toThrow("capacity.strategy"); - }); - - it("hands capacity.config to the selected strategy's own validator", async () => { - await expect( - load({ capacity: { strategy: "fixed", config: { maxRunning: 0 } } }), - ).rejects.toThrow("capacity.config.maxRunning"); - }); - - it("warns when capacity.config carries another strategy's keys", async () => { - const warn = vi.fn(); - await load({ capacity: { strategy: "fixed", config: { ramBudget: {} } } }, { warn }); - - expect(warn).toHaveBeenCalledWith('Unknown config key: "capacity.config.ramBudget"'); - }); -}); - -describe("loadConfig legacy capacity keys", () => { - async function load( - contents: unknown, - options: { - readonly overrides?: Parameters[0]["overrides"]; - readonly warn?: (message: string) => void; - } = {}, - ) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - return loadConfig({ - configPath, - filesystem, - systemStats: createStats(), - ...(options.overrides === undefined ? {} : { overrides: options.overrides }), - ...(options.warn === undefined ? {} : { warn: options.warn }), - }); - } - - it("folds top-level limits and ramBudget into the resource strategy's options", async () => { - const warn = vi.fn(); - const config = await load( - { - limits: { ios: { maxDevices: 3, maxRunning: 3 }, maxRunning: 5 }, - ramBudget: { iosBytesPerDevice: 2 * gibibyte }, - }, - { warn }, - ); - - expect(config.capacity.strategy).toBe("resource"); - expect(resourceOptions(config).limits).toMatchObject({ - ios: { maxDevices: 3, maxRunning: 3 }, - maxRunning: 5, - }); - expect(resourceOptions(config).ramBudget.iosBytesPerDevice).toBe(2 * gibibyte); - expect(warn).not.toHaveBeenCalled(); - }); - - it("loads the slim sizes from capacity.config.ramBudget and from the legacy top-level ramBudget, absent by default", async () => { - const fromCapacity = await load({ - capacity: { - strategy: "resource", - config: { - ramBudget: { - androidSlimBytesPerDevice: 2 * gibibyte, - iosSlimBytesPerDevice: 0.75 * gibibyte, - }, - }, - }, - }); - const fromLegacy = await load({ ramBudget: { iosSlimBytesPerDevice: 0.5 * gibibyte } }); - const byDefault = await load({}); - - expect(resourceOptions(fromCapacity).ramBudget).toEqual({ - androidBytesPerDevice: 4 * gibibyte, - androidSlimBytesPerDevice: 2 * gibibyte, - iosBytesPerDevice: 1.5 * gibibyte, - iosSlimBytesPerDevice: 0.75 * gibibyte, - }); - expect(resourceOptions(fromLegacy).ramBudget.iosSlimBytesPerDevice).toBe(0.5 * gibibyte); - expect(resourceOptions(byDefault).ramBudget).not.toHaveProperty("iosSlimBytesPerDevice"); - expect(resourceOptions(byDefault).ramBudget).not.toHaveProperty("androidSlimBytesPerDevice"); - }); - - it.each([ - ["iosSlimBytesPerDevice", -1], - ["iosSlimBytesPerDevice", "1GiB"], - ["androidSlimBytesPerDevice", -1], - ["androidSlimBytesPerDevice", null], - ])("rejects a %s of %j, naming the key", async (key, value) => { - await expect( - load({ capacity: { strategy: "resource", config: { ramBudget: { [key]: value } } } }), - ).rejects.toThrow(`capacity.config.ramBudget.${key}`); - await expect(load({ ramBudget: { [key]: value } })).rejects.toThrow(`ramBudget.${key}`); - }); - - it("prefers capacity.config over the legacy spelling within one layer", async () => { - const config = await load({ - capacity: { strategy: "resource", config: { limits: { maxRunning: 9 } } }, - limits: { maxRunning: 2 }, - }); - - expect(resourceOptions(config).limits.maxRunning).toBe(9); - }); - - it("keeps layer precedence when the layers disagree about spelling", async () => { - const config = await load( - { limits: { maxRunning: 8 } }, - { overrides: { capacity: { strategy: "resource", config: { limits: { maxRunning: 4 } } } } }, - ); - - expect(resourceOptions(config).limits.maxRunning).toBe(4); - }); - - it("lets a legacy override win over a capacity.config file value", async () => { - const config = await load( - { capacity: { strategy: "resource", config: { limits: { maxRunning: 8 } } } }, - { overrides: { limits: { maxRunning: 4 } } }, - ); - - expect(resourceOptions(config).limits.maxRunning).toBe(4); - }); - - it("warns and ignores legacy keys when another strategy is selected", async () => { - const warn = vi.fn(); - const config = await load( - { capacity: { strategy: "fixed", config: { maxRunning: 3 } }, limits: { maxRunning: 8 } }, - { warn }, - ); - - expect(config.capacity).toEqual({ strategy: "fixed", config: { maxRunning: 3 } }); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("Ignoring limits")); - }); -}); - -/** - * ADR 0005 §1/§2: `mode` selects what the daemon is, and a gateway reads only `mode`, `http.*`, - * `log.*`, `lease.*`, `eventBuffer.*` and `gateway.*`. - */ -describe("loadConfig modes (ADR 0005)", () => { - async function load( - contents: unknown, - options: { - readonly overrides?: Parameters[0]["overrides"]; - readonly role?: "worker" | "gateway"; - readonly warn?: (message: string) => void; - } = {}, - ) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - return loadConfig({ - configPath, - filesystem, - systemStats: createStats(), - ...(options.role === undefined ? {} : { role: options.role }), - ...(options.overrides === undefined ? {} : { overrides: options.overrides }), - ...(options.warn === undefined ? {} : { warn: options.warn }), - }); - } - - it("defaults to worker mode, with HTTP on and a 24h disconnected-view retention", async () => { - const config = await load({}); - - expect(config.mode).toBe("worker"); - expect(config.http.enabled).toBe(true); - expect(config.gateway).toEqual({ - disconnectedRetentionMs: 24 * 60 * 60_000, - execTimeoutMs: 11 * 60_000, - leaseRequestTimeoutMs: 5 * 60_000, - routing: "warm-then-free", - alerts: { destinations: [] }, - admission: { timeoutMs: 5_000 }, - }); - }); - - it("an operator's gateway.admission.command and gateway.admission.timeoutMs are checked at load: an argv list is read, an empty list, an empty string or text is refused, the timeout defaults to 5000, and 0 or over 60000 is refused", async () => { - const admission = (written: unknown) => load({ gateway: { admission: written } }); - const argument = (length: number) => "a".repeat(length); - - const read = await admission({ command: ["bun", "/etc/simlock/policy.ts"], timeoutMs: 60_000 }); - expect(read.gateway.admission).toEqual({ - command: ["bun", "/etc/simlock/policy.ts"], - timeoutMs: 60_000, - }); - expect((await admission({ command: [argument(4096)] })).gateway.admission?.timeoutMs).toBe( - 5_000, - ); - expect((await load({})).gateway.admission).toEqual({ timeoutMs: 5_000 }); - - const refused = [ - [{ command: [] }, "gateway.admission.command"], - [{ command: ["bun", ""] }, "gateway.admission.command"], - [{ command: "bun policy.ts" }, "gateway.admission.command"], - [{ command: [argument(4097)] }, "gateway.admission.command"], - [{ command: Array.from({ length: 65 }, () => "a") }, "gateway.admission.command"], - [{ timeoutMs: 0 }, "gateway.admission.timeoutMs"], - [{ timeoutMs: 60_001 }, "gateway.admission.timeoutMs"], - [{ timeoutMs: 1.5 }, "gateway.admission.timeoutMs"], - ] as const; - for (const [written, key] of refused) { - await expect(admission(written), JSON.stringify(written)).rejects.toThrow(key); - } - }); - - it.each([ - ["file worker", { contents: { mode: "worker" } }], - ["file gateway", { contents: { mode: "gateway" } }], - ["override gateway", { contents: {}, overrides: { mode: "gateway" } }], - ] as const)( - "refuses a config that sets mode, either value, from the file or the overrides, with the message that names --gateway (%s)", - async (_name, { contents, ...options }) => { - await expect(load(contents, options)).rejects.toThrow( - '"mode" is no longer a config key. A daemon started with --gateway is a gateway; any other is a worker. Remove "mode" from config.json.', - ); - }, - ); - - it("turns HTTP on by default in gateway mode -- a gateway is the fleet's contact point", async () => { - const config = await load({}, { role: "gateway" }); - - expect(config.http.enabled).toBe(true); - expect(config.http.port).toBe(4700); - }); - - it("refuses a gateway that switches HTTP off, naming the key, and loads a worker that does", async () => { - await expect(load({ http: { enabled: false } }, { role: "gateway" })).rejects.toThrow( - "http.enabled", - ); - // HTTP is genuinely optional for a worker. - await expect(load({ http: { enabled: false } })).resolves.toMatchObject({ - http: { enabled: false }, - mode: "worker", - }); - }); - - it("keeps an explicit gateway http host/port", async () => { - const config = await load({ http: { host: "0.0.0.0", port: 4800 } }, { role: "gateway" }); - - expect(config.http).toEqual({ enabled: true, host: "0.0.0.0", port: 4800 }); - }); - - it("warns about worker-only keys in a gateway's config and ignores them", async () => { - const warn = vi.fn(); - const config = await load( - { - capacity: { strategy: "fixed", config: { maxRunning: 3 } }, - ios: { defaultMode: "slim" }, - lease: { defaultTtlMs: 60_000 }, - }, - { role: "gateway", warn }, - ); - - expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "capacity"')); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "ios"')); - // Ignored, not rejected: the file still loads, and the gateway-readable keys still apply. - expect(config.lease.defaultTtlMs).toBe(60_000); - // "Ignored" means "nothing in gateway mode reads it" -- the value is still parsed and - // merged, so a config file can be shared between a worker and a gateway with only `mode` - // differing. The warning is what makes that visible rather than silent. - expect(warn).not.toHaveBeenCalledWith(expect.stringContaining('Ignoring "lease"')); - }); - - it("reads lease.maxPerRequester in gateway mode without an ignored-key warning", async () => { - const warn = vi.fn(); - - const config = await load({ lease: { maxPerRequester: 2 } }, { role: "gateway", warn }); - - expect(config.lease).toMatchObject({ maxPerRequester: 2 }); - expect(warn).not.toHaveBeenCalled(); - }); - - it("reads metrics.retention in a gateway config without a warning", async () => { - const warn = vi.fn(); - - const config = await load( - { metrics: { retention: 120 * 24 * 3_600_000 } }, - { role: "gateway", warn }, - ); - - expect(config.metrics).toEqual({ retention: 120 * 24 * 3_600_000 }); - expect(warn).not.toHaveBeenCalled(); - }); - - it("reads update.check in a gateway config without a warning", async () => { - const warn = vi.fn(); - - const config = await load({ update: { check: false } }, { role: "gateway", warn }); - - expect(config.update).toEqual({ check: false }); - expect(warn).not.toHaveBeenCalled(); - }); - - it("raises the unknown-key warning for eventLog.rotateBytes in a gateway config", async () => { - const warn = vi.fn(); - await load({ eventLog: { rotateBytes: 1024 } }, { role: "gateway", warn }); - - expect(warn).toHaveBeenCalledWith('Unknown config key: "eventLog.rotateBytes"'); - }); - - it("warns about android.emulator on a gateway without refusing the config", async () => { - const warn = vi.fn(); - const config = await load( - { android: { defaultModels: {}, emulator: { headless: true } } }, - { role: "gateway", warn }, - ); - - expect(warn).toHaveBeenCalledWith( - 'Ignoring "android": it configures a worker, and this daemon runs in gateway mode.', - ); - // Ignored, not rejected: the gateway still starts. - expect(config.mode).toBe("gateway"); - }); - - it("warns about the worker-side gateway.* keys on a gateway, but not the gateway's own", async () => { - const warn = vi.fn(); - const config = await load( - { - gateway: { - url: "ws://127.0.0.1:4700/v1/uplink", - token: "secret", - label: "ignored", - disconnectedRetentionMs: 60_000, - }, - }, - { role: "gateway" }, - ); - - expect(config.gateway.disconnectedRetentionMs).toBe(60_000); - - const warned = await load( - { gateway: { url: "ws://127.0.0.1:4700/v1/uplink" } }, - { role: "gateway", warn }, - ); - expect(warned.mode).toBe("gateway"); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "gateway.url"')); - expect(warn).not.toHaveBeenCalledWith( - expect.stringContaining('Ignoring "gateway.disconnectedRetentionMs"'), - ); - - const withHeaders = vi.fn(); - await load({ gateway: { headers: { "x-a": "1" } } }, { role: "gateway", warn: withHeaders }); - expect(withHeaders.mock.calls.map(([message]) => message)).toEqual([ - expect.stringContaining('Ignoring "gateway.headers"'), - ]); - const withoutHeaders = vi.fn(); - await load( - { gateway: { disconnectedRetentionMs: 60_000 } }, - { role: "gateway", warn: withoutHeaders }, - ); - expect(withoutHeaders).not.toHaveBeenCalled(); - }); - - it("says nothing about worker keys on a worker, which is every daemon today", async () => { - const warn = vi.fn(); - await load({ capacity: { strategy: "fixed", config: { maxRunning: 3 } } }, { warn }); - - expect(warn).not.toHaveBeenCalled(); - }); - - it("keeps a worker's gateway.url/token/label", async () => { - const config = await load({ - gateway: { url: "wss://fleet.example", token: "join-secret", label: "mac-1" }, - }); - - expect(config.gateway.url).toBe("wss://fleet.example"); - expect(config.gateway.token).toBe("join-secret"); - expect(config.gateway.label).toBe("mac-1"); - }); - - it("refuses a worker that names a gateway but no join token, and the reverse", async () => { - // Either alone is an operator half-finishing a join, and a worker that silently never - // joined looks exactly like one whose gateway is down -- so this fails the start instead. - await expect(load({ gateway: { url: "ws://fleet.example" } })).rejects.toThrow("gateway.token"); - await expect(load({ gateway: { token: "join-secret" } })).rejects.toThrow("gateway.url"); - }); - - it("stops the load on a bad gateway.headers with the message naming it, and keeps every accepted row", async () => { - const key = (what: string) => `Invalid config value for "gateway.headers": expected ${what}`; - const shape = "an object of header names to text values"; - const joined = { url: "wss://fleet.example", token: "join-secret" }; - const many = Object.fromEntries(Array.from({ length: 17 }, (_, i) => [`x-h${i}`, "v"])); - const fat = Object.fromEntries( - Array.from({ length: 3 }, (_, i) => [`x-h${i}`, "a".repeat(3000)]), - ); - const refused: readonly (readonly [string, unknown, string])[] = [ - ["an array", ["a"], shape], - ["a non-text value", { "x-ok": 1 }, shape], - ["17 entries", many, "at most 16 headers"], - [ - "a name with a space", - { "X Bad": "v" }, - "a header name of 1 to 100 letters, digits and HTTP name symbols (got X Bad)", - ], - [ - "an empty name", - { "": "x" }, - "a header name of 1 to 100 letters, digits and HTTP name symbols (got )", - ], - [ - "a 101-character name", - { ["a".repeat(101)]: "v" }, - `a header name of 1 to 100 letters, digits and HTTP name symbols (got ${"a".repeat(101)})`, - ], - ["one name twice", { "X-A": "1", "x-a": "2" }, "each header name once (got x-a twice)"], - [ - "a line break", - { "x-ok": "line\nbreak" }, - "a header value of at most 4096 plain ASCII characters (in x-ok)", - ], - [ - "a non-ASCII value", - { "x-ok": "café" }, - "a header value of at most 4096 plain ASCII characters (in x-ok)", - ], - [ - "a 4097-character value", - { "x-ok": "a".repeat(4097) }, - "a header value of at most 4096 plain ASCII characters (in x-ok)", - ], - ["over 8192 in all", fat, "at most 8192 characters of header names and values in all"], - [ - "8193 in all", - { "x-a": "a".repeat(4096), "x-b": "b".repeat(4091) }, - "at most 8192 characters of header names and values in all", - ], - ...[ - "authorization", - "x-simlock-worker-id", - "x-simlock-worker-label", - "host", - "connection", - "upgrade", - "origin", - ].map((name): readonly [string, unknown, string] => [ - `${name} in capitals`, - { [name.toUpperCase()]: "x" }, - `no header Simlock sets itself (${name.toUpperCase()})`, - ]), - ["Authorization", { Authorization: "x" }, "no header Simlock sets itself (Authorization)"], - [ - "a sec-websocket- name", - { "Sec-WebSocket-Key": "x" }, - "no header Simlock sets itself (Sec-WebSocket-Key)", - ], - ]; - for (const role of ["worker", "gateway"] as const) { - for (const [name, headers, what] of refused) { - await expect( - load({ gateway: { ...joined, headers } }, { role }), - `${role}: ${name}`, - ).rejects.toThrow(key(what)); - } - } - // why: the edge itself is accepted, so moving a cap by one fails a named row. - const sixteen = Object.fromEntries(Array.from({ length: 16 }, (_, i) => [`x-h${i}`, "v"])); - const longestName = { ["a".repeat(100)]: "v" }; - for (const headers of [sixteen, longestName]) { - await expect(load({ gateway: { ...joined, headers } })).resolves.toBeDefined(); - } - await expect(load({ gateway: { headers: { "x-a": "1" } } })).rejects.toThrow( - key("gateway.url to be set too, since gateway.headers is set"), - ); - await expect(load({ gateway: { token: "t", headers: { "x-a": "1" } } })).rejects.toThrow( - 'Invalid config value for "gateway.url": expected a gateway URL, since gateway.token is set', - ); - - const exactly8192 = { "x-a": "a".repeat(4096), "x-b": "b".repeat(4090) }; - expect((await load({ gateway: { ...joined, headers: exactly8192 } })).gateway.headers).toEqual( - exactly8192, - ); - const accepted = { "CF-Access-Client-Id": "abc.access", "x-empty": "", "x-tab": "a\tb" }; - expect((await load({ gateway: { ...joined, headers: accepted } })).gateway.headers).toEqual( - accepted, - ); - expect((await load({ gateway: { headers: {} } })).gateway.url).toBeUndefined(); - }); - - it("refuses a joined worker's own HTTP listener with the exact error, and keeps every valid and invalid neighbour as it was", async () => { - const refusal = - 'Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)'; - const joined = { url: "wss://fleet.example", token: "join-secret" }; - type Case = { - readonly name: string; - readonly contents: Record; - readonly overrides?: Parameters[0]["overrides"]; - readonly role?: "worker" | "gateway"; - readonly error?: string; - readonly http?: boolean; - }; - const cases: readonly Case[] = [ - { name: "file only", contents: { gateway: joined, http: { enabled: true } }, error: refusal }, - { - name: "overrides only", - contents: {}, - overrides: { gateway: joined, http: { enabled: true } }, - error: refusal, - }, - { - name: "membership in file, HTTP in overrides", - contents: { gateway: joined }, - overrides: { http: { enabled: true } }, - error: refusal, - }, - { - name: "HTTP in file, membership in overrides", - contents: { http: { enabled: true } }, - overrides: { gateway: joined }, - error: refusal, - }, - { - name: "override false repairs file true", - contents: { gateway: joined, http: { enabled: true } }, - overrides: { http: { enabled: false } }, - http: false, - }, - { - name: "ws URL, HTTP false", - contents: { gateway: { ...joined, url: "ws://fleet.example/x" }, http: { enabled: false } }, - http: false, - }, - { name: "wss URL, HTTP omitted", contents: { gateway: joined }, http: false }, - { name: "standalone, HTTP true", contents: { http: { enabled: true } }, http: true }, - { name: "standalone, HTTP false", contents: { http: { enabled: false } }, http: false }, - { name: "standalone, HTTP omitted", contents: {}, http: true }, - { name: "gateway, HTTP omitted", contents: {}, role: "gateway", http: true }, - { - name: "gateway, HTTP true", - contents: { http: { enabled: true } }, - role: "gateway", - http: true, - }, - { - name: "gateway, HTTP false", - contents: { http: { enabled: false } }, - role: "gateway", - error: "http.enabled", - }, - { - name: "gateway ignoring membership keys", - contents: { gateway: joined, http: { enabled: true } }, - role: "gateway", - http: true, - }, - { - name: "empty URL", - contents: { gateway: { url: "", token: "t" }, http: { enabled: true } }, - error: "a ws:// or wss:// URL", - }, - { - name: "malformed URL", - contents: { gateway: { url: "not a url", token: "t" }, http: { enabled: true } }, - error: "a ws:// or wss:// URL", - }, - { - name: "http URL", - contents: { gateway: { url: "http://fleet.example", token: "t" }, http: { enabled: true } }, - error: "a ws:// or wss:// URL", - }, - { - name: "https URL", - contents: { - gateway: { url: "https://fleet.example", token: "t" }, - http: { enabled: true }, - }, - error: "a ws:// or wss:// URL", - }, - { - name: "URL without token", - contents: { gateway: { url: "wss://fleet.example" }, http: { enabled: true } }, - error: "gateway.token", - }, - { - name: "token without URL", - contents: { gateway: { token: "t" }, http: { enabled: true } }, - error: "a gateway URL, since gateway.token is set", - }, - { - name: "removed mode key", - contents: { mode: "worker", gateway: joined, http: { enabled: true } }, - error: '"mode" is no longer a config key', - }, - ]; - - for (const { name, contents, overrides, role, error, http } of cases) { - const outcome = load(contents, { - ...(overrides === undefined ? {} : { overrides }), - ...(role === undefined ? {} : { role }), - }); - if (error === undefined) { - await expect(outcome, name).resolves.toMatchObject({ http: { enabled: http } }); - } else if (error === refusal) { - await expect(outcome, name).rejects.toMatchObject({ message: refusal }); - } else { - await expect(outcome, name).rejects.toThrow(error); - } - } - }); - - it.each(["http://fleet.example/v1/uplink", "fleet.example", "not a url"])( - "rejects a gateway.url that is not a WebSocket URL: %s", - async (url) => { - await expect(load({ gateway: { url } })).rejects.toThrow("a ws:// or wss:// URL"); - }, - ); - - it("rejects a non-positive disconnected retention", async () => { - await expect(load({ gateway: { disconnectedRetentionMs: 0 } })).rejects.toThrow( - "gateway.disconnectedRetentionMs", - ); - }); - - it("rejects a non-positive lease-request timeout", async () => { - // why: P2 (round 2 review): bounds the one forwarded uplink call that used to have none of its - // own -- see FleetForwarding.withLeaseRequestTimeout. - await expect(load({ gateway: { leaseRequestTimeoutMs: 0 } })).rejects.toThrow( - "gateway.leaseRequestTimeoutMs", - ); - }); - - it("defaults the gateway lease-request timeout well under the exec backstop", async () => { - const config = await load({}, { role: "gateway" }); - expect(config.gateway.leaseRequestTimeoutMs).toBeLessThan(config.gateway.execTimeoutMs); - }); - - it("reports the role it was loaded for as config.mode", async () => { - expect((await load({}, { role: "gateway" })).mode).toBe("gateway"); - expect((await load({}, { role: "worker" })).mode).toBe("worker"); - }); - - it("defaults the memoryPressure section, refuses a bad value naming its key, and warns a gateway it is worker-only", async () => { - expect((await load({})).memoryPressure).toEqual({ - clearAfterMs: 60_000, - enabled: true, - pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, - pauseWarming: { level: "warn", minAvailablePercent: 20 }, - }); - const accepted = await load({ - memoryPressure: { pauseWarming: { level: "off", minAvailablePercent: 0 }, clearAfterMs: 0 }, - }); - expect(accepted.memoryPressure).toMatchObject({ - clearAfterMs: 0, - pauseWarming: { level: "off", minAvailablePercent: 0 }, - }); - - const refused = [ - { memoryPressure: { pauseWarming: { level: "bogus" } } }, - { memoryPressure: { pauseNewDevices: { minAvailablePercent: 101 } } }, - { memoryPressure: { pauseNewDevices: { minAvailablePercent: -1 } } }, - { memoryPressure: { clearAfterMs: -1 } }, - ]; - const keys = [ - "memoryPressure.pauseWarming.level", - "memoryPressure.pauseNewDevices.minAvailablePercent", - "memoryPressure.pauseNewDevices.minAvailablePercent", - "memoryPressure.clearAfterMs", - ]; - for (const [index, file] of refused.entries()) { - await expect(load(file)).rejects.toThrow(keys[index]!); - } - - const warn = vi.fn(); - await load({ memoryPressure: { enabled: false } }, { role: "gateway", warn }); - expect(warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring "memoryPressure"')); - }); - - it("HTTP defaults to port 4700 for a gateway and 4701 for a worker", async () => { - expect((await load({}, { role: "gateway" })).http.port).toBe(4700); - expect((await load({})).http.port).toBe(4701); - }); -}); - -describe("effectiveAllowDownload", () => { - it("grants downloads for every request under the always policy", () => { - expect(effectiveAllowDownload("always", false)).toBe(true); - expect(effectiveAllowDownload("always", true)).toBe(true); - }); - - it("forbids downloads for every request under the never policy, even an explicit true", () => { - expect(effectiveAllowDownload("never", false)).toBe(false); - expect(effectiveAllowDownload("never", true)).toBe(false); - }); - - it("defers to the request's own flag under the on-request policy", () => { - expect(effectiveAllowDownload("on-request", false)).toBe(false); - expect(effectiveAllowDownload("on-request", true)).toBe(true); - }); -}); - -describe("redactConfig", () => { - async function load(contents: unknown, role?: "gateway" | "worker"): Promise { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); - return loadConfig({ - configPath, - filesystem, - ...(role ? { role } : {}), - systemStats: createStats(), - }); - } - - it("replaces a set gateway.token with the marker, keeps every other key, and leaves the input untouched", async () => { - const config = await load({ - gateway: { label: "mac-1", token: "secret-join-token", url: "wss://gw.example:4700" }, - }); - - const before = structuredClone(config); - - const redacted = redactConfig(config); - - expect(redacted).toEqual({ ...before, gateway: { ...before.gateway, token: REDACTED_VALUE } }); - expect(config).toEqual(before); - expect(config.gateway.token).toBe("secret-join-token"); - }); - - it("masks every destination's url and secret, keeps name, format and events, and leaves its input untouched (a slack destination gets no secret key)", async () => { - const destinations = [ - { - name: "audit", - format: "generic", - url: "https://n8n.example.com/hook-token", - events: ["worker.connected"], - secret: "s3cr3t", - }, - { - name: "ops", - format: "generic", - url: "http://localhost:5678/x", - events: ["worker.disconnected"], - secret: "other", - }, - { - name: "chat", - format: "slack", - url: "https://hooks.slack.com/services/T0/B0/token", - events: ["worker.connected"], - }, - ]; - for (const token of ["join-token", undefined]) { - const config = await load({ gateway: { token, alerts: { destinations } } }, "gateway"); - const before = structuredClone(config); - - const redacted = redactConfig(config); - - expect(redacted.gateway.alerts.destinations).toEqual( - destinations.map((destination) => ({ - ...destination, - url: REDACTED_VALUE, - ...("secret" in destination ? { secret: REDACTED_VALUE } : {}), - })), - ); - expect(redacted.gateway.token).toBe(token === undefined ? undefined : REDACTED_VALUE); - expect(config).toEqual(before); - } - }); - - it("adds no gateway.token to a config that has none, so unset still reads as unset", async () => { - const config = await load({}); - - const redacted = redactConfig(config); - - expect(redacted).toEqual(config); - expect(Object.keys(redacted.gateway)).not.toContain("token"); - }); -}); - -describe("configSchema agrees with the config the daemon actually loads", () => { - /** - * `config.get` declares `configSchema` as its output (`contract/operations.ts`), and zod's - * object mode strips unknown keys -- so a key this schema does not list is not a type error - * anywhere, it simply vanishes on the way to the caller. - * - * Round 6 review found `gateway.leaseRequestTimeoutMs` doing exactly that: declared on - * `Config`, validated by `loadConfig`, documented in CONFIGURATION.md as inspectable via - * `simlock config get`, and silently dropped. Nothing referenced `configSchema` from any test, - * so the drift was invisible in both directions. - * - * This compares key sets per block rather than asserting one key, so the next block that gains - * a field fails here instead of shipping a `config get` that quietly omits it (testing rule 4: - * a test that enforces a rule has to see everything the rule covers). - */ - it("strips no key from any block of a freshly loaded config", async () => { - const config = await loadConfig({ - configPath, - filesystem: new MemoryFilesystem(), - systemStats: createStats(), - }); - - const parsed = configSchema.parse(config); - const declared = dottedLeafKeys(config); - const onTheWire = new Set(dottedLeafKeys(parsed)); - - expect(declared.filter((key) => !onTheWire.has(key))).toEqual([]); - }); - - it("carries gateway.leaseRequestTimeoutMs through to what config get answers", async () => { - const config = await loadConfig({ - configPath, - filesystem: new MemoryFilesystem(), - systemStats: createStats(), - }); + filesystem: new MemoryFilesystem(), + systemStats: createStats(), + }); const parsed = configSchema.parse(config) as unknown as Config; expect(dottedValue(parsed, "gateway.leaseRequestTimeoutMs")).toBe( @@ -2330,156 +458,3 @@ describe("configSchema agrees with the config the daemon actually loads", () => ); }); }); - -describe("loadConfig over the auth strategies' config keys", () => { - const fake: AuthStrategyDefinition = { - name: "fake", - config: { - key: "fake", - schema: z.object({ limit: z.number().int().positive() }), - defaults: { limit: 5 }, - }, - enabled: () => true, - create: () => ({ authenticate: async () => ({ kind: "not-mine", sawCredential: false }) }), - }; - - async function load(http: unknown) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic(configPath, JSON.stringify({ http })); - return loadConfig({ - authStrategies: [fake], - configPath, - filesystem, - systemStats: createStats(), - }); - } - - it("fills a registered strategy's defaults and validates the key it owns", async () => { - expect(dottedValue(await load({}), "http.fake.limit")).toBe(5); - expect(dottedValue(await load({ fake: { limit: 9 } }), "http.fake.limit")).toBe(9); - await expect(load({ fake: { limit: "many" } })).rejects.toThrow("http.fake"); - }); -}); - -describe("loadConfig over http.proxyAuth", () => { - const complete = { - header: "cf-access-jwt-assertion", - jwksUrl: "https://team.cloudflareaccess.com/cdn-cgi/access/certs", - issuer: "https://team.cloudflareaccess.com", - audience: "4f1c0e", - }; - - async function load(proxyAuth: unknown) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/home/agent/.simlock"); - await filesystem.writeFileAtomic( - configPath, - JSON.stringify(proxyAuth === undefined ? {} : { http: { proxyAuth } }), - ); - return loadConfig({ configPath, filesystem, systemStats: createStats() }); - } - - function proxyAuthOf(config: Config): unknown { - return dottedValue(config, "http.proxyAuth"); - } - - it("starts with proxy sign-in off when http.proxyAuth is not set, and on with the four keys", async () => { - expect(proxyAuthOf(await load(undefined))).toBeUndefined(); - expect(proxyAuthOf(await load(complete))).toEqual(complete); - expect(proxyAuthOf(await load({ ...complete, signOutUrl: "http://localhost:1/out" }))).toEqual({ - ...complete, - signOutUrl: "http://localhost:1/out", - }); - }); - - it.each(["http://127.0.0.1:9000/keys", "http://localhost:9000/keys", "http://[::1]:9000/keys"])( - "accepts the key URL %s over http on the local host", - async (jwksUrl) => { - expect(proxyAuthOf(await load({ ...complete, jwksUrl }))).toEqual({ ...complete, jwksUrl }); - }, - ); - - it("refuses a key URL that is neither https nor http on 127.0.0.1, ::1 or localhost, naming the key", async () => { - await expect(load(complete)).resolves.toBeDefined(); - - for (const jwksUrl of ["http://keys.team.dev/certs", "not a url"]) { - await expect(load({ ...complete, jwksUrl }), jwksUrl).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.jwksUrl": expected an https:// URL, or http:// on 127.0.0.1, ::1 or localhost', - ); - } - }); - - it("refuses an incomplete set, naming the first missing key", async () => { - await expect(load(complete)).resolves.toBeDefined(); - - for (const missing of ["header", "jwksUrl", "issuer", "audience"] as const) { - const rest = Object.fromEntries(Object.entries(complete).filter(([key]) => key !== missing)); - await expect(load(rest), missing).rejects.toThrow( - `Invalid config value for "http.proxyAuth.${missing}": expected set with the other http.proxyAuth keys`, - ); - } - await expect(load({ header: complete.header })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.jwksUrl": expected set with the other http.proxyAuth keys', - ); - await expect(load({ signOutUrl: "https://team.dev/out" })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.header": expected set with the other http.proxyAuth keys', - ); - }); - - it("accepts a header written in any case, and refuses one that is not an HTTP header name", async () => { - expect( - proxyAuthOf(await load({ ...complete, header: "Cf-Access-Jwt-Assertion" })), - ).toMatchObject({ header: "Cf-Access-Jwt-Assertion" }); - - await expect(load({ ...complete, header: "x bad" })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.header": expected an HTTP header name', - ); - }); - - it("bounds the header name to 64 characters, the audience to 512, and refuses an empty issuer or audience", async () => { - await expect(load({ ...complete, header: "h".repeat(64) })).resolves.toBeDefined(); - await expect(load({ ...complete, header: "h".repeat(65) })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.header": expected an HTTP header name', - ); - await expect(load({ ...complete, audience: "a".repeat(512) })).resolves.toBeDefined(); - await expect(load({ ...complete, audience: "a".repeat(513) })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.audience": expected at most 512 characters', - ); - await expect(load({ ...complete, issuer: "" })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.issuer": expected a non-empty string', - ); - await expect(load({ ...complete, audience: "" })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.audience": expected a non-empty string', - ); - }); - - it("refuses a sign-out URL that is not an http or https URL", async () => { - for (const signOutUrl of ["not a url", "ftp://team.dev/out"]) { - await expect(load({ ...complete, signOutUrl }), signOutUrl).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.signOutUrl": expected an http:// or https:// URL', - ); - } - }); - - it("refuses a wrong-type header, jwksUrl or signOutUrl with our message", async () => { - const cases = [ - ["header", "an HTTP header name"], - ["jwksUrl", "an https:// URL, or http:// on 127.0.0.1, ::1 or localhost"], - ["signOutUrl", "an http:// or https:// URL"], - ] as const; - for (const [key, message] of cases) { - await expect(load({ ...complete, [key]: 7 }), key).rejects.toThrow( - `Invalid config value for "http.proxyAuth.${key}": expected ${message}`, - ); - } - }); - - it("accepts an issuer of 128 characters and refuses one of 129", async () => { - await expect(load({ ...complete, issuer: "i".repeat(128) })).resolves.toBeDefined(); - - await expect(load({ ...complete, issuer: "i".repeat(129) })).rejects.toThrow( - 'Invalid config value for "http.proxyAuth.issuer": expected at most 128 characters', - ); - }); -}); diff --git a/src/core/test-config.ts b/src/core/test-config.ts new file mode 100644 index 000000000..5080ceff1 --- /dev/null +++ b/src/core/test-config.ts @@ -0,0 +1,32 @@ +import type { Config } from "./index.js"; +import type { ResourceStrategyOptions } from "./capacity/index.js"; +import { FakeSystemStats } from "../ports/index.js"; + +/** Narrows the capacity block for assertions on resource-strategy configs. */ +export function resourceOptions(config: Config): ResourceStrategyOptions { + if (config.capacity.strategy !== "resource") throw new Error("expected the resource strategy"); + return config.capacity.config; +} + +/** + * The walk `simlock config get ` performs over the daemon's config, reproduced here + * so a driver block is proven reachable by dotted key and not just present in the object. + */ +export function dottedValue(config: Config, key: string): unknown { + let current: unknown = config; + for (const segment of key.split(".")) { + if (typeof current !== "object" || current === null) return undefined; + current = (current as Record)[segment]; + } + return current; +} + +export const configPath = "/home/agent/.simlock/config.json"; +export const gibibyte = 1024 ** 3; + +export function createStats(): FakeSystemStats { + return new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }); +} From bb7808afb70e2b1bd067790ea781aedc93670799 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:24:34 +0100 Subject: [PATCH 30/83] test(core): split doctor.test.ts by topic --- src/core/doctor-observed-state.test.ts | 624 +++++++ src/core/doctor-physical-devices.test.ts | 266 +++ src/core/doctor-unobservable-platform.test.ts | 276 +++ src/core/doctor-warm-pool-targets.test.ts | 453 +++++ src/core/doctor.test.ts | 1604 +---------------- src/core/test-doctor-observed.ts | 12 + 6 files changed, 1635 insertions(+), 1600 deletions(-) create mode 100644 src/core/doctor-observed-state.test.ts create mode 100644 src/core/doctor-physical-devices.test.ts create mode 100644 src/core/doctor-unobservable-platform.test.ts create mode 100644 src/core/doctor-warm-pool-targets.test.ts create mode 100644 src/core/test-doctor-observed.ts diff --git a/src/core/doctor-observed-state.test.ts b/src/core/doctor-observed-state.test.ts new file mode 100644 index 000000000..abd5ce40a --- /dev/null +++ b/src/core/doctor-observed-state.test.ts @@ -0,0 +1,624 @@ +import { describe, expect, it, vi } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { DeviceOperationClaims } from "./device-operation-claims.js"; +import { Doctor } from "./doctor.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import { FakeDriver } from "./fake-driver.js"; +import { QuarantineCoordinator } from "./quarantine-coordinator.js"; +import { type Registry } from "./registry.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { loadTestRegistry } from "./testing.js"; +import { config, readyDevice, sequence } from "./test-doctor.js"; + +async function shutdownDevice( + registry: Registry, + driverDeviceId: string, + platform: "ios" | "android", +) { + await readyDevice(registry, driverDeviceId, platform); + const device = registry.snapshot.devices.find( + (candidate) => candidate.driverDeviceId === driverDeviceId, + )!; + await registry.transitionDevice(device.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: device.id, initiator: "test" }, + }); + return registry.snapshot.devices.find((candidate) => candidate.id === device.id)!; +} + +// why: pure logic with many cases: Doctor's drift, stall and finding classification, built +// directly over a fake driver and a test registry rather than a whole daemon. +describe("Doctor", () => { + it("reports foreign-state-change for a device booted outside Simlock, on both platforms", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const iosDevice = await shutdownDevice(registry, "simlock-ios-1", "ios"); + const androidDevice = await shutdownDevice(registry, "simlock_android-1", "android"); + + const iosDriver = new FakeDriver({ clock, platform: "ios" }); + iosDriver.setManagedReality({ + devices: [ + { + address: "simlock-ios-1-address", + deviceId: "simlock-ios-1", + driverData: {}, + runState: "running", + }, + ], + processes: [], + }); + const androidDriver = new FakeDriver({ clock, platform: "android" }); + androidDriver.setManagedReality({ + devices: [ + { + address: "simlock_android-1-address", + deviceId: "simlock_android-1", + driverData: {}, + runState: "running", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [iosDriver, androidDriver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ + { + deviceId: iosDevice.id, + expected: "stopped", + kind: "foreign-state-change", + observed: "running", + platform: "ios", + }, + { + deviceId: androidDevice.id, + expected: "stopped", + kind: "foreign-state-change", + observed: "running", + platform: "android", + }, + ]); + }); + + it("does not report foreign-state-change for a device this daemon holds an operation claim on", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + // why: A lease-path boot from `shutdown` runs `makeReady` while the committed record still + // says `shutdown` -- observed reality goes `running` before that boot's own transition + // commits. Claiming the device is what tells doctor this is in-flight work, not drift. + const iosDevice = await shutdownDevice(registry, "simlock-ios-1", "ios"); + + const iosDriver = new FakeDriver({ clock, platform: "ios" }); + iosDriver.setManagedReality({ + devices: [ + { + address: "simlock-ios-1-address", + deviceId: "simlock-ios-1", + driverData: {}, + runState: "running", + }, + ], + processes: [], + }); + + const claims = new DeviceOperationClaims(); + const claim = claims.tryClaim(iosDevice.id, "boot"); + expect(claim).toBeDefined(); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + claims, + clock, + config: config(), + drivers: [iosDriver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( + [], + ); + }); + + it("reports foreign-state-change for a device shut down outside Simlock, on both platforms", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const iosDevice = await readyDevice(registry, "simlock-ios-2", "ios"); + const androidDevice = await readyDevice(registry, "simlock_android-2", "android"); + + const iosDriver = new FakeDriver({ clock, platform: "ios" }); + iosDriver.setManagedReality({ + devices: [ + { + address: "simlock-ios-2-address", + deviceId: "simlock-ios-2", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + const androidDriver = new FakeDriver({ clock, platform: "android" }); + androidDriver.setManagedReality({ + devices: [ + { + address: "simlock_android-2-address", + deviceId: "simlock_android-2", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [iosDriver, androidDriver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ + { + deviceId: iosDevice.id, + expected: "running", + kind: "foreign-state-change", + observed: "stopped", + platform: "ios", + }, + { + deviceId: androidDevice.id, + expected: "running", + kind: "foreign-state-change", + observed: "stopped", + platform: "android", + }, + ]); + }); + + it("never reports drift when the observed state is transitioning", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + await readyDevice(registry, "simlock-transition", "ios"); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-transition-address", + deviceId: "simlock-transition", + driverData: {}, + runState: "transitioning", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( + [], + ); + }); + + it("never reports drift for provisioning or reclaiming registry devices", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + await registry.registerDevice({ + driverData: { fakeDeviceId: "simlock-provisioning" }, + driverDeviceId: "simlock-provisioning", + provisionDuration: 0, + spec: { physical: false, model: "Phone", osVersion: "1", platform: "ios" }, + }); + const reclaiming = await readyDevice(registry, "simlock-reclaiming", "ios"); + const lease = await registry.createLease({ + deviceId: reclaiming.id, + requesterId: "agent", + ownerId: "agent", + ttlMs: 60_000, + ttlDeadline: 999_999, + }); + await registry.beginRelease(lease.id); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-provisioning-address", + deviceId: "simlock-provisioning", + driverData: {}, + runState: "stopped", + }, + { + address: "simlock-reclaiming-address", + deviceId: "simlock-reclaiming", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( + [], + ); + }); + + it("reports no live-drift finding for a quarantined simulator while its retry runs, and registry-device-missing once the fake no longer lists it", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const device = await readyDevice(registry, "simlock-quarantined", "ios"); + const lease = await registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlDeadline: 99_999, + ttlMs: 60_000, + }); + await registry.beginRelease(lease.id); + await registry.enterQuarantine(device.id, 10_100, { kind: "purge-failed" }); + const driver = new FakeDriver({ clock, platform: "ios" }); + const listed = { + devices: [ + { + address: "simlock-quarantined-address", + deviceId: "simlock-quarantined", + driverData: {}, + runState: "running" as const, + }, + ], + processes: [], + }; + driver.setManagedReality(listed); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const original = driver.destroy.bind(driver); + vi.spyOn(driver, "destroy").mockImplementation(async (target) => { + await gate; + return original(target); + }); + const claims = new DeviceOperationClaims(); + const quarantine = new QuarantineCoordinator({ + claims, + clock, + config: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + decisions: new SerializedDecision(), + drivers: new DriverCatalog([driver]), + eventBus, + notifyAvailability: () => {}, + registry, + }); + quarantine.restore(); + clock.advance(100); + await vi.waitFor(() => expect(claims.isClaimed(device.id)).toBe(true)); + const reconcile = () => + new Doctor({ + decisions: new SerializedDecision(), + claims, + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }) + .reconcile() + .then((report) => + report.findings.filter( + (finding) => "deviceId" in finding && finding.deviceId === device.id, + ), + ); + + const whileListed = await reconcile(); + driver.setManagedReality({ devices: [], processes: [] }); + const whileGone = await reconcile(); + release(); + + expect(whileListed).toEqual([]); + expect(whileGone).toEqual([ + expect.objectContaining({ kind: "registry-device-missing", platform: "ios" }), + ]); + quarantine.dispose(); + }); + + it("--fix reconciles an unleased device's state to observed reality in both directions", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const bootedOutside = await shutdownDevice(registry, "simlock-booted", "ios"); + const shutdownOutside = await readyDevice(registry, "simlock-shutdown", "ios"); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-booted-address", + deviceId: "simlock-booted", + driverData: {}, + runState: "running", + }, + { + address: "simlock-shutdown-address", + deviceId: "simlock-shutdown", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + + await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + fix: true, + }); + + const snapshot = registry.snapshot; + const booted = snapshot.devices.find((device) => device.id === bootedOutside.id); + const shutdown = snapshot.devices.find((device) => device.id === shutdownOutside.id); + expect(booted?.state).toBe("ready"); + expect(booted?.foreignStateDetectedAt).toBeUndefined(); + expect(shutdown?.state).toBe("shutdown"); + expect(shutdown?.foreignStateDetectedAt).toBeUndefined(); + }); + + it("--fix keeps a spent fresh device shutdown when it is observed running, so its delete still finds it", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + leaseIdentity: { android: "reusable", ios: "fresh" }, + }); + const spent = await registry.registerDevice({ + driverData: { fakeDeviceId: "simlock-spent" }, + driverDeviceId: "simlock-spent", + provisionDuration: 0, + spec: { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" }, + }); + await registry.transitionDevice(spent.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: spent.id }, + }); + const lease = await registry.createLease({ + deviceId: spent.id, + ownerId: "gone", + requesterId: "gone", + ttlDeadline: 60_000, + ttlMs: 60_000, + }); + await registry.beginRelease(lease.id); + await registry.completeReclaimWithoutPurge(spent.id); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "spent-address", + deviceId: "simlock-spent", + driverData: {}, + runState: "running", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ fix: true }); + + expect(report.findings.map((finding) => finding.kind)).toContain("foreign-state-change"); + expect(registry.snapshot.devices.find((device) => device.id === spent.id)?.state).toBe( + "shutdown", + ); + }); + + it("--fix leaves a leased device untouched while still reporting the finding", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const device = await readyDevice(registry, "simlock-leased", "ios"); + await registry.createLease({ + deviceId: device.id, + requesterId: "agent", + ownerId: "agent", + ttlMs: 60_000, + ttlDeadline: 999_999, + }); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-leased-address", + deviceId: "simlock-leased", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + fix: true, + }); + + expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ + { + deviceId: device.id, + expected: "running", + kind: "foreign-state-change", + observed: "stopped", + platform: "ios", + }, + ]); + const snapshot = registry.snapshot; + const stillLeased = snapshot.devices.find((candidate) => candidate.id === device.id); + expect(stillLeased?.state).toBe("leased"); + expect(stillLeased?.foreignStateDetectedAt).toBeDefined(); + }); + + it("emits device.foreign-state-detected once per finding, post-commit", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const bootedOutside = await shutdownDevice(registry, "simlock-booted", "ios"); + const shutdownOutside = await readyDevice(registry, "simlock-shutdown", "ios"); + + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-booted-address", + deviceId: "simlock-booted", + driverData: {}, + runState: "running", + }, + { + address: "simlock-shutdown-address", + deviceId: "simlock-shutdown", + driverData: {}, + runState: "stopped", + }, + ], + processes: [], + }); + + await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + fix: true, + }); + + const events = eventBus.replay(); + const foreignStateEvents = events.filter( + (event) => event.event === "device.foreign-state-detected", + ); + expect(foreignStateEvents).toHaveLength(2); + expect(foreignStateEvents).toEqual([ + expect.objectContaining({ + payload: { + deviceId: bootedOutside.id, + expected: "stopped", + observed: "running", + platform: "ios", + }, + }), + expect.objectContaining({ + payload: { + deviceId: shutdownOutside.id, + expected: "running", + observed: "stopped", + platform: "ios", + }, + }), + ]); + + // why: Post-commit: the registry mutations that resolved the drift are already committed + // by the time the fact is published, and doctor.reconciled is the final event. + const commitEvents = events.filter( + (event) => event.event === "device.ready" || event.event === "device.shutdown", + ); + const lastCommitSeq = Math.max(...commitEvents.map((event) => event.seq)); + const firstForeignStateSeq = Math.min(...foreignStateEvents.map((event) => event.seq)); + expect(firstForeignStateSeq).toBeGreaterThan(lastCommitSeq); + expect(events.at(-1)?.event).toBe("doctor.reconciled"); + }); +}); diff --git a/src/core/doctor-physical-devices.test.ts b/src/core/doctor-physical-devices.test.ts new file mode 100644 index 000000000..981e19f8d --- /dev/null +++ b/src/core/doctor-physical-devices.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it, vi } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { DeviceOperationClaims } from "./device-operation-claims.js"; +import { Doctor } from "./doctor.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import { FakeDriver } from "./fake-driver.js"; +import { QuarantineCoordinator } from "./quarantine-coordinator.js"; +import { type Registry } from "./registry.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { loadTestRegistry } from "./testing.js"; +import { config, sequence } from "./test-doctor.js"; + +// why: pure logic with many cases: Doctor's drift, stall and finding classification, built +// directly over a fake driver and a test registry rather than a whole daemon. +describe("Doctor and physical devices (ADR 0022 §4)", () => { + async function physicalHarness(states: readonly ("ready" | "leased" | "reclaiming")[]) { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + for (const state of states) { + const device = await registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: `00008110-${state}`, + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + if (state === "ready") continue; + const lease = await registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlDeadline: 10_000_000, + ttlMs: 60_000, + }); + if (state === "reclaiming") await registry.beginRelease(lease.id); + } + clock.advance(99_000); + const driver = new FakeDriver({ + clock, + estimateMs: { boot: 2_000, provision: 1_000, reclaim: 1_000 }, + platform: "ios", + }); + driver.setManagedReality({ devices: [], processes: [] }); + return { clock, driver, eventBus, registry }; + } + + it("raises no drift finding for a ready or a leased physical record that no root holds, and --fix leaves both as they are", async () => { + const { clock, driver, eventBus, registry } = await physicalHarness(["ready", "leased"]); + const destroy = vi.spyOn(driver, "destroy"); + const shutdown = vi.spyOn(driver, "shutdown"); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ fix: true }); + + expect(report.findings).toEqual([]); + expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["ready", "leased"]); + expect(destroy).not.toHaveBeenCalled(); + expect(shutdown).not.toHaveBeenCalled(); + }); + + it("does not report as an orphan the ID a physical record holds, and a purge destroys only the orphan, as a virtual driver device", async () => { + const { clock, driver, eventBus, registry } = await physicalHarness(["ready"]); + driver.setManagedReality({ + devices: [ + { + address: "a", + deviceId: "00008110-ready", + driverData: {}, + runState: "running", + }, + { address: "b", deviceId: "simlock-orphan", driverData: {}, runState: "running" }, + ], + processes: [], + }); + const destroy = vi.spyOn(driver, "destroy"); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ purgeOrphans: true }); + + expect(report.findings).toEqual([]); + expect(destroy).toHaveBeenCalledTimes(1); + expect(destroy).toHaveBeenCalledWith( + expect.objectContaining({ deviceId: "simlock-orphan", physical: false }), + ); + }); + + it("reports stalled-transition for a reclaiming physical record past its threshold, priced at the physical reclaim estimate, and --fix quarantines it", async () => { + const { clock, driver, eventBus, registry } = await physicalHarness(["reclaiming"]); + const quarantine = new QuarantineCoordinator({ + claims: new DeviceOperationClaims(), + clock, + config: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + decisions: new SerializedDecision(), + drivers: new DriverCatalog([driver]), + eventBus, + notifyAvailability: () => {}, + registry, + }); + const doctor = new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config({ minimumThresholdMs: 0, thresholdMultiplier: 1 }), + drivers: [driver], + eventBus, + quarantine, + registry, + }); + + const report = await doctor.reconcile({ fix: true }); + + expect(report.findings).toEqual([ + expect.objectContaining({ + deviceId: registry.snapshot.devices[0]?.id, + kind: "stalled-transition", + thresholdMs: 90_000, + }), + ]); + expect(registry.snapshot.devices[0]?.state).toBe("quarantined"); + quarantine.dispose(); + }); +}); + +describe("Doctor and a physical reclaim in flight (ADR 0022 §7, §8)", () => { + async function harness() { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const driver = new FakeDriver({ + clock, + estimateMs: { boot: 2_000, provision: 1_000, reclaim: 1_000 }, + platform: "ios", + }); + driver.setManagedReality({ devices: [], processes: [] }); + const claims = new DeviceOperationClaims(); + const quarantine = new QuarantineCoordinator({ + claims, + clock, + config: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + decisions: new SerializedDecision(), + drivers: new DriverCatalog([driver]), + eventBus, + notifyAvailability: () => {}, + registry, + }); + const doctor = new Doctor({ + claims, + decisions: new SerializedDecision(), + clock, + config: config({ minimumThresholdMs: 0, thresholdMultiplier: 1 }), + drivers: [driver], + eventBus, + quarantine, + registry, + }); + return { claims, clock, doctor, eventBus, quarantine, registry }; + } + + async function enrollReclaiming(registry: Registry, name: string) { + const device = await registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: `00008110-${name}`, + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + const lease = await registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlDeadline: 10_000_000, + ttlMs: 60_000, + }); + await registry.beginRelease(lease.id); + return device; + } + + it("a physical reclaim interrupted by a killed daemon is aged from reclaimingSince by stalled-transition", async () => { + const { clock, doctor, quarantine, registry } = await harness(); + const oldStart = await enrollReclaiming(registry, "old_start"); + const newStart = await enrollReclaiming(registry, "new_start"); + await registry.transitionDevice( + newStart.id, + "absent", + { + event: "device.absence-detected", + payload: { absentReason: "locked", deviceId: newStart.id, previousState: "reclaiming" }, + }, + { absentReason: "locked" }, + ); + clock.advance(98_000); + await registry.transitionDevice(newStart.id, "reclaiming", { + event: "device.returned", + payload: { deviceId: newStart.id, osVersion: "18.4" }, + }); + clock.advance(1_000); + + const report = await doctor.reconcile(); + + expect( + report.findings + .filter((finding) => finding.kind === "stalled-transition") + .map((finding) => finding.deviceId), + ).toEqual([oldStart.id]); + quarantine.dispose(); + }); + + it("doctor --fix on a stalled reclaiming device that is claimed by the time the fix runs writes nothing, emits no device.quarantined and arms no retry", async () => { + const { claims, clock, doctor, eventBus, quarantine, registry } = await harness(); + const stalled = await enrollReclaiming(registry, "stalled"); + clock.advance(99_000); + let commits = 0; + registry.onCommit(() => { + commits += 1; + }); + let claimTaken = false; + eventBus.subscribe("device.stalled-transition-detected", () => { + claimTaken = claims.tryClaim(stalled.id, "reclaim") !== undefined; + }); + + const report = await doctor.reconcile({ fix: true }); + + expect(report.findings).toContainEqual( + expect.objectContaining({ deviceId: stalled.id, kind: "stalled-transition" }), + ); + expect(claimTaken).toBe(true); + expect(registry.snapshot.devices[0]?.state).toBe("reclaiming"); + expect(commits).toBe(0); + expect(eventBus.replay().filter((event) => event.event === "device.quarantined")).toEqual([]); + expect(clock.pendingTimerCount).toBe(0); + quarantine.dispose(); + }); +}); diff --git a/src/core/doctor-unobservable-platform.test.ts b/src/core/doctor-unobservable-platform.test.ts new file mode 100644 index 000000000..4030b874c --- /dev/null +++ b/src/core/doctor-unobservable-platform.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { Doctor } from "./doctor.js"; +import { FakeDriver } from "./fake-driver.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { config, loadRegistry, readyDevice, readyIosDevice, rootRejection } from "./test-doctor.js"; +import { driverDevice, observed } from "./test-doctor-observed.js"; + +async function legacyDeviceSetup() { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + await readyDevice(registry, "stranded", "ios"); + const driver = new FakeDriver({ + clock, + legacyDevices: { + stranded: { device: driverDevice("stranded"), path: "/Library/Devices/stranded" }, + }, + platform: "ios", + }); + driver.setManagedReality({ devices: [], processes: [] }); + return { clock, driver, eventBus, registry }; +} + +// why: pure logic with many cases: Doctor's drift, stall and finding classification, built +// directly over a fake driver and a test registry rather than a whole daemon. +describe("Doctor with a platform it cannot observe", () => { + it("never reports a registry device missing because no driver could look for it", async () => { + const { eventBus, registry } = await readyIosDevice(); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock: new FakeClock(10_000), + config: config(), + drivers: [], + driverRejections: [rootRejection()], + eventBus, + registry, + }).reconcile({ fix: true }); + + // why: "I could not look" is not "the device is gone". Marking these `deleted` would strand + // every simulator in the root behind a registry with no record of it -- the permanent + // multi-gigabyte leak ADR 0001 exists to prevent, reachable with a `chmod`. + expect(report.findings.map((finding) => finding.kind)).toEqual(["driver-unavailable"]); + expect(registry.snapshot.devices[0]?.state).toBe("ready"); + }); + + it("stays silent about a platform that simply has no driver on this host", async () => { + const { eventBus, registry } = await readyIosDevice(); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock: new FakeClock(10_000), + config: config(), + drivers: [], + eventBus, + registry, + }).reconcile({ fix: true }); + + expect(report.findings).toEqual([]); + expect(registry.snapshot.devices[0]?.state).toBe("ready"); + }); + + it("destroys orphans only when the purge is asked for, and reports what it destroyed", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const driver = new FakeDriver({ clock, deviceRoot: "/roots/ios", platform: "ios" }); + driver.setManagedReality({ + devices: [observed("orphan-1", "running")], + processes: [driverDevice("orphan-1")], + }); + const doctor = new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }); + + const reported = await doctor.reconcile({ fix: true }); + expect(reported.findings.map((finding) => finding.kind)).toEqual([ + "orphan-device", + "orphan-process", + ]); + expect(driver.calls.filter((call) => call.operation === "destroy")).toEqual([]); + + const purged = await doctor.reconcile({ purgeOrphans: true }); + + expect(purged.findings).toEqual([]); + expect(eventBus.replay()).toContainEqual( + expect.objectContaining({ + event: "device.orphan-purged", + payload: { deviceRoot: "/roots/ios", driverDeviceId: "orphan-1", platform: "ios" }, + }), + ); + }); + + it("re-proves the root before the first destroy of a purge", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ devices: [observed("orphan-1", "stopped")], processes: [] }); + + await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + purgeOrphans: true, + }); + + expect( + driver.calls.map((call) => call.operation).filter((operation) => operation !== "listSources"), + ).toEqual(["listManaged", "revalidateRoot", "destroy"]); + }); + + it("destroys nothing at all when a root can no longer be proven", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const iosDriver = new FakeDriver({ clock, platform: "ios" }); + iosDriver.setManagedReality({ devices: [observed("ios-orphan", "stopped")], processes: [] }); + const androidDriver = new FakeDriver({ clock, platform: "android" }); + androidDriver.setManagedReality({ + devices: [observed("android-orphan", "stopped")], + processes: [], + }); + androidDriver.failOn("revalidateRoot", 1, new Error("Refusing the android device root")); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [iosDriver, androidDriver], + eventBus, + registry, + }).reconcile({ purgeOrphans: true }); + + // why: A daemon that cannot prove one of its roots is not one to keep destroying on, and + // the roots are proven before anything is destroyed -- so this costs a re-run, never a + // half-purge. + expect( + [...iosDriver.calls, ...androidDriver.calls].filter((call) => call.operation === "destroy"), + ).toEqual([]); + expect(report.findings.map((finding) => finding.kind)).toEqual([ + "orphan-device", + "orphan-device", + ]); + expect(eventBus.replay().some((event) => event.event === "device.orphan-purged")).toBe(false); + }); + + it("leaves an orphan it could not destroy standing and purges the rest", async () => { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [observed("stubborn", "stopped"), observed("orphan-2", "stopped")], + processes: [], + }); + driver.failOn("destroy", 1, new Error("simctl delete failed")); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ purgeOrphans: true }); + + expect(report.findings).toEqual([ + { + device: expect.objectContaining({ deviceId: "stubborn" }), + kind: "orphan-device", + platform: "ios", + }, + ]); + expect( + eventBus.replay().filter((event) => event.event === "device.orphan-purged"), + ).toHaveLength(1); + }); + + it("reports a device left in the pre-root location as legacy rather than missing", async () => { + const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile(); + + expect(report.findings).toEqual([ + { + device: expect.objectContaining({ deviceId: "stranded" }), + deviceId: registry.snapshot.devices[0]?.id, + kind: "legacy-device", + path: "/Library/Devices/stranded", + platform: "ios", + }, + ]); + }); + + it("destroys a legacy device through its old path and then records it missing", async () => { + const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); + + await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + fix: true, + }); + + expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toHaveLength(1); + expect(registry.snapshot.devices[0]?.state).toBe("deleted"); + }); + + it("never destroys a legacy device that a lease still references", async () => { + const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); + const device = registry.snapshot.devices[0]!; + await registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlMs: 60_000, + ttlDeadline: 90_000, + }); + + await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ + fix: true, + }); + + expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toEqual([]); + expect(registry.snapshot.devices[0]?.state).toBe("leased"); + }); + + it("reports a missing device as missing when the legacy lookup itself fails", async () => { + const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); + driver.failOn("findLegacy", 1, new Error("simctl list failed")); + + const report = await new Doctor({ + decisions: new SerializedDecision(), + clock, + config: config(), + drivers: [driver], + eventBus, + registry, + }).reconcile({ fix: true }); + + // why: "I could not look outside the root" is not "it is out there": the conservative + // finding is the one whose fix only writes to the registry. + expect(report.findings.map((finding) => finding.kind)).toEqual(["registry-device-missing"]); + expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toEqual([]); + }); +}); diff --git a/src/core/doctor-warm-pool-targets.test.ts b/src/core/doctor-warm-pool-targets.test.ts new file mode 100644 index 000000000..75196fc88 --- /dev/null +++ b/src/core/doctor-warm-pool-targets.test.ts @@ -0,0 +1,453 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { type Config } from "./config.js"; +import { Doctor, type DoctorFinding } from "./doctor.js"; +import { FakeDriver } from "./fake-driver.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { config, loadRegistry } from "./test-doctor.js"; + +// why: pure logic with many cases: Doctor's drift, stall and finding classification, built +// directly over a fake driver and a test registry rather than a whole daemon. +describe("Doctor: warm pool targets", () => { + const entry = (maxRunning: number) => ({ + maxRunning, + overLimit: false, + reserved: 0, + running: 0, + }); + const limits = (limit: { ios?: number; android?: number; global?: number }) => ({ + android: entry(limit.android ?? 10), + global: entry(limit.global ?? 20), + ios: entry(limit.ios ?? 10), + }); + const figuresOf = ( + targets: readonly { + model: string; + origin?: "configured" | "learned"; + osVersion?: string; + platform?: "ios" | "android"; + short?: "runtime-missing" | "unknown-model" | "ram-budget"; + }[], + ) => ({ + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + targets: targets.map((target) => ({ + booting: 0, + count: 1, + mode: "full" as const, + origin: "configured" as const, + platform: "ios" as const, + ready: 0, + ...target, + })), + }); + + async function findings(options: { + readonly targets?: Config["warmPool"]["targets"]; + readonly reserve?: { ios: number; android: number }; + readonly enabled?: boolean; + readonly limit?: { ios?: number; android?: number; global?: number }; + readonly figures?: ReturnType; + readonly driver?: Omit[0], "clock">; + }): Promise { + const clock = new FakeClock(10_000); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const base = config(); + const doctor = new Doctor({ + decisions: new SerializedDecision(), + clock, + config: { + ...base, + warmPool: { + ...base.warmPool, + enabled: options.enabled ?? true, + reserveRunning: options.reserve ?? { android: 0, ios: 0 }, + targets: options.targets ?? [], + }, + }, + drivers: [new FakeDriver({ clock, platform: "ios", ...options.driver })], + eventBus, + registry, + warmPool: { + figures: () => options.figures ?? figuresOf([]), + runningCapacity: () => limits(options.limit ?? {}), + }, + }); + const report = await doctor.reconcile(); + return report.findings.filter((finding) => finding.kind === "warm-pool-target-unreachable"); + } + + it("reports a configured target short with runtime-missing as unreachable, naming the install command, and no learned entry, even one short runtime-missing", async () => { + const found = await findings({ + figures: figuresOf([ + { model: "iPhone 17", osVersion: "26.0" }, + { model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }, + { model: "iPhone 16", origin: "learned", osVersion: "27.0", short: "runtime-missing" }, + { model: "iPhone 15", origin: "learned", osVersion: "26.0", short: "unknown-model" }, + ]), + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: "iOS 27.0 is not installed", + platform: "ios", + reason: "runtime-missing", + remedy: "run simlock component install ios 27.0", + target: "iPhone 17 / 27.0 / full", + }, + ]); + }); + + it.each([ + ["ios", "iOS", "iPhone 17", ["18.0", "26.0"], "26.0"], + ["android", "Android", "Television (1080p)", ["34", "35"], "35"], + ] as const)( + "names simlock catalog, not component install, for an installed %s version the target's model does not pair with", + async (platform, name, model, runtimes, unpaired) => { + const found = await findings({ + driver: { + availableOsVersions: [...runtimes], + knownModels: [model], + modelRuntimes: { [model]: runtimes.filter((runtime) => runtime !== unpaired) }, + platform, + }, + figures: figuresOf([{ model, osVersion: unpaired, platform, short: "runtime-missing" }]), + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: `${model} does not pair with ${name} ${unpaired}`, + platform, + reason: "runtime-missing", + remedy: `run simlock catalog --platform ${platform}`, + target: `${model} / ${unpaired} / full`, + }, + ]); + }, + ); + + it("keeps the install text for a version the platform's catalog does not list", async () => { + const found = await findings({ + driver: { + availableOsVersions: ["26.0"], + knownModels: ["iPhone 17"], + platform: "ios", + }, + figures: figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + message: "iOS 27.0 is not installed", + remedy: "run simlock component install ios 27.0", + }), + ]); + }); + + it("reports a target short with unknown-model as unreachable", async () => { + const found = await findings({ + figures: figuresOf([{ model: "iPhone 99", osVersion: "26.0", short: "unknown-model" }]), + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + kind: "warm-pool-target-unreachable", + platform: "ios", + reason: "unknown-model", + target: "iPhone 99 / 26.0 / full", + }), + ]); + }); + + it("reports one finding for two iOS targets of 2 on a running limit of 3, naming the sum and the limit", async () => { + const found = await findings({ + limit: { ios: 3 }, + targets: [ + { count: 2, model: "iPhone 17", platform: "ios" }, + { count: 2, model: "iPhone 16", platform: "ios" }, + ], + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: + "the ios targets want 4 running devices, and the ios running limit is 3, leaving room for 3", + platform: "ios", + reason: "over-limit", + remedy: "lower the counts of the ios warmPool.targets, or raise the ios running limit", + target: "ios targets", + }, + ]); + }); + + it("names the reserve a platform's limit holds back, and offers lowering it", async () => { + const found = await findings({ + limit: { ios: 3 }, + reserve: { android: 0, ios: 1 }, + targets: [{ count: 3, model: "iPhone 17", platform: "ios" }], + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + message: + "the ios targets want 3 running devices, and the ios running limit is 3, of which warmPool.reserveRunning holds 1, leaving room for 2", + remedy: + "lower the counts of the ios warmPool.targets, raise the ios running limit, or lower warmPool.reserveRunning", + }), + ]); + }); + + it("names the summed reserves in the machine's finding", async () => { + const found = await findings({ + limit: { android: 5, global: 4, ios: 5 }, + reserve: { android: 1, ios: 1 }, + targets: [ + { count: 2, model: "iPhone 17", platform: "ios" }, + { count: 1, model: "Pixel 8", platform: "android" }, + ], + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + message: + "the targets want 3 running devices, and the running limit is 4, of which warmPool.reserveRunning holds 2, leaving room for 2", + target: "all targets", + }), + ]); + }); + + it("caps a platform's reserve at its running limit, so a platform with no targets is not over it", async () => { + const noTargets = await findings({ + limit: { global: 5, ios: 2 }, + reserve: { android: 0, ios: 4 }, + }); + expect(noTargets).toEqual([]); + expect( + await findings({ + limit: { global: 5, ios: 2 }, + reserve: { android: 0, ios: 4 }, + targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], + }), + ).toEqual([ + expect.objectContaining({ + message: + "the ios targets want 1 running devices, and the ios running limit is 2, of which warmPool.reserveRunning holds 2, leaving room for 0", + platform: "ios", + }), + ]); + }); + + it("reports nothing for reserves that add up past the machine's limit when no target is configured", async () => { + const limit = { android: 5, global: 4, ios: 5 }; + const reserve = { android: 2, ios: 3 }; + + expect(await findings({ limit, reserve })).toEqual([]); + }); + + it("caps the machine's reserve at its running limit, so a target past it leaves room for zero and never below", async () => { + const found = await findings({ + limit: { android: 5, global: 4, ios: 5 }, + reserve: { android: 2, ios: 3 }, + targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + message: + "the targets want 1 running devices, and the running limit is 4, of which warmPool.reserveRunning holds 4, leaving room for 0", + target: "all targets", + }), + ]); + }); + + it("prints no install command for a target whose OS is a range, which names no version to install", async () => { + const found = await findings({ + figures: figuresOf([{ model: "iPhone 17", osVersion: ">=27", short: "runtime-missing" }]), + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: "no installed iOS runtime satisfies >=27", + platform: "ios", + reason: "runtime-missing", + remedy: + "run simlock component list --platform ios to see the versions, then simlock component install ios for one inside >=27", + target: "iPhone 17 / >=27 / full", + }, + ]); + }); + + it("takes the platform's reserve off the room its targets have", async () => { + const targets = [{ count: 3, model: "iPhone 17", platform: "ios" as const }]; + + expect(await findings({ limit: { ios: 3 }, reserve: { android: 0, ios: 1 }, targets })).toEqual( + [expect.objectContaining({ platform: "ios", reason: "over-limit" })], + ); + expect(await findings({ limit: { ios: 3 }, targets })).toEqual([]); + }); + + it("reports one finding for targets that fit each platform but not the machine's running limit together", async () => { + const found = await findings({ + limit: { android: 5, global: 3, ios: 5 }, + targets: [ + { count: 2, model: "iPhone 17", platform: "ios" }, + { count: 2, model: "Pixel 8", platform: "android" }, + ], + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: + "the targets want 4 running devices, and the running limit is 3, leaving room for 3", + reason: "over-limit", + remedy: "lower the counts of the warmPool.targets, or raise the running limit", + target: "all targets", + }, + ]); + }); + + it("pins the unknown-model message and remedy, naming the model and the platform", async () => { + const found = await findings({ + figures: figuresOf([ + { + model: "Pixel 99", + osVersion: "35", + platform: "android", + short: "unknown-model", + }, + ]), + }); + + expect(found).toStrictEqual([ + { + kind: "warm-pool-target-unreachable", + message: "Pixel 99 is not a known Android model", + platform: "android", + reason: "unknown-model", + remedy: "run simlock catalog --platform android to list the models", + target: "Pixel 99 / 35 / full", + }, + ]); + }); + + it("names an Android runtime as Android, and a target that names no OS without one", async () => { + const found = await findings({ + figures: figuresOf([ + { + model: "Pixel 8", + osVersion: "35", + platform: "android", + short: "runtime-missing", + }, + { model: "iPhone 17", short: "runtime-missing" }, + { model: "iPhone 99", short: "unknown-model" }, + ]), + }); + + expect(found).toStrictEqual([ + expect.objectContaining({ + message: "Android 35 is not installed", + remedy: "run simlock component install android 35", + }), + { + kind: "warm-pool-target-unreachable", + message: "iOS runtime is not installed", + platform: "ios", + reason: "runtime-missing", + remedy: + "run simlock component list --platform ios to see the versions, then simlock component install ios ", + target: "iPhone 17 / full", + }, + expect.objectContaining({ target: "iPhone 99 / full" }), + ]); + }); + + it("takes both reserves off the machine's room, and reports nothing for targets that exactly fill what is left", async () => { + const reserve = { android: 1, ios: 1 }; + const targets = (ios: number) => [ + { count: ios, model: "iPhone 17", platform: "ios" as const }, + { count: 1, model: "Pixel 8", platform: "android" as const }, + ]; + const limit = { android: 5, global: 4, ios: 5 }; + + expect(await findings({ limit, reserve, targets: targets(2) })).toEqual([ + expect.objectContaining({ reason: "over-limit", target: "all targets" }), + ]); + expect(await findings({ limit, reserve, targets: targets(1) })).toEqual([]); + expect( + await findings({ + limit: { ios: 3 }, + targets: [{ count: 3, model: "iPhone 17", platform: "ios" }], + }), + ).toEqual([]); + }); + + it("does not report a target short for ram-budget, or one that fits", async () => { + const found = await findings({ + figures: figuresOf([{ model: "iPhone 17", osVersion: "26.0", short: "ram-budget" }]), + targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], + }); + + expect(found).toEqual([]); + }); + + it("reports nothing with the pool off", async () => { + const found = await findings({ + enabled: false, + figures: figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), + limit: { ios: 1 }, + targets: [{ count: 5, model: "iPhone 17", platform: "ios" }], + }); + + expect(found).toEqual([]); + }); + + it("leaves the finding out of the doctor.reconciled event's driftFindings, which keeps the drift beside it", async () => { + const clock = new FakeClock(10_000); + const driver = new FakeDriver({ clock, platform: "ios" }); + driver.setManagedReality({ + devices: [ + { + address: "simlock-orphan-address", + deviceId: "simlock-orphan", + driverData: { fakeDeviceId: "simlock-orphan" }, + runState: "running", + }, + ], + processes: [], + }); + const eventBus = new EventBus(clock); + const registry = await loadRegistry(clock, eventBus); + const base = config(); + const doctor = new Doctor({ + decisions: new SerializedDecision(), + clock, + config: base, + drivers: [driver], + eventBus, + registry, + warmPool: { + figures: () => + figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), + runningCapacity: () => limits({}), + }, + }); + + const report = await doctor.reconcile(); + + const reconciled = eventBus.replay().find((event) => event.event === "doctor.reconciled"); + expect(report.findings.map((finding) => finding.kind).sort()).toEqual([ + "orphan-device", + "warm-pool-target-unreachable", + ]); + expect(reconciled?.payload).toMatchObject({ driftFindings: [{ kind: "orphan-device" }] }); + expect(reconciled?.payload).toHaveProperty("driftFindings.length", 1); + }); +}); diff --git a/src/core/doctor.test.ts b/src/core/doctor.test.ts index 2617b5114..0318e5f01 100644 --- a/src/core/doctor.test.ts +++ b/src/core/doctor.test.ts @@ -1,26 +1,12 @@ -import { describe, expect, it, vi } from "vitest"; - +import { describe, expect, it } from "vitest"; import { EventBus } from "../bus/index.js"; -import { FakeClock } from "../ports/index.js"; -import { FakeSystemStats } from "../ports/index.js"; -import type { Config } from "./config.js"; -import { DeviceOperationClaims } from "./device-operation-claims.js"; -import { Doctor, type DoctorFinding } from "./doctor.js"; -import { DriverCatalog } from "./driver-catalog.js"; +import { FakeClock, FakeSystemStats } from "../ports/index.js"; +import { Doctor } from "./doctor.js"; import { FakeDriver } from "./fake-driver.js"; -import { QuarantineCoordinator } from "./quarantine-coordinator.js"; -import type { Registry } from "./registry.js"; import { SerializedDecision } from "./serialized-decision.js"; import { loadTestRegistry, testComponentWiring, testStateOf } from "./testing.js"; import { createTestEngine } from "../leasing/testing.js"; -import { - config, - loadRegistry, - readyDevice, - readyIosDevice, - rootRejection, - sequence, -} from "./test-doctor.js"; +import { config, sequence } from "./test-doctor.js"; // why: pure logic with many cases: Doctor's drift, stall and finding classification, built // directly over a fake driver and a test registry rather than a whole daemon. @@ -680,1586 +666,4 @@ describe("Doctor", () => { expect(registry.snapshot.leases).toEqual([]); expect(eventBus.replay()).toContainEqual(expect.objectContaining({ event: "lease.expired" })); }); - - it("reports foreign-state-change for a device booted outside Simlock, on both platforms", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const iosDevice = await shutdownDevice(registry, "simlock-ios-1", "ios"); - const androidDevice = await shutdownDevice(registry, "simlock_android-1", "android"); - - const iosDriver = new FakeDriver({ clock, platform: "ios" }); - iosDriver.setManagedReality({ - devices: [ - { - address: "simlock-ios-1-address", - deviceId: "simlock-ios-1", - driverData: {}, - runState: "running", - }, - ], - processes: [], - }); - const androidDriver = new FakeDriver({ clock, platform: "android" }); - androidDriver.setManagedReality({ - devices: [ - { - address: "simlock_android-1-address", - deviceId: "simlock_android-1", - driverData: {}, - runState: "running", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [iosDriver, androidDriver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ - { - deviceId: iosDevice.id, - expected: "stopped", - kind: "foreign-state-change", - observed: "running", - platform: "ios", - }, - { - deviceId: androidDevice.id, - expected: "stopped", - kind: "foreign-state-change", - observed: "running", - platform: "android", - }, - ]); - }); - - it("does not report foreign-state-change for a device this daemon holds an operation claim on", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - // why: A lease-path boot from `shutdown` runs `makeReady` while the committed record still - // says `shutdown` -- observed reality goes `running` before that boot's own transition - // commits. Claiming the device is what tells doctor this is in-flight work, not drift. - const iosDevice = await shutdownDevice(registry, "simlock-ios-1", "ios"); - - const iosDriver = new FakeDriver({ clock, platform: "ios" }); - iosDriver.setManagedReality({ - devices: [ - { - address: "simlock-ios-1-address", - deviceId: "simlock-ios-1", - driverData: {}, - runState: "running", - }, - ], - processes: [], - }); - - const claims = new DeviceOperationClaims(); - const claim = claims.tryClaim(iosDevice.id, "boot"); - expect(claim).toBeDefined(); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - claims, - clock, - config: config(), - drivers: [iosDriver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( - [], - ); - }); - - it("reports foreign-state-change for a device shut down outside Simlock, on both platforms", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const iosDevice = await readyDevice(registry, "simlock-ios-2", "ios"); - const androidDevice = await readyDevice(registry, "simlock_android-2", "android"); - - const iosDriver = new FakeDriver({ clock, platform: "ios" }); - iosDriver.setManagedReality({ - devices: [ - { - address: "simlock-ios-2-address", - deviceId: "simlock-ios-2", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - const androidDriver = new FakeDriver({ clock, platform: "android" }); - androidDriver.setManagedReality({ - devices: [ - { - address: "simlock_android-2-address", - deviceId: "simlock_android-2", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [iosDriver, androidDriver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ - { - deviceId: iosDevice.id, - expected: "running", - kind: "foreign-state-change", - observed: "stopped", - platform: "ios", - }, - { - deviceId: androidDevice.id, - expected: "running", - kind: "foreign-state-change", - observed: "stopped", - platform: "android", - }, - ]); - }); - - it("never reports drift when the observed state is transitioning", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - await readyDevice(registry, "simlock-transition", "ios"); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-transition-address", - deviceId: "simlock-transition", - driverData: {}, - runState: "transitioning", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( - [], - ); - }); - - it("never reports drift for provisioning or reclaiming registry devices", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - await registry.registerDevice({ - driverData: { fakeDeviceId: "simlock-provisioning" }, - driverDeviceId: "simlock-provisioning", - provisionDuration: 0, - spec: { physical: false, model: "Phone", osVersion: "1", platform: "ios" }, - }); - const reclaiming = await readyDevice(registry, "simlock-reclaiming", "ios"); - const lease = await registry.createLease({ - deviceId: reclaiming.id, - requesterId: "agent", - ownerId: "agent", - ttlMs: 60_000, - ttlDeadline: 999_999, - }); - await registry.beginRelease(lease.id); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-provisioning-address", - deviceId: "simlock-provisioning", - driverData: {}, - runState: "stopped", - }, - { - address: "simlock-reclaiming-address", - deviceId: "simlock-reclaiming", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual( - [], - ); - }); - - it("reports no live-drift finding for a quarantined simulator while its retry runs, and registry-device-missing once the fake no longer lists it", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const device = await readyDevice(registry, "simlock-quarantined", "ios"); - const lease = await registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlDeadline: 99_999, - ttlMs: 60_000, - }); - await registry.beginRelease(lease.id); - await registry.enterQuarantine(device.id, 10_100, { kind: "purge-failed" }); - const driver = new FakeDriver({ clock, platform: "ios" }); - const listed = { - devices: [ - { - address: "simlock-quarantined-address", - deviceId: "simlock-quarantined", - driverData: {}, - runState: "running" as const, - }, - ], - processes: [], - }; - driver.setManagedReality(listed); - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const original = driver.destroy.bind(driver); - vi.spyOn(driver, "destroy").mockImplementation(async (target) => { - await gate; - return original(target); - }); - const claims = new DeviceOperationClaims(); - const quarantine = new QuarantineCoordinator({ - claims, - clock, - config: { - maxRetries: 3, - maxRetryBackoffMs: 300_000, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - }, - decisions: new SerializedDecision(), - drivers: new DriverCatalog([driver]), - eventBus, - notifyAvailability: () => {}, - registry, - }); - quarantine.restore(); - clock.advance(100); - await vi.waitFor(() => expect(claims.isClaimed(device.id)).toBe(true)); - const reconcile = () => - new Doctor({ - decisions: new SerializedDecision(), - claims, - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }) - .reconcile() - .then((report) => - report.findings.filter( - (finding) => "deviceId" in finding && finding.deviceId === device.id, - ), - ); - - const whileListed = await reconcile(); - driver.setManagedReality({ devices: [], processes: [] }); - const whileGone = await reconcile(); - release(); - - expect(whileListed).toEqual([]); - expect(whileGone).toEqual([ - expect.objectContaining({ kind: "registry-device-missing", platform: "ios" }), - ]); - quarantine.dispose(); - }); - - it("--fix reconciles an unleased device's state to observed reality in both directions", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const bootedOutside = await shutdownDevice(registry, "simlock-booted", "ios"); - const shutdownOutside = await readyDevice(registry, "simlock-shutdown", "ios"); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-booted-address", - deviceId: "simlock-booted", - driverData: {}, - runState: "running", - }, - { - address: "simlock-shutdown-address", - deviceId: "simlock-shutdown", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - - await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - fix: true, - }); - - const snapshot = registry.snapshot; - const booted = snapshot.devices.find((device) => device.id === bootedOutside.id); - const shutdown = snapshot.devices.find((device) => device.id === shutdownOutside.id); - expect(booted?.state).toBe("ready"); - expect(booted?.foreignStateDetectedAt).toBeUndefined(); - expect(shutdown?.state).toBe("shutdown"); - expect(shutdown?.foreignStateDetectedAt).toBeUndefined(); - }); - - it("--fix keeps a spent fresh device shutdown when it is observed running, so its delete still finds it", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - leaseIdentity: { android: "reusable", ios: "fresh" }, - }); - const spent = await registry.registerDevice({ - driverData: { fakeDeviceId: "simlock-spent" }, - driverDeviceId: "simlock-spent", - provisionDuration: 0, - spec: { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" }, - }); - await registry.transitionDevice(spent.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: spent.id }, - }); - const lease = await registry.createLease({ - deviceId: spent.id, - ownerId: "gone", - requesterId: "gone", - ttlDeadline: 60_000, - ttlMs: 60_000, - }); - await registry.beginRelease(lease.id); - await registry.completeReclaimWithoutPurge(spent.id); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "spent-address", - deviceId: "simlock-spent", - driverData: {}, - runState: "running", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ fix: true }); - - expect(report.findings.map((finding) => finding.kind)).toContain("foreign-state-change"); - expect(registry.snapshot.devices.find((device) => device.id === spent.id)?.state).toBe( - "shutdown", - ); - }); - - it("--fix leaves a leased device untouched while still reporting the finding", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const device = await readyDevice(registry, "simlock-leased", "ios"); - await registry.createLease({ - deviceId: device.id, - requesterId: "agent", - ownerId: "agent", - ttlMs: 60_000, - ttlDeadline: 999_999, - }); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-leased-address", - deviceId: "simlock-leased", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - fix: true, - }); - - expect(report.findings.filter((finding) => finding.kind === "foreign-state-change")).toEqual([ - { - deviceId: device.id, - expected: "running", - kind: "foreign-state-change", - observed: "stopped", - platform: "ios", - }, - ]); - const snapshot = registry.snapshot; - const stillLeased = snapshot.devices.find((candidate) => candidate.id === device.id); - expect(stillLeased?.state).toBe("leased"); - expect(stillLeased?.foreignStateDetectedAt).toBeDefined(); - }); - - it("emits device.foreign-state-detected once per finding, post-commit", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const bootedOutside = await shutdownDevice(registry, "simlock-booted", "ios"); - const shutdownOutside = await readyDevice(registry, "simlock-shutdown", "ios"); - - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-booted-address", - deviceId: "simlock-booted", - driverData: {}, - runState: "running", - }, - { - address: "simlock-shutdown-address", - deviceId: "simlock-shutdown", - driverData: {}, - runState: "stopped", - }, - ], - processes: [], - }); - - await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - fix: true, - }); - - const events = eventBus.replay(); - const foreignStateEvents = events.filter( - (event) => event.event === "device.foreign-state-detected", - ); - expect(foreignStateEvents).toHaveLength(2); - expect(foreignStateEvents).toEqual([ - expect.objectContaining({ - payload: { - deviceId: bootedOutside.id, - expected: "stopped", - observed: "running", - platform: "ios", - }, - }), - expect.objectContaining({ - payload: { - deviceId: shutdownOutside.id, - expected: "running", - observed: "stopped", - platform: "ios", - }, - }), - ]); - - // why: Post-commit: the registry mutations that resolved the drift are already committed - // by the time the fact is published, and doctor.reconciled is the final event. - const commitEvents = events.filter( - (event) => event.event === "device.ready" || event.event === "device.shutdown", - ); - const lastCommitSeq = Math.max(...commitEvents.map((event) => event.seq)); - const firstForeignStateSeq = Math.min(...foreignStateEvents.map((event) => event.seq)); - expect(firstForeignStateSeq).toBeGreaterThan(lastCommitSeq); - expect(events.at(-1)?.event).toBe("doctor.reconciled"); - }); -}); - -describe("Doctor with a platform it cannot observe", () => { - it("never reports a registry device missing because no driver could look for it", async () => { - const { eventBus, registry } = await readyIosDevice(); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock: new FakeClock(10_000), - config: config(), - drivers: [], - driverRejections: [rootRejection()], - eventBus, - registry, - }).reconcile({ fix: true }); - - // why: "I could not look" is not "the device is gone". Marking these `deleted` would strand - // every simulator in the root behind a registry with no record of it -- the permanent - // multi-gigabyte leak ADR 0001 exists to prevent, reachable with a `chmod`. - expect(report.findings.map((finding) => finding.kind)).toEqual(["driver-unavailable"]); - expect(registry.snapshot.devices[0]?.state).toBe("ready"); - }); - - it("stays silent about a platform that simply has no driver on this host", async () => { - const { eventBus, registry } = await readyIosDevice(); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock: new FakeClock(10_000), - config: config(), - drivers: [], - eventBus, - registry, - }).reconcile({ fix: true }); - - expect(report.findings).toEqual([]); - expect(registry.snapshot.devices[0]?.state).toBe("ready"); - }); - - it("destroys orphans only when the purge is asked for, and reports what it destroyed", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const driver = new FakeDriver({ clock, deviceRoot: "/roots/ios", platform: "ios" }); - driver.setManagedReality({ - devices: [observed("orphan-1", "running")], - processes: [driverDevice("orphan-1")], - }); - const doctor = new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }); - - const reported = await doctor.reconcile({ fix: true }); - expect(reported.findings.map((finding) => finding.kind)).toEqual([ - "orphan-device", - "orphan-process", - ]); - expect(driver.calls.filter((call) => call.operation === "destroy")).toEqual([]); - - const purged = await doctor.reconcile({ purgeOrphans: true }); - - expect(purged.findings).toEqual([]); - expect(eventBus.replay()).toContainEqual( - expect.objectContaining({ - event: "device.orphan-purged", - payload: { deviceRoot: "/roots/ios", driverDeviceId: "orphan-1", platform: "ios" }, - }), - ); - }); - - it("re-proves the root before the first destroy of a purge", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ devices: [observed("orphan-1", "stopped")], processes: [] }); - - await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - purgeOrphans: true, - }); - - expect( - driver.calls.map((call) => call.operation).filter((operation) => operation !== "listSources"), - ).toEqual(["listManaged", "revalidateRoot", "destroy"]); - }); - - it("destroys nothing at all when a root can no longer be proven", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const iosDriver = new FakeDriver({ clock, platform: "ios" }); - iosDriver.setManagedReality({ devices: [observed("ios-orphan", "stopped")], processes: [] }); - const androidDriver = new FakeDriver({ clock, platform: "android" }); - androidDriver.setManagedReality({ - devices: [observed("android-orphan", "stopped")], - processes: [], - }); - androidDriver.failOn("revalidateRoot", 1, new Error("Refusing the android device root")); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [iosDriver, androidDriver], - eventBus, - registry, - }).reconcile({ purgeOrphans: true }); - - // why: A daemon that cannot prove one of its roots is not one to keep destroying on, and - // the roots are proven before anything is destroyed -- so this costs a re-run, never a - // half-purge. - expect( - [...iosDriver.calls, ...androidDriver.calls].filter((call) => call.operation === "destroy"), - ).toEqual([]); - expect(report.findings.map((finding) => finding.kind)).toEqual([ - "orphan-device", - "orphan-device", - ]); - expect(eventBus.replay().some((event) => event.event === "device.orphan-purged")).toBe(false); - }); - - it("leaves an orphan it could not destroy standing and purges the rest", async () => { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [observed("stubborn", "stopped"), observed("orphan-2", "stopped")], - processes: [], - }); - driver.failOn("destroy", 1, new Error("simctl delete failed")); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ purgeOrphans: true }); - - expect(report.findings).toEqual([ - { - device: expect.objectContaining({ deviceId: "stubborn" }), - kind: "orphan-device", - platform: "ios", - }, - ]); - expect( - eventBus.replay().filter((event) => event.event === "device.orphan-purged"), - ).toHaveLength(1); - }); - - it("reports a device left in the pre-root location as legacy rather than missing", async () => { - const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile(); - - expect(report.findings).toEqual([ - { - device: expect.objectContaining({ deviceId: "stranded" }), - deviceId: registry.snapshot.devices[0]?.id, - kind: "legacy-device", - path: "/Library/Devices/stranded", - platform: "ios", - }, - ]); - }); - - it("destroys a legacy device through its old path and then records it missing", async () => { - const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); - - await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - fix: true, - }); - - expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toHaveLength(1); - expect(registry.snapshot.devices[0]?.state).toBe("deleted"); - }); - - it("never destroys a legacy device that a lease still references", async () => { - const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); - const device = registry.snapshot.devices[0]!; - await registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlMs: 60_000, - ttlDeadline: 90_000, - }); - - await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ - fix: true, - }); - - expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toEqual([]); - expect(registry.snapshot.devices[0]?.state).toBe("leased"); - }); - - it("reports a missing device as missing when the legacy lookup itself fails", async () => { - const { clock, driver, eventBus, registry } = await legacyDeviceSetup(); - driver.failOn("findLegacy", 1, new Error("simctl list failed")); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ fix: true }); - - // why: "I could not look outside the root" is not "it is out there": the conservative - // finding is the one whose fix only writes to the registry. - expect(report.findings.map((finding) => finding.kind)).toEqual(["registry-device-missing"]); - expect(driver.calls.filter((call) => call.operation === "destroyLegacy")).toEqual([]); - }); -}); - -async function legacyDeviceSetup() { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - await readyDevice(registry, "stranded", "ios"); - const driver = new FakeDriver({ - clock, - legacyDevices: { - stranded: { device: driverDevice("stranded"), path: "/Library/Devices/stranded" }, - }, - platform: "ios", - }); - driver.setManagedReality({ devices: [], processes: [] }); - return { clock, driver, eventBus, registry }; -} - -function driverDevice(deviceId: string) { - return { - physical: false, - address: `${deviceId}-address`, - deviceId, - driverData: { fakeDeviceId: deviceId }, - }; -} - -function observed(deviceId: string, runState: "running" | "stopped") { - return { ...driverDevice(deviceId), runState }; -} - -async function shutdownDevice( - registry: Registry, - driverDeviceId: string, - platform: "ios" | "android", -) { - await readyDevice(registry, driverDeviceId, platform); - const device = registry.snapshot.devices.find( - (candidate) => candidate.driverDeviceId === driverDeviceId, - )!; - await registry.transitionDevice(device.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: device.id, initiator: "test" }, - }); - return registry.snapshot.devices.find((candidate) => candidate.id === device.id)!; -} - -describe("Doctor and physical devices (ADR 0022 §4)", () => { - async function physicalHarness(states: readonly ("ready" | "leased" | "reclaiming")[]) { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - for (const state of states) { - const device = await registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: `00008110-${state}`, - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - if (state === "ready") continue; - const lease = await registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlDeadline: 10_000_000, - ttlMs: 60_000, - }); - if (state === "reclaiming") await registry.beginRelease(lease.id); - } - clock.advance(99_000); - const driver = new FakeDriver({ - clock, - estimateMs: { boot: 2_000, provision: 1_000, reclaim: 1_000 }, - platform: "ios", - }); - driver.setManagedReality({ devices: [], processes: [] }); - return { clock, driver, eventBus, registry }; - } - - it("raises no drift finding for a ready or a leased physical record that no root holds, and --fix leaves both as they are", async () => { - const { clock, driver, eventBus, registry } = await physicalHarness(["ready", "leased"]); - const destroy = vi.spyOn(driver, "destroy"); - const shutdown = vi.spyOn(driver, "shutdown"); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ fix: true }); - - expect(report.findings).toEqual([]); - expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["ready", "leased"]); - expect(destroy).not.toHaveBeenCalled(); - expect(shutdown).not.toHaveBeenCalled(); - }); - - it("does not report as an orphan the ID a physical record holds, and a purge destroys only the orphan, as a virtual driver device", async () => { - const { clock, driver, eventBus, registry } = await physicalHarness(["ready"]); - driver.setManagedReality({ - devices: [ - { - address: "a", - deviceId: "00008110-ready", - driverData: {}, - runState: "running", - }, - { address: "b", deviceId: "simlock-orphan", driverData: {}, runState: "running" }, - ], - processes: [], - }); - const destroy = vi.spyOn(driver, "destroy"); - - const report = await new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config(), - drivers: [driver], - eventBus, - registry, - }).reconcile({ purgeOrphans: true }); - - expect(report.findings).toEqual([]); - expect(destroy).toHaveBeenCalledTimes(1); - expect(destroy).toHaveBeenCalledWith( - expect.objectContaining({ deviceId: "simlock-orphan", physical: false }), - ); - }); - - it("reports stalled-transition for a reclaiming physical record past its threshold, priced at the physical reclaim estimate, and --fix quarantines it", async () => { - const { clock, driver, eventBus, registry } = await physicalHarness(["reclaiming"]); - const quarantine = new QuarantineCoordinator({ - claims: new DeviceOperationClaims(), - clock, - config: { - maxRetries: 3, - maxRetryBackoffMs: 300_000, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - }, - decisions: new SerializedDecision(), - drivers: new DriverCatalog([driver]), - eventBus, - notifyAvailability: () => {}, - registry, - }); - const doctor = new Doctor({ - decisions: new SerializedDecision(), - clock, - config: config({ minimumThresholdMs: 0, thresholdMultiplier: 1 }), - drivers: [driver], - eventBus, - quarantine, - registry, - }); - - const report = await doctor.reconcile({ fix: true }); - - expect(report.findings).toEqual([ - expect.objectContaining({ - deviceId: registry.snapshot.devices[0]?.id, - kind: "stalled-transition", - thresholdMs: 90_000, - }), - ]); - expect(registry.snapshot.devices[0]?.state).toBe("quarantined"); - quarantine.dispose(); - }); -}); - -describe("Doctor and a physical reclaim in flight (ADR 0022 §7, §8)", () => { - async function harness() { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const driver = new FakeDriver({ - clock, - estimateMs: { boot: 2_000, provision: 1_000, reclaim: 1_000 }, - platform: "ios", - }); - driver.setManagedReality({ devices: [], processes: [] }); - const claims = new DeviceOperationClaims(); - const quarantine = new QuarantineCoordinator({ - claims, - clock, - config: { - maxRetries: 3, - maxRetryBackoffMs: 300_000, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - }, - decisions: new SerializedDecision(), - drivers: new DriverCatalog([driver]), - eventBus, - notifyAvailability: () => {}, - registry, - }); - const doctor = new Doctor({ - claims, - decisions: new SerializedDecision(), - clock, - config: config({ minimumThresholdMs: 0, thresholdMultiplier: 1 }), - drivers: [driver], - eventBus, - quarantine, - registry, - }); - return { claims, clock, doctor, eventBus, quarantine, registry }; - } - - async function enrollReclaiming(registry: Registry, name: string) { - const device = await registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: `00008110-${name}`, - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - const lease = await registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlDeadline: 10_000_000, - ttlMs: 60_000, - }); - await registry.beginRelease(lease.id); - return device; - } - - it("a physical reclaim interrupted by a killed daemon is aged from reclaimingSince by stalled-transition", async () => { - const { clock, doctor, quarantine, registry } = await harness(); - const oldStart = await enrollReclaiming(registry, "old_start"); - const newStart = await enrollReclaiming(registry, "new_start"); - await registry.transitionDevice( - newStart.id, - "absent", - { - event: "device.absence-detected", - payload: { absentReason: "locked", deviceId: newStart.id, previousState: "reclaiming" }, - }, - { absentReason: "locked" }, - ); - clock.advance(98_000); - await registry.transitionDevice(newStart.id, "reclaiming", { - event: "device.returned", - payload: { deviceId: newStart.id, osVersion: "18.4" }, - }); - clock.advance(1_000); - - const report = await doctor.reconcile(); - - expect( - report.findings - .filter((finding) => finding.kind === "stalled-transition") - .map((finding) => finding.deviceId), - ).toEqual([oldStart.id]); - quarantine.dispose(); - }); - - it("doctor --fix on a stalled reclaiming device that is claimed by the time the fix runs writes nothing, emits no device.quarantined and arms no retry", async () => { - const { claims, clock, doctor, eventBus, quarantine, registry } = await harness(); - const stalled = await enrollReclaiming(registry, "stalled"); - clock.advance(99_000); - let commits = 0; - registry.onCommit(() => { - commits += 1; - }); - let claimTaken = false; - eventBus.subscribe("device.stalled-transition-detected", () => { - claimTaken = claims.tryClaim(stalled.id, "reclaim") !== undefined; - }); - - const report = await doctor.reconcile({ fix: true }); - - expect(report.findings).toContainEqual( - expect.objectContaining({ deviceId: stalled.id, kind: "stalled-transition" }), - ); - expect(claimTaken).toBe(true); - expect(registry.snapshot.devices[0]?.state).toBe("reclaiming"); - expect(commits).toBe(0); - expect(eventBus.replay().filter((event) => event.event === "device.quarantined")).toEqual([]); - expect(clock.pendingTimerCount).toBe(0); - quarantine.dispose(); - }); -}); - -describe("Doctor: warm pool targets", () => { - const entry = (maxRunning: number) => ({ - maxRunning, - overLimit: false, - reserved: 0, - running: 0, - }); - const limits = (limit: { ios?: number; android?: number; global?: number }) => ({ - android: entry(limit.android ?? 10), - global: entry(limit.global ?? 20), - ios: entry(limit.ios ?? 10), - }); - const figuresOf = ( - targets: readonly { - model: string; - origin?: "configured" | "learned"; - osVersion?: string; - platform?: "ios" | "android"; - short?: "runtime-missing" | "unknown-model" | "ram-budget"; - }[], - ) => ({ - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - targets: targets.map((target) => ({ - booting: 0, - count: 1, - mode: "full" as const, - origin: "configured" as const, - platform: "ios" as const, - ready: 0, - ...target, - })), - }); - - async function findings(options: { - readonly targets?: Config["warmPool"]["targets"]; - readonly reserve?: { ios: number; android: number }; - readonly enabled?: boolean; - readonly limit?: { ios?: number; android?: number; global?: number }; - readonly figures?: ReturnType; - readonly driver?: Omit[0], "clock">; - }): Promise { - const clock = new FakeClock(10_000); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const base = config(); - const doctor = new Doctor({ - decisions: new SerializedDecision(), - clock, - config: { - ...base, - warmPool: { - ...base.warmPool, - enabled: options.enabled ?? true, - reserveRunning: options.reserve ?? { android: 0, ios: 0 }, - targets: options.targets ?? [], - }, - }, - drivers: [new FakeDriver({ clock, platform: "ios", ...options.driver })], - eventBus, - registry, - warmPool: { - figures: () => options.figures ?? figuresOf([]), - runningCapacity: () => limits(options.limit ?? {}), - }, - }); - const report = await doctor.reconcile(); - return report.findings.filter((finding) => finding.kind === "warm-pool-target-unreachable"); - } - - it("reports a configured target short with runtime-missing as unreachable, naming the install command, and no learned entry, even one short runtime-missing", async () => { - const found = await findings({ - figures: figuresOf([ - { model: "iPhone 17", osVersion: "26.0" }, - { model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }, - { model: "iPhone 16", origin: "learned", osVersion: "27.0", short: "runtime-missing" }, - { model: "iPhone 15", origin: "learned", osVersion: "26.0", short: "unknown-model" }, - ]), - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: "iOS 27.0 is not installed", - platform: "ios", - reason: "runtime-missing", - remedy: "run simlock component install ios 27.0", - target: "iPhone 17 / 27.0 / full", - }, - ]); - }); - - it.each([ - ["ios", "iOS", "iPhone 17", ["18.0", "26.0"], "26.0"], - ["android", "Android", "Television (1080p)", ["34", "35"], "35"], - ] as const)( - "names simlock catalog, not component install, for an installed %s version the target's model does not pair with", - async (platform, name, model, runtimes, unpaired) => { - const found = await findings({ - driver: { - availableOsVersions: [...runtimes], - knownModels: [model], - modelRuntimes: { [model]: runtimes.filter((runtime) => runtime !== unpaired) }, - platform, - }, - figures: figuresOf([{ model, osVersion: unpaired, platform, short: "runtime-missing" }]), - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: `${model} does not pair with ${name} ${unpaired}`, - platform, - reason: "runtime-missing", - remedy: `run simlock catalog --platform ${platform}`, - target: `${model} / ${unpaired} / full`, - }, - ]); - }, - ); - - it("keeps the install text for a version the platform's catalog does not list", async () => { - const found = await findings({ - driver: { - availableOsVersions: ["26.0"], - knownModels: ["iPhone 17"], - platform: "ios", - }, - figures: figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - message: "iOS 27.0 is not installed", - remedy: "run simlock component install ios 27.0", - }), - ]); - }); - - it("reports a target short with unknown-model as unreachable", async () => { - const found = await findings({ - figures: figuresOf([{ model: "iPhone 99", osVersion: "26.0", short: "unknown-model" }]), - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - kind: "warm-pool-target-unreachable", - platform: "ios", - reason: "unknown-model", - target: "iPhone 99 / 26.0 / full", - }), - ]); - }); - - it("reports one finding for two iOS targets of 2 on a running limit of 3, naming the sum and the limit", async () => { - const found = await findings({ - limit: { ios: 3 }, - targets: [ - { count: 2, model: "iPhone 17", platform: "ios" }, - { count: 2, model: "iPhone 16", platform: "ios" }, - ], - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: - "the ios targets want 4 running devices, and the ios running limit is 3, leaving room for 3", - platform: "ios", - reason: "over-limit", - remedy: "lower the counts of the ios warmPool.targets, or raise the ios running limit", - target: "ios targets", - }, - ]); - }); - - it("names the reserve a platform's limit holds back, and offers lowering it", async () => { - const found = await findings({ - limit: { ios: 3 }, - reserve: { android: 0, ios: 1 }, - targets: [{ count: 3, model: "iPhone 17", platform: "ios" }], - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - message: - "the ios targets want 3 running devices, and the ios running limit is 3, of which warmPool.reserveRunning holds 1, leaving room for 2", - remedy: - "lower the counts of the ios warmPool.targets, raise the ios running limit, or lower warmPool.reserveRunning", - }), - ]); - }); - - it("names the summed reserves in the machine's finding", async () => { - const found = await findings({ - limit: { android: 5, global: 4, ios: 5 }, - reserve: { android: 1, ios: 1 }, - targets: [ - { count: 2, model: "iPhone 17", platform: "ios" }, - { count: 1, model: "Pixel 8", platform: "android" }, - ], - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - message: - "the targets want 3 running devices, and the running limit is 4, of which warmPool.reserveRunning holds 2, leaving room for 2", - target: "all targets", - }), - ]); - }); - - it("caps a platform's reserve at its running limit, so a platform with no targets is not over it", async () => { - const noTargets = await findings({ - limit: { global: 5, ios: 2 }, - reserve: { android: 0, ios: 4 }, - }); - expect(noTargets).toEqual([]); - expect( - await findings({ - limit: { global: 5, ios: 2 }, - reserve: { android: 0, ios: 4 }, - targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], - }), - ).toEqual([ - expect.objectContaining({ - message: - "the ios targets want 1 running devices, and the ios running limit is 2, of which warmPool.reserveRunning holds 2, leaving room for 0", - platform: "ios", - }), - ]); - }); - - it("reports nothing for reserves that add up past the machine's limit when no target is configured", async () => { - const limit = { android: 5, global: 4, ios: 5 }; - const reserve = { android: 2, ios: 3 }; - - expect(await findings({ limit, reserve })).toEqual([]); - }); - - it("caps the machine's reserve at its running limit, so a target past it leaves room for zero and never below", async () => { - const found = await findings({ - limit: { android: 5, global: 4, ios: 5 }, - reserve: { android: 2, ios: 3 }, - targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - message: - "the targets want 1 running devices, and the running limit is 4, of which warmPool.reserveRunning holds 4, leaving room for 0", - target: "all targets", - }), - ]); - }); - - it("prints no install command for a target whose OS is a range, which names no version to install", async () => { - const found = await findings({ - figures: figuresOf([{ model: "iPhone 17", osVersion: ">=27", short: "runtime-missing" }]), - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: "no installed iOS runtime satisfies >=27", - platform: "ios", - reason: "runtime-missing", - remedy: - "run simlock component list --platform ios to see the versions, then simlock component install ios for one inside >=27", - target: "iPhone 17 / >=27 / full", - }, - ]); - }); - - it("takes the platform's reserve off the room its targets have", async () => { - const targets = [{ count: 3, model: "iPhone 17", platform: "ios" as const }]; - - expect(await findings({ limit: { ios: 3 }, reserve: { android: 0, ios: 1 }, targets })).toEqual( - [expect.objectContaining({ platform: "ios", reason: "over-limit" })], - ); - expect(await findings({ limit: { ios: 3 }, targets })).toEqual([]); - }); - - it("reports one finding for targets that fit each platform but not the machine's running limit together", async () => { - const found = await findings({ - limit: { android: 5, global: 3, ios: 5 }, - targets: [ - { count: 2, model: "iPhone 17", platform: "ios" }, - { count: 2, model: "Pixel 8", platform: "android" }, - ], - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: - "the targets want 4 running devices, and the running limit is 3, leaving room for 3", - reason: "over-limit", - remedy: "lower the counts of the warmPool.targets, or raise the running limit", - target: "all targets", - }, - ]); - }); - - it("pins the unknown-model message and remedy, naming the model and the platform", async () => { - const found = await findings({ - figures: figuresOf([ - { - model: "Pixel 99", - osVersion: "35", - platform: "android", - short: "unknown-model", - }, - ]), - }); - - expect(found).toStrictEqual([ - { - kind: "warm-pool-target-unreachable", - message: "Pixel 99 is not a known Android model", - platform: "android", - reason: "unknown-model", - remedy: "run simlock catalog --platform android to list the models", - target: "Pixel 99 / 35 / full", - }, - ]); - }); - - it("names an Android runtime as Android, and a target that names no OS without one", async () => { - const found = await findings({ - figures: figuresOf([ - { - model: "Pixel 8", - osVersion: "35", - platform: "android", - short: "runtime-missing", - }, - { model: "iPhone 17", short: "runtime-missing" }, - { model: "iPhone 99", short: "unknown-model" }, - ]), - }); - - expect(found).toStrictEqual([ - expect.objectContaining({ - message: "Android 35 is not installed", - remedy: "run simlock component install android 35", - }), - { - kind: "warm-pool-target-unreachable", - message: "iOS runtime is not installed", - platform: "ios", - reason: "runtime-missing", - remedy: - "run simlock component list --platform ios to see the versions, then simlock component install ios ", - target: "iPhone 17 / full", - }, - expect.objectContaining({ target: "iPhone 99 / full" }), - ]); - }); - - it("takes both reserves off the machine's room, and reports nothing for targets that exactly fill what is left", async () => { - const reserve = { android: 1, ios: 1 }; - const targets = (ios: number) => [ - { count: ios, model: "iPhone 17", platform: "ios" as const }, - { count: 1, model: "Pixel 8", platform: "android" as const }, - ]; - const limit = { android: 5, global: 4, ios: 5 }; - - expect(await findings({ limit, reserve, targets: targets(2) })).toEqual([ - expect.objectContaining({ reason: "over-limit", target: "all targets" }), - ]); - expect(await findings({ limit, reserve, targets: targets(1) })).toEqual([]); - expect( - await findings({ - limit: { ios: 3 }, - targets: [{ count: 3, model: "iPhone 17", platform: "ios" }], - }), - ).toEqual([]); - }); - - it("does not report a target short for ram-budget, or one that fits", async () => { - const found = await findings({ - figures: figuresOf([{ model: "iPhone 17", osVersion: "26.0", short: "ram-budget" }]), - targets: [{ count: 1, model: "iPhone 17", platform: "ios" }], - }); - - expect(found).toEqual([]); - }); - - it("reports nothing with the pool off", async () => { - const found = await findings({ - enabled: false, - figures: figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), - limit: { ios: 1 }, - targets: [{ count: 5, model: "iPhone 17", platform: "ios" }], - }); - - expect(found).toEqual([]); - }); - - it("leaves the finding out of the doctor.reconciled event's driftFindings, which keeps the drift beside it", async () => { - const clock = new FakeClock(10_000); - const driver = new FakeDriver({ clock, platform: "ios" }); - driver.setManagedReality({ - devices: [ - { - address: "simlock-orphan-address", - deviceId: "simlock-orphan", - driverData: { fakeDeviceId: "simlock-orphan" }, - runState: "running", - }, - ], - processes: [], - }); - const eventBus = new EventBus(clock); - const registry = await loadRegistry(clock, eventBus); - const base = config(); - const doctor = new Doctor({ - decisions: new SerializedDecision(), - clock, - config: base, - drivers: [driver], - eventBus, - registry, - warmPool: { - figures: () => - figuresOf([{ model: "iPhone 17", osVersion: "27.0", short: "runtime-missing" }]), - runningCapacity: () => limits({}), - }, - }); - - const report = await doctor.reconcile(); - - const reconciled = eventBus.replay().find((event) => event.event === "doctor.reconciled"); - expect(report.findings.map((finding) => finding.kind).sort()).toEqual([ - "orphan-device", - "warm-pool-target-unreachable", - ]); - expect(reconciled?.payload).toMatchObject({ driftFindings: [{ kind: "orphan-device" }] }); - expect(reconciled?.payload).toHaveProperty("driftFindings.length", 1); - }); }); diff --git a/src/core/test-doctor-observed.ts b/src/core/test-doctor-observed.ts new file mode 100644 index 000000000..75a41a6f2 --- /dev/null +++ b/src/core/test-doctor-observed.ts @@ -0,0 +1,12 @@ +export function driverDevice(deviceId: string) { + return { + physical: false, + address: `${deviceId}-address`, + deviceId, + driverData: { fakeDeviceId: deviceId }, + }; +} + +export function observed(deviceId: string, runState: "running" | "stopped") { + return { ...driverDevice(deviceId), runState }; +} From ef896365a793942e27c89325973870984c6f2681 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:22:31 +0100 Subject: [PATCH 31/83] test(daemon): split dispatcher.test.ts by topic --- src/daemon/dispatcher-baseline-lease.test.ts | 271 +++ src/daemon/dispatcher-baseline-list.test.ts | 132 ++ src/daemon/dispatcher-baseline-remove.test.ts | 184 ++ .../dispatcher-baseline-replace.test.ts | 279 +++ src/daemon/dispatcher-baseline-save.test.ts | 326 ++++ src/daemon/dispatcher-events.test.ts | 235 +++ src/daemon/dispatcher-usage.test.ts | 209 +++ src/daemon/dispatcher.test.ts | 1631 +---------------- 8 files changed, 1639 insertions(+), 1628 deletions(-) create mode 100644 src/daemon/dispatcher-baseline-lease.test.ts create mode 100644 src/daemon/dispatcher-baseline-list.test.ts create mode 100644 src/daemon/dispatcher-baseline-remove.test.ts create mode 100644 src/daemon/dispatcher-baseline-replace.test.ts create mode 100644 src/daemon/dispatcher-baseline-save.test.ts create mode 100644 src/daemon/dispatcher-events.test.ts create mode 100644 src/daemon/dispatcher-usage.test.ts diff --git a/src/daemon/dispatcher-baseline-lease.test.ts b/src/daemon/dispatcher-baseline-lease.test.ts new file mode 100644 index 000000000..ec43a2321 --- /dev/null +++ b/src/daemon/dispatcher-baseline-lease.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { DispatchError } from "./dispatcher.js"; +import { classifyError } from "./error-code.js"; +import { buildDispatcher, gibibyte, session } from "./test-dispatcher.js"; + +describe("Dispatcher: lease.request naming a baseline", () => { + it("gives a copy of the baseline, and the grant, lease.granted and device.provisioned carry its name and version, and lease.requested, lease.rejected and lease.declined the name the request gave", async () => { + const { dispatcher, driver, eventBus, registry } = await buildDispatcher({ + capacity: { + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 3, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + strategy: "resource", + }, + driverOptions: { knownModels: ["iPhone 17 Pro"] }, + }); + const first = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === first.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/login.txt", "signed in"); + await dispatcher.dispatch( + "baseline.save", + { leaseId: first.lease.id, name: "checkout" }, + session({ role: "admin" }), + ); + + const grant = await dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform: "ios" }, + session(), + ); + + expect(grant.device.spec).toEqual({ + baseline: { name: "checkout", version: 1 }, + model: "iPhone 17 Pro", + osVersion: "26.5", + physical: false, + platform: "ios", + }); + expect(grant.device.id).not.toBe(first.device.id); + const copy = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + expect(driver.deviceFile(copy?.driverDeviceId ?? "", "data/login.txt")).toBe("signed in"); + expect( + eventBus + .replay() + .filter((envelope) => envelope.event === "lease.granted") + .map((envelope) => envelope.payload) + .at(-1), + ).toMatchObject({ baseline: { name: "checkout", version: 1 }, deviceId: grant.device.id }); + + const full = { baseline: "checkout", noWait: true, platform: "ios" } as const; + await expect( + dispatcher.dispatch("lease.request", { ...full, requesterId: "agent-2" }, session()), + ).rejects.toMatchObject({ + code: "NO_CAPACITY", + }); + await expect( + dispatcher.dispatch( + "lease.request", + { ...full, fleetRequestId: "req_gw1" }, + session({ isGatewayUplink: true, principal: "tok_gateway", role: "admin" }), + ), + ).rejects.toMatchObject({ code: "NO_CAPACITY" }); + const specOf = (event: string, field: "requestSpec" | "spec") => + eventBus + .replay() + .filter((envelope) => envelope.event === event) + .map((envelope) => (envelope.payload as Record)[field]) + .map((spec) => spec?.baseline) + .filter((baseline) => baseline !== undefined); + expect({ + declined: specOf("lease.declined", "requestSpec"), + provisioned: specOf("device.provisioned", "spec"), + rejected: specOf("lease.rejected", "requestSpec"), + requested: specOf("lease.requested", "requestSpec"), + }).toEqual({ + declined: ["checkout"], + provisioned: [{ name: "checkout", version: 1 }], + rejected: ["checkout"], + requested: ["checkout", "checkout", "checkout"], + }); + + let leaseId = grant.lease.id; + for (const matching of [{ model: "iphone 17 pro" }, { osVersion: ">=26" }]) { + await dispatcher.dispatch("lease.release", { leaseId }, session()); + const again = await dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform: "ios", ...matching }, + session(), + ); + expect(again.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); + leaseId = again.lease.id; + } + }); +}); + +describe("Dispatcher: lease.request naming a baseline that cannot serve it", () => { + it("fails at once, provisioning nothing, as UNKNOWN_BASELINE, BASELINE_MISMATCH naming the first differing field, BASELINE_STALE or BAD_REQUEST", async () => { + const clock = new FakeClock(1_000); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const { dispatcher, driver, registry } = await buildDispatcher({ + clock, + driverOptions: { knownModels: ["iPhone 17 Pro"] }, + extraDrivers: [android], + }); + const admin = session({ role: "admin" }); + const first = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + await dispatcher.dispatch( + "baseline.save", + { leaseId: first.lease.id, name: "checkout" }, + admin, + ); + const provisions = (): number => + driver.calls.filter((call) => call.operation === "provision").length; + const failure = async (input: Record) => { + const error = await dispatcher + .dispatch("lease.request", input as never, session()) + .catch((caught: unknown) => caught); + return error instanceof Error + ? { + code: classifyError(error), + details: (error as DispatchError).details, + message: error.message, + } + : error; + }; + const named = { baseline: "checkout", platform: "ios" }; + + const control = await dispatcher.dispatch( + "lease.request", + { ...named, mode: "full", model: "iPhone 17 Pro", osVersion: "26.5" } as never, + session(), + ); + expect(control.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); + await dispatcher.dispatch("lease.release", { leaseId: control.lease.id }, session()); + const provisioned = provisions(); + + const mismatch = (field: string, message: string) => ({ + code: "BASELINE_MISMATCH", + details: { baseline: "checkout", field }, + message, + }); + const failures = []; + for (const input of [ + { ...named, baseline: "chekout" }, + { ...named, platform: "android" }, + { ...named, model: "iPhone 16" }, + { ...named, osVersion: "25.0" }, + { ...named, mode: "slim" }, + { ...named, imageTag: "google_apis" }, + { ...named, mode: "slim", osVersion: "25.0" }, + { ...named, imageTag: "google_apis", mode: "slim" }, + ]) { + failures.push(await failure(input)); + } + expect(failures).toEqual([ + { + code: "UNKNOWN_BASELINE", + details: { name: "chekout" }, + message: 'no baseline named "chekout" on this worker', + }, + mismatch("platform", "baseline checkout is ios, the request asks for android"), + mismatch("model", "baseline checkout is iPhone 17 Pro, the request asks for iPhone 16"), + mismatch("osVersion", "baseline checkout is iOS 26.5, the request asks for 25.0"), + mismatch("mode", "baseline checkout is full, the request asks for slim"), + mismatch("imageTag", "baseline checkout is image tag none, the request asks for google_apis"), + mismatch("osVersion", "baseline checkout is iOS 26.5, the request asks for 25.0"), + mismatch("mode", "baseline checkout is full, the request asks for slim"), + ]); + + const [source] = await driver.listSources(); + const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; + driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); + await dispatcher.dispatch("baseline.list", {}, admin); + expect(await failure(named)).toEqual({ + code: "BASELINE_STALE", + details: { baseline: "checkout", reason }, + message: `baseline checkout is stale: ${reason}`, + }); + expect(provisions()).toBe(provisioned); + expect(registry.snapshot.devices.filter((device) => device.state !== "deleted")).toEqual([]); + + expect(await failure({ ...named, physical: true })).toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining("`baseline` has no meaning for a physical device"), + }); + expect(await failure({ ...named, class: "tablet" })).toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining( + "a request names a baseline or a class, not both: send only one of `baseline` and `class`", + ), + }); + }); +}); + +describe("Dispatcher: a baseline lease is released", () => { + it.each([ + { platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, + { platform: "android", model: "Pixel 8", osVersion: "36" }, + ] as const)( + "the next lease naming the baseline gets a new copy with none of the previous lease's changes ($platform)", + async ({ model, osVersion, platform }) => { + const clock = new FakeClock(1_000); + const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); + const { dispatcher, driver, registry } = await buildDispatcher({ + clock, + driverOptions: { knownModels: ["iPhone 17 Pro"] }, + extraDrivers: [android], + }); + const device = platform === "ios" ? driver : android; + const fileOf = (deviceId: string, path: string): string | undefined => + device.deviceFile( + registry.snapshot.virtualDevices.find((candidate) => candidate.id === deviceId) + ?.driverDeviceId ?? "", + path, + ); + const operator = await dispatcher.dispatch( + "lease.request", + { model, osVersion, platform }, + session(), + ); + const prepared = registry.snapshot.virtualDevices.find( + (candidate) => candidate.id === operator.device.id, + ); + device.writeDeviceFile(prepared?.driverDeviceId ?? "", "data/login.txt", "signed in"); + await dispatcher.dispatch( + "baseline.save", + { leaseId: operator.lease.id, name: "checkout" }, + session({ role: "admin" }), + ); + + const first = await dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform }, + session(), + ); + expect(fileOf(first.device.id, "data/login.txt")).toBe("signed in"); + device.writeDeviceFile( + registry.snapshot.virtualDevices.find((candidate) => candidate.id === first.device.id) + ?.driverDeviceId ?? "", + "data/session.txt", + "left behind", + ); + await dispatcher.dispatch("lease.release", { leaseId: first.lease.id }, session()); + const second = await dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform }, + session(), + ); + + expect(second.device.id).not.toBe(first.device.id); + expect(second.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); + expect(fileOf(second.device.id, "data/login.txt")).toBe("signed in"); + expect(fileOf(second.device.id, "data/session.txt")).toBeUndefined(); + }, + ); +}); diff --git a/src/daemon/dispatcher-baseline-list.test.ts b/src/daemon/dispatcher-baseline-list.test.ts new file mode 100644 index 000000000..f97492567 --- /dev/null +++ b/src/daemon/dispatcher-baseline-list.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +describe("Dispatcher: baseline.list", () => { + it("lists the newest version of each baseline by name with its spec, note, size, saved time and stale reason, cleared when the runtime is back, and an agent may list", async () => { + const { clock, dispatcher, driver, registry } = await buildDispatcher(); + const agent = session({ role: "agent" }); + const admin = session({ role: "admin" }); + expect(await dispatcher.dispatch("baseline.list", {}, agent)).toEqual({ baselines: [] }); + const save = async (name: string, content: string, note?: string): Promise => { + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", content); + await dispatcher.dispatch( + "baseline.save", + { leaseId: grant.lease.id, name, ...(note === undefined ? {} : { note }) }, + admin, + ); + }; + await save("login", "signed in", "signed in"); + await save("checkout", "v1"); + await save("checkout", "version two"); + + expect(await dispatcher.dispatch("baseline.list", {}, agent)).toEqual({ + baselines: [ + { + mode: "full", + model: "iPhone 17 Pro", + name: "checkout", + osVersion: "26.5", + platform: "ios", + savedAt: new Date(clock.now()).toISOString(), + sizeBytes: "version two".length, + version: 2, + }, + { + mode: "full", + model: "iPhone 17 Pro", + name: "login", + note: "signed in", + osVersion: "26.5", + platform: "ios", + savedAt: new Date(clock.now()).toISOString(), + sizeBytes: "signed in".length, + version: 1, + }, + ], + }); + + const sources = await driver.listSources(); + const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; + const staleNames = async (): Promise<(string | undefined)[]> => + (await dispatcher.dispatch("baseline.list", {}, agent)).baselines.map( + (baseline) => baseline.staleReason, + ); + const [loginSource] = sources; + driver.setSources( + sources.map((source) => + source === loginSource ? { ...source, staleReason: reason } : source, + ), + ); + expect(await staleNames()).toEqual([undefined, reason]); + driver.setSources(sources); + expect(await staleNames()).toEqual([undefined, undefined]); + }); +}); + +describe("Dispatcher: baseline.list without a driver", () => { + it("lists a baseline whose platform has no driver as stale, without storing the reason or recording an event", async () => { + const { dispatcher, eventBus } = await buildDispatcher({ + seedState: ({ database }) => { + database + .prepare( + "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES ('droid', 1, 'android', 'Pixel 9', '36', 'full', '2026-11-02T13:40:00.000Z', 'src-droid')", + ) + .run(); + }, + }); + const agent = session({ role: "agent" }); + + const listed = await dispatcher.dispatch("baseline.list", {}, agent); + expect(listed.baselines).toMatchObject([ + { name: "droid", staleReason: "the android driver is not available on this worker" }, + ]); + expect(eventBus.replay().filter((envelope) => envelope.event === "baseline.stale")).toEqual([]); + }); +}); + +describe("Dispatcher: baselines beside a gateway's worker view", () => { + it("answers worker.list with the baselines its own baseline.list gives, and with none when no baseline service is wired", async () => { + const { dispatcher, driver, registry } = await buildDispatcher(); + const admin = session({ role: "admin" }); + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", "signed in"); + await dispatcher.dispatch("baseline.save", { leaseId: grant.lease.id, name: "login" }, admin); + + const { baselines } = await dispatcher.dispatch("baseline.list", {}, admin); + const { workers } = await dispatcher.dispatch("worker.list", {}, admin); + + expect(baselines.map((baseline) => baseline.name)).toEqual(["login"]); + expect(workers[0]?.baselines).toEqual(baselines); + + const bare = await buildDispatcher({ withoutBaselines: true }); + const listed = await bare.dispatcher.dispatch("worker.list", {}, admin); + expect(listed.workers[0]).not.toHaveProperty("baselines"); + }); + + it("refuses baseline.remove with a workerId, which names a gateway's worker", async () => { + const { dispatcher } = await buildDispatcher(); + + await expect( + dispatcher.dispatch( + "baseline.remove", + { name: "login", workerId: "wrk_a" }, + session({ role: "admin" }), + ), + ).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: + "workerId names a gateway's worker, and this daemon is not a gateway; remove without --worker", + }); + }); +}); diff --git a/src/daemon/dispatcher-baseline-remove.test.ts b/src/daemon/dispatcher-baseline-remove.test.ts new file mode 100644 index 000000000..9b38c8807 --- /dev/null +++ b/src/daemon/dispatcher-baseline-remove.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import type { StateDatabase } from "../state/index.js"; +import type { DispatchSession } from "./dispatcher.js"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +/** why: #1207 writes a device's baseline at provision; until then a test writes the registry rows a remove reads. */ +function seedBaselineRows(state: StateDatabase): void { + const { database } = state; + const spec = ["ios", "iPhone 17 Pro", "26.5", "full"]; + for (const [name, sourceId] of [ + ["login", "src-login"], + ["checkout", "src-checkout"], + ["busy", "src-busy"], + ] as const) { + database + .prepare( + "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES (?, 1, ?, ?, ?, ?, '2026-11-02T13:40:00.000Z', ?)", + ) + .run(name, ...spec, sourceId); + } + for (const [id, fakeId, deviceState, baseline] of [ + ["dev_idle", "fake-ios-1", "shutdown", "checkout"], + ["dev_idle_b", "fake-ios-4", "shutdown", "checkout"], + ["dev_leased", "fake-ios-2", "leased", "busy"], + ["dev_quarantined", "fake-ios-3", "quarantined", "login"], + ] as const) { + database + .prepare( + "INSERT INTO devices (id, platform, physical, driver_device_id, state, model, os_version, mode, created_at, baseline_name, baseline_version) VALUES (?, 'ios', 0, ?, ?, 'iPhone 17 Pro', '26.5', 'full', 1, ?, 1)", + ) + .run(id, fakeId, deviceState, baseline); + database + .prepare("INSERT INTO fake_device_data (device_id, fake_device_id) VALUES (?, ?)") + .run(id, fakeId); + } + database + .prepare( + "INSERT INTO leases (id, device_id, requester_id, owner_id, granted_at, ttl_ms, ttl_deadline, last_renewed_at, id_chosen_by_requester) VALUES ('lse_busy', 'dev_leased', 'tok_a', 'tok_a', 1, 3600000, 3600001, 1, 0)", + ) + .run(); +} + +describe("Dispatcher: baseline.remove", () => { + it("removes an unused baseline with its source and the idle devices made from it, refuses one a lease uses, an unknown name and an agent", async () => { + const { dispatcher, driver, engine, registry } = await buildDispatcher({ + seedState: seedBaselineRows, + }); + const spec = { + model: "iPhone 17 Pro", + osVersion: "26.5", + physical: false, + platform: "ios", + } as const; + for (let count = 0; count < 4; count += 1) await driver.provision(spec); + driver.setSources( + ["login", "checkout", "busy"].map((name) => ({ + id: `src-${name}`, + kind: "baseline" as const, + model: "iPhone 17 Pro", + osVersion: "26.5", + })), + ); + const admin = session({ role: "admin" }); + const refused = async (name: string, caller: DispatchSession): Promise => + dispatcher.dispatch("baseline.remove", { name }, caller).then( + () => undefined, + (rejected: unknown) => rejected, + ); + const listed = async (): Promise => + (await dispatcher.dispatch("baseline.list", {}, admin)).baselines.map(({ name }) => name); + + expect(await refused("login", session({ role: "agent" }))).toMatchObject({ + code: "FORBIDDEN", + }); + expect(await refused("nothere", admin)).toMatchObject({ + code: "UNKNOWN_BASELINE", + details: { name: "nothere" }, + message: 'no baseline named "nothere" on this worker', + }); + expect(await refused("Checkout", admin)).toMatchObject({ code: "BAD_REQUEST" }); + expect(await refused("busy", admin)).toMatchObject({ + code: "BASELINE_IN_USE", + details: { leaseId: "lse_busy", name: "busy" }, + message: "baseline busy is in use by lease lse_busy", + }); + expect(await listed()).toEqual(["busy", "checkout", "login"]); + + expect(await dispatcher.dispatch("baseline.remove", { name: "login" }, admin)).toEqual({ + deletedDevices: [], + name: "login", + versions: [1], + }); + expect(engine.claimReader.isClaimed("dev_quarantined")).toBe(false); + // why: the first idle device's destroy throws; the second must still be disposed and no cleanup claim left held. + driver.failOn("destroy", 1, new Error("destroy failed")); + await expect( + dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin), + ).rejects.toThrow("destroy failed"); + expect(engine.claimReader.isClaimed("dev_idle")).toBe(false); + expect(engine.claimReader.isClaimed("dev_idle_b")).toBe(false); + expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toContainEqual([ + "dev_idle_b", + "deleted", + ]); + expect(await listed()).toEqual(["busy", "checkout"]); + expect(await dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin)).toEqual({ + deletedDevices: ["dev_idle"], + name: "checkout", + versions: [1], + }); + + expect(await listed()).toEqual(["busy"]); + expect((await driver.listSources()).map((source) => source.id)).toEqual(["src-busy"]); + expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toEqual([ + ["dev_idle", "deleted"], + ["dev_idle_b", "deleted"], + ["dev_leased", "leased"], + ["dev_quarantined", "quarantined"], + ]); + }); +}); + +describe("Dispatcher: doctor baseline sources", () => { + it("lists a baseline's source with its name and version, a ready baseline source no row names as an orphan only doctor --purge-orphans deletes, and finds a stale baseline", async () => { + const { dispatcher, driver, eventBus, registry } = await buildDispatcher(); + const admin = session({ role: "admin" }); + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", "abc"); + await dispatcher.dispatch( + "baseline.save", + { leaseId: grant.lease.id, name: "checkout" }, + admin, + ); + const [named] = await driver.listSources(); + const unnamed = { + id: "left-over", + kind: "baseline", + model: "iPhone 17 Pro", + osVersion: "26.5", + } as const; + driver.setSources([named as NonNullable, unnamed]); + + const listed = await dispatcher.dispatch("doctor.run", {}, admin); + expect(listed.findings).toEqual([]); + expect(listed.sources).toEqual([ + { + baselineName: "checkout", + baselineVersion: 1, + kind: "baseline", + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + sizeBytes: 3, + }, + { + kind: "baseline", + model: "iPhone 17 Pro", + orphan: true, + osVersion: "26.5", + platform: "ios", + }, + ]); + expect((await driver.listSources()).map((source) => source.id)).toContain("left-over"); + + const purged = await dispatcher.dispatch("doctor.run", { purgeOrphans: true }, admin); + expect(purged.sources.map((source) => source.orphan)).toEqual([undefined]); + expect((await driver.listSources()).map((source) => source.id)).not.toContain("left-over"); + + const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; + driver.setSources([{ ...(named as NonNullable), staleReason: reason }]); + await dispatcher.dispatch("doctor.run", {}, admin); + expect( + eventBus + .replay() + .filter((envelope) => envelope.event === "baseline.stale") + .map((envelope) => envelope.payload), + ).toEqual([{ name: "checkout", reason, version: 1 }]); + }); +}); diff --git a/src/daemon/dispatcher-baseline-replace.test.ts b/src/daemon/dispatcher-baseline-replace.test.ts new file mode 100644 index 000000000..aabb9ed93 --- /dev/null +++ b/src/daemon/dispatcher-baseline-replace.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { buildDispatcher, gibibyte, session } from "./test-dispatcher.js"; + +// why: an idle copy of version 1 is what a warm pool leaves behind; the registry is written as the pool's provisioner would. +async function seedIdleCopy( + driver: FakeDriver, + registry: Awaited>["registry"], + model: string, +): Promise<{ readonly id: string }> { + const [source] = await driver.listSources(); + const spec = { + baseline: { name: "checkout", version: 1 }, + model, + osVersion: "26.5", + physical: false, + platform: "ios", + } as const; + const driverDevice = await driver.provision(spec, { source: source?.id ?? "" }); + const booted = await driver.makeReady(driverDevice, { mode: "full", purpose: "prepare" }); + const registered = await registry.registerDevice({ + driverData: driverDevice.driverData, + driverDeviceId: driverDevice.deviceId, + provisionDuration: 0, + spec, + }); + return registry.transitionDevice( + registered.id, + "ready", + { event: "device.ready", payload: { bootDuration: 0, deviceId: registered.id } }, + booted.mode === undefined ? undefined : { mode: booted.mode }, + ); +} + +type ReplaceScenario = "running" | "idle" | "waiting" | "making"; + +const replaceModel = "iPhone 17 Pro"; + +async function buildReplaceHarness(scenario: ReplaceScenario) { + const clock = new FakeClock(1_000); + const oneDevice = { + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 3, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + strategy: "resource", + } as const; + const built = await buildDispatcher({ + ...(scenario === "waiting" ? { capacity: oneDevice } : {}), + clock, + driverOptions: { + knownModels: [replaceModel], + ...(scenario === "making" ? { latencyMs: { provision: 5_000 } } : {}), + }, + }); + const { dispatcher, driver, engine, eventBus, registry, state } = built; + const settle = async (pending: Promise, step = 5_000): Promise => { + let done = false; + void pending.then( + () => (done = true), + () => (done = true), + ); + while (!done) { + clock.advance(step); + await new Promise((resolve) => setImmediate(resolve)); + } + return pending; + }; + const lease = (input: Record) => + settle(dispatcher.dispatch("lease.request", { platform: "ios", ...input } as never, session())); + const driverIdOf = (deviceId: string): string => + registry.snapshot.virtualDevices.find((device) => device.id === deviceId)?.driverDeviceId ?? ""; + const saveCheckout = (leaseId: string, step?: number) => + settle( + dispatcher.dispatch( + "baseline.save", + { leaseId, name: "checkout" }, + session({ role: "admin" }), + ), + step, + ); + return { + clock, + dispatcher, + driver, + engine, + fileOf: (deviceId: string) => driver.deviceFile(driverIdOf(deviceId), "data/version.txt"), + lease, + registry, + copiesOf: (version: number): number => + registry.snapshot.virtualDevices.filter((device) => device.spec.baseline?.version === version) + .length, + removed: () => + eventBus + .replay() + .filter((envelope) => envelope.event === "baseline.removed") + .map((envelope) => envelope.payload), + saveCheckout, + saveFrom: async (requesterId: string, marker: string): Promise => { + const held = await lease({ model: replaceModel, requesterId }); + driver.writeDeviceFile(driverIdOf(held.device.id), "data/version.txt", marker); + await saveCheckout(held.lease.id); + }, + settle, + stateOf: (deviceId: string): string | undefined => + registry.snapshot.virtualDevices.find((device) => device.id === deviceId)?.state, + versions: (): number[] => + state.database + .prepare("SELECT version FROM baselines WHERE name = 'checkout' ORDER BY version") + .all() + .map((row) => Number(row.version)), + writeFile: (deviceId: string, marker: string) => + driver.writeDeviceFile(driverIdOf(deviceId), "data/version.txt", marker), + }; +} + +type ReplaceHarness = Awaited>; + +/** why: the v1 baseline exists; the scenario's lease, idle copy or waiting request is arranged around the operator's lease. */ +async function arrangeReplace(harness: ReplaceHarness, scenario: ReplaceScenario) { + await harness.saveFrom("operator", "one"); + expect(harness.versions()).toEqual([1]); + const running = + scenario === "running" + ? await harness.lease({ baseline: "checkout", requesterId: "agent-1" }) + : undefined; + const idle = + scenario === "idle" + ? await seedIdleCopy(harness.driver, harness.registry, replaceModel) + : undefined; + const operator = await harness.lease({ model: replaceModel, requesterId: "operator" }); + const waiting = + scenario === "waiting" || scenario === "making" + ? harness.dispatcher.dispatch( + "lease.request", + { baseline: "checkout", platform: "ios", requesterId: "agent-2" }, + session(), + ) + : undefined; + for (let turn = 0; waiting !== undefined && turn < 10; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + return { idle, operator, running, waiting }; +} + +describe("Dispatcher: an operator replaces a baseline", () => { + it.each([ + { label: "a lease is running", case: "running", oldCopies: 1 }, + { label: "no lease is running", case: "idle", oldCopies: 1 }, + { + label: "the running lease's delete fails at its release", + case: "running", + oldCopies: 1, + failDelete: true, + }, + { label: "a request is waiting", case: "waiting", oldCopies: 0 }, + { label: "a copy is being made", case: "making", oldCopies: 1 }, + ] as const)( + "running leases keep their device, the next lease gets the new version, a waiting request too, idle old-version devices are destroyed, a copy of the old version finished after the save is deleted and never granted, and the old version goes with its last device ($label)", + async ({ case: scenario, oldCopies, ...rest }) => { + const failDelete = "failDelete" in rest; + const harness = await buildReplaceHarness(scenario); + const { idle, operator, running, waiting } = await arrangeReplace(harness, scenario); + harness.writeFile(operator.device.id, "two"); + // why: the first save in the arrangement and this lease's own save destroy a device before the idle copy. + if (failDelete) harness.driver.failOn("destroy", 3, new Error("destroy failed")); + + await harness.saveCheckout(operator.lease.id, 100); + if (running !== undefined) { + expect(harness.stateOf(running.device.id)).toBe("leased"); + expect(harness.versions()).toEqual([1, 2]); + expect(harness.removed()).toEqual([]); + } + if (idle !== undefined) expect(harness.stateOf(idle.id)).toBe("deleted"); + const next = await (waiting === undefined + ? harness.lease({ baseline: "checkout", requesterId: "agent-3" }) + : harness.settle(waiting)); + expect(next.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); + expect(harness.fileOf(next.device.id)).toBe("two"); + expect(harness.copiesOf(1)).toBe(oldCopies); + if (running !== undefined) { + await harness.dispatcher.dispatch( + "lease.release", + { leaseId: running.lease.id }, + session(), + ); + } + await new Promise((resolve) => setImmediate(resolve)); + if (failDelete && running !== undefined) { + // why: the last device of version 1 leaves through the quarantine's delete retry, and that drops the version. + expect(harness.stateOf(running.device.id)).toBe("quarantined"); + expect(harness.versions()).toEqual([1, 2]); + harness.clock.advance(60 * 60_000); + for (let turn = 0; turn < 10; turn += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + expect(harness.stateOf(running.device.id)).toBe("deleted"); + } + expect(harness.versions()).toEqual([2]); + expect(harness.removed()).toEqual([ + expect.objectContaining({ name: "checkout", reason: "replaced", version: 1 }), + ]); + expect(await harness.driver.listSources()).toHaveLength(1); + }, + ); +}); + +describe("Dispatcher: an operator replaces a baseline while an idle old copy will not delete", () => { + it("the save succeeds, the idle old-version device stays shut down, and version 1 stays with it", async () => { + const harness = await buildReplaceHarness("idle"); + const { idle, operator } = await arrangeReplace(harness, "idle"); + harness.writeFile(operator.device.id, "two"); + // why: the first save in the arrangement and this lease's own save destroy a device before the idle copy. + harness.driver.failOn("destroy", 3, new Error("destroy failed")); + + await harness.saveCheckout(operator.lease.id, 100); + + if (idle === undefined) throw new Error("the arrangement left no idle device"); + expect(harness.stateOf(idle.id)).toBe("shutdown"); + expect(harness.versions()).toEqual([1, 2]); + expect(harness.removed()).toEqual([]); + const next = await harness.lease({ baseline: "checkout", requesterId: "agent-3" }); + expect(next.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); + }); +}); + +describe("Dispatcher: two copies of one baseline version are made at once", () => { + it("keeps the old version after a replace until the second copy is registered, though the first copy is gone", async () => { + const harness = await buildReplaceHarness("making"); + const flush = async (): Promise => { + for (let turn = 0; turn < 10; turn += 1) + await new Promise((resolve) => setImmediate(resolve)); + }; + await harness.saveFrom("operator", "one"); + const operator = await harness.lease({ model: replaceModel, requesterId: "operator" }); + harness.writeFile(operator.device.id, "two"); + // why: a warm pool pass makes copies without a request, so two can be in flight for one version. + const copy = () => + harness.engine.core.provisioner.provision( + { + baseline: { name: "checkout", version: 1 }, + model: replaceModel, + osVersion: "26.5", + physical: false, + platform: "ios", + }, + { reservation: { release: () => undefined } }, + ); + + const first = copy(); + await flush(); + harness.clock.advance(2_000); + const second = copy(); + await flush(); + await harness.saveCheckout(operator.lease.id, 100); + expect(harness.versions()).toEqual([1, 2]); + + harness.clock.advance(2_900); + const firstCopy = await first; + const lifecycle = harness.engine.core.deviceLifecycle; + firstCopy.claim.release(); + const stopped = await lifecycle.shutdown(firstCopy.device, "lease-engine", "cleanup"); + await lifecycle.destroy(stopped as NonNullable, "lease-engine", "cleanup"); + await flush(); + expect(harness.stateOf(firstCopy.device.id)).toBe("deleted"); + expect(harness.versions()).toEqual([1, 2]); + expect(harness.removed()).toEqual([]); + + harness.clock.advance(2_000); + const secondCopy = await second; + expect(harness.stateOf(secondCopy.device.id)).toBe("ready"); + expect(harness.versions()).toEqual([1, 2]); + }); +}); diff --git a/src/daemon/dispatcher-baseline-save.test.ts b/src/daemon/dispatcher-baseline-save.test.ts new file mode 100644 index 000000000..f9105657f --- /dev/null +++ b/src/daemon/dispatcher-baseline-save.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it, vi } from "vitest"; +import { OPERATIONS } from "../contract/index.js"; +import type { DispatchSession } from "./dispatcher.js"; +import { classifyError } from "./error-code.js"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +describe("Dispatcher: baseline.save", () => { + it("ends the held lease, saves its device as version 1 of the named baseline with the files it held, and deletes the device", async () => { + const { clock, dispatcher, driver, eventBus, registry } = await buildDispatcher(); + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/login.txt", "signed in"); + + const saved = await dispatcher.dispatch( + "baseline.save", + { leaseId: grant.lease.id, name: "checkout", note: "app 4.2" }, + session({ role: "admin" }), + ); + + expect(saved).toEqual({ + baseline: { + mode: "full", + model: "iPhone 17 Pro", + name: "checkout", + note: "app 4.2", + osVersion: "26.5", + platform: "ios", + savedAt: new Date(clock.now()).toISOString(), + sizeBytes: "signed in".length, + version: 1, + }, + leaseId: grant.lease.id, + }); + expect(registry.snapshot.leases).toEqual([]); + expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["deleted"]); + const [source] = await driver.listSources(); + expect(driver.sourceFiles(source?.id ?? "")).toEqual( + new Map([["data/login.txt", "signed in"]]), + ); + expect( + eventBus + .replay() + .filter((envelope) => envelope.event === "lease.released") + .map((envelope) => envelope.payload), + ).toMatchObject([{ leaseId: grant.lease.id, reason: "explicit" }]); + }); + + it("records the image tag the saved device carries when its lease named none, so a lease naming that tag is granted and another tag is refused", async () => { + const { dispatcher } = await buildDispatcher({ + driverOptions: { + availableOsVersions: ["34"], + defaultImageTag: "google_apis_playstore", + images: [ + { abi: "arm64-v8a", runtime: "34", tag: "google_apis" }, + { abi: "arm64-v8a", runtime: "34", tag: "google_apis_playstore" }, + ], + platform: "android", + }, + }); + const grant = await dispatcher.dispatch( + "lease.request", + { model: "Pixel 8", osVersion: "34", platform: "android" }, + session(), + ); + const admin = session({ role: "admin" }); + + const saved = await dispatcher.dispatch( + "baseline.save", + { leaseId: grant.lease.id, name: "android-test" }, + admin, + ); + const listed = await dispatcher.dispatch("baseline.list", {}, admin); + const named = { baseline: "android-test", platform: "android" }; + const granted = await dispatcher.dispatch( + "lease.request", + { ...named, imageTag: "google_apis_playstore" }, + session(), + ); + await dispatcher.dispatch("lease.release", { leaseId: granted.lease.id }, session()); + const refused = await dispatcher + .dispatch("lease.request", { ...named, imageTag: "google_apis" }, session()) + .catch((caught: unknown) => caught); + + expect(saved.baseline.imageTag).toBe("google_apis_playstore"); + expect(listed.baselines.map((entry) => entry.imageTag)).toEqual(["google_apis_playstore"]); + expect(granted.device.spec.baseline).toEqual({ name: "android-test", version: 1 }); + expect(refused).toMatchObject({ + message: + "baseline android-test is image tag google_apis_playstore, the request asks for google_apis", + }); + }); +}); + +describe("Dispatcher: baseline.save versions", () => { + it("numbers each save under one name one more than the last, one at a time or two at once, and keeps only the newest version's source", async () => { + const { dispatcher, driver, registry } = await buildDispatcher(); + const admin = session({ role: "admin" }); + const lease = async (who: string): Promise => { + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session({ principal: who }), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/who", who); + return grant.lease.id; + }; + const save = (leaseId: string) => + dispatcher.dispatch("baseline.save", { leaseId, name: "checkout" }, admin); + const keptSources = async () => + (await driver.listSources()).map((source) => driver.sourceFiles(source.id)); + + const first = await save(await lease("tok_a")); + const second = await save(await lease("tok_b")); + expect([first.baseline.version, second.baseline.version]).toEqual([1, 2]); + expect(await keptSources()).toEqual([new Map([["data/who", "tok_b"]])]); + + const [c, d] = [await lease("tok_c"), await lease("tok_d")]; + const results = await Promise.all([save(c), save(d)]); + expect(results.map((result) => result.baseline.version).sort()).toEqual([3, 4]); + const newest = + results.find((result) => result.baseline.version === 4)?.leaseId === c ? "tok_c" : "tok_d"; + expect(await keptSources()).toEqual([new Map([["data/who", newest]])]); + }); +}); + +describe("Dispatcher: baseline.save and baseline.remove queue", () => { + it("runs a remove sent during a save after it, so the remove deletes the version the save made", async () => { + const { clock, dispatcher, registry } = await buildDispatcher({ + driverOptions: { latencyMs: { saveSource: 1_000 } }, + }); + const admin = session({ role: "admin" }); + const lease = async (): Promise => + ( + await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ) + ).lease.id; + const settled = async (...pending: Promise[]): Promise => { + let open = pending.length; + for (const promise of pending) void promise.then(() => (open -= 1)); + await vi.waitFor(() => { + clock.advance(1_000); + expect(open).toBe(0); + }); + }; + const first = dispatcher.dispatch( + "baseline.save", + { leaseId: await lease(), name: "checkout" }, + admin, + ); + await settled(first); + + const second = dispatcher.dispatch( + "baseline.save", + { leaseId: await lease(), name: "checkout" }, + admin, + ); + const removed = dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin); + await settled(second, removed); + + expect(await removed).toEqual({ deletedDevices: [], name: "checkout", versions: [2] }); + expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); + expect(registry.snapshot.leases).toEqual([]); + }); +}); + +describe("Dispatcher: baseline.save and nuke", () => { + it("makes nuke wait for a save in flight and remove its baseline, and answers a save queued behind nuke's maintenance with an unknown lease since nuke ended its lease", async () => { + const { clock, dispatcher, registry } = await buildDispatcher({ + driverOptions: { latencyMs: { saveSource: 1_000 } }, + includeNuke: true, + }); + const admin = session({ role: "admin" }); + const lease = async (who: string): Promise => + ( + await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session({ principal: who }), + ) + ).lease.id; + const [held, queued] = [await lease("tok_a"), await lease("tok_b")]; + const outcome = (promise: Promise): { settled: boolean } => { + const state = { settled: false }; + void promise.then( + () => (state.settled = true), + () => (state.settled = true), + ); + return state; + }; + const saving = dispatcher.dispatch("baseline.save", { leaseId: held, name: "first" }, admin); + const savingState = outcome(saving); + await vi.waitFor(() => { + expect(registry.snapshot.devices.map((device) => device.state)).toContain("reclaiming"); + }); + + const nuking = dispatcher.dispatch("nuke.run", { deleteDevices: true }, admin); + const nukeState = outcome(nuking); + const blocked = dispatcher.dispatch( + "baseline.save", + { leaseId: queued, name: "second" }, + admin, + ); + const blockedError = blocked.then( + () => undefined, + (rejected: unknown) => rejected, + ); + await vi.waitFor(() => { + clock.advance(1_000); + expect(savingState.settled).toBe(true); + }); + const [nuked, refusal] = await Promise.all([nuking, blockedError]); + + expect(nukeState.settled).toBe(true); + expect(nuked).toMatchObject({ releasedLeaseIds: [queued], removedBaselines: ["first"] }); + expect(refusal).toMatchObject({ leaseId: queued, name: "UnknownLeaseError" }); + expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); + expect(registry.snapshot.leases).toEqual([]); + }); +}); + +describe("Dispatcher: baseline.save failure", () => { + it("answers BASELINE_SAVE_FAILED when the driver cannot save, with the lease ended, the device quarantined and no baseline recorded", async () => { + const { dispatcher, driver, eventBus, registry } = await buildDispatcher(); + const admin = session({ role: "admin" }); + const request = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; + const grant = await dispatcher.dispatch("lease.request", request, session()); + driver.failOn("saveSource", 1, new Error("disk full")); + + const error = await dispatcher + .dispatch("baseline.save", { leaseId: grant.lease.id, name: "checkout" }, admin) + .then( + () => undefined, + (rejected: unknown) => rejected, + ); + + expect(error).toMatchObject({ + code: "BASELINE_SAVE_FAILED", + details: { leaseId: grant.lease.id, name: "checkout" }, + message: `baseline checkout was not saved: disk full. Lease ${grant.lease.id} has ended.`, + }); + expect(registry.snapshot.leases).toEqual([]); + expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["quarantined"]); + expect(await driver.listSources()).toEqual([]); + expect( + eventBus + .replay() + .filter((envelope) => envelope.event === "device.quarantined") + .map((envelope) => envelope.payload), + ).toMatchObject([{ kind: "shutdown-failed" }]); + const next = await dispatcher.dispatch("lease.request", request, session()); + const saved = await dispatcher.dispatch( + "baseline.save", + { leaseId: next.lease.id, name: "checkout" }, + admin, + ); + expect(saved.baseline.version).toBe(1); + }); +}); + +describe("Dispatcher: baseline.save refusals", () => { + it("refuses a bad name or note, a non-admin, an unknown lease and a physical lease, and ends no lease", async () => { + const { dispatcher, driver, engine, registry } = await buildDispatcher({ + physicalDevices: [ + { + createdAt: 1, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: [], + id: "dev_phys", + spec: { class: "tablet", model: "iPad Pro", osVersion: "18.4" }, + state: "ready", + }, + ], + }); + await engine.convergeRunningCapacity(); + driver.setPhysicalPresence("00008110-001A2B3C4D5E", true); + const virtual = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const physical = await dispatcher.dispatch( + "lease.request", + { model: "iPad Pro", physical: true, platform: "ios" }, + session({ principal: "tok_physical" }), + ); + const admin = session({ role: "admin" }); + const accepted = { leaseId: virtual.lease.id, name: "checkout" }; + expect(OPERATIONS["baseline.save"].input.safeParse(accepted).success).toBe(true); + + const refusals: readonly [string, unknown, DispatchSession, string, string?][] = [ + ["Checkout", { ...accepted, name: "Checkout" }, admin, "BAD_REQUEST"], + ["-checkout", { ...accepted, name: "-checkout" }, admin, "BAD_REQUEST"], + ["65 characters", { ...accepted, name: "a".repeat(65) }, admin, "BAD_REQUEST"], + ["201-character note", { ...accepted, note: "n".repeat(201) }, admin, "BAD_REQUEST"], + ["note with a newline", { ...accepted, note: "two\nlines" }, admin, "BAD_REQUEST"], + ["an agent token", accepted, session({ role: "agent" }), "FORBIDDEN"], + ["unknown lease", { ...accepted, leaseId: "lse_gone" }, admin, "UNKNOWN_LEASE"], + [ + "physical lease", + { ...accepted, leaseId: physical.lease.id }, + admin, + "BAD_REQUEST", + `lease ${physical.lease.id} is on a physical device; only a simulator or emulator can be saved as a baseline`, + ], + ]; + for (const [label, input, caller, code, message] of refusals) { + const error = await dispatcher.dispatch("baseline.save", input as never, caller).then( + () => undefined, + (rejected: unknown) => rejected, + ); + expect(classifyError(error), label).toBe(code); + if (message !== undefined) expect((error as Error).message, label).toBe(message); + expect(registry.snapshot.leases.map((lease) => lease.id).sort(), label).toEqual( + [virtual.lease.id, physical.lease.id].sort(), + ); + } + }); +}); diff --git a/src/daemon/dispatcher-events.test.ts b/src/daemon/dispatcher-events.test.ts new file mode 100644 index 000000000..eec128993 --- /dev/null +++ b/src/daemon/dispatcher-events.test.ts @@ -0,0 +1,235 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { type EventEnvelope } from "../bus/index.js"; +import { plainProjection, removeUsageRigs, T0, usageRig } from "../history/testing.js"; +import { Dispatcher } from "./dispatcher.js"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +describe("Dispatcher: baseline events", () => { + it("records baseline.saved, baseline.removed with its reason (nuke --delete-devices removes every baseline, plain nuke keeps them) and baseline.stale, once each, after the change is committed", async () => { + const { dispatcher, driver, eventBus, registry, state } = await buildDispatcher({ + driverOptions: { componentSizes: { "27.0": 1_000 }, knownModels: ["iPhone 17 Pro"] }, + includeNuke: true, + }); + const admin = session({ principal: "tok_admin", role: "admin" }); + // why: what the table held at the moment each event reached a subscriber, to prove the change is committed before the event. + const seen: { removedRows: number[]; staleReasons: unknown[] } = { + removedRows: [], + staleReasons: [], + }; + eventBus.subscribe("baseline.removed", ({ payload }) => { + const row = state.database + .prepare("SELECT COUNT(*) AS count FROM baselines WHERE name = ? AND version = ?") + .get(payload.name, payload.version); + seen.removedRows.push(Number(row?.count)); + }); + eventBus.subscribe("baseline.stale", ({ payload }) => { + const row = state.database + .prepare("SELECT stale_reason FROM baselines WHERE name = ? AND version = ?") + .get(payload.name, payload.version); + seen.staleReasons.push(row?.stale_reason); + }); + await dispatcher.dispatch("component.install", { platform: "ios", version: "27.0" }, admin); + const events = (name: string): unknown[] => + eventBus + .replay() + .filter((envelope) => envelope.event === name) + .map((envelope) => envelope.payload); + const save = async (content: string, name = "checkout"): Promise => { + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); + driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", content); + await dispatcher.dispatch("baseline.save", { leaseId: grant.lease.id, name }, admin); + return grant.lease.id; + }; + + const first = await save("v1"); + expect(events("baseline.saved")).toEqual([ + { + leaseId: first, + mode: "full", + model: "iPhone 17 Pro", + name: "checkout", + osVersion: "26.5", + platform: "ios", + requesterId: "tok_admin", + sizeBytes: 2, + version: 1, + }, + ]); + expect(events("device.deleted")).toMatchObject([{ initiator: "baseline-save" }]); + expect(events("baseline.removed")).toEqual([]); + + await save("v2"); + expect(events("baseline.saved")).toHaveLength(2); + expect(events("baseline.removed")).toEqual([ + { name: "checkout", reason: "replaced", requesterId: "tok_admin", version: 1 }, + ]); + + const [source] = await driver.listSources(); + const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; + const check = async (staleReason?: string): Promise => { + driver.setSources([ + { + ...(source as NonNullable), + ...(staleReason === undefined ? {} : { staleReason }), + }, + ]); + await dispatcher.dispatch("baseline.list", {}, admin); + }; + await check(reason); + await check(reason); + expect(events("baseline.stale")).toEqual([{ name: "checkout", reason, version: 2 }]); + await check(); + await check(reason); + expect(events("baseline.stale")).toHaveLength(2); + + await check(); + driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); + await Promise.all([ + dispatcher.dispatch("baseline.list", {}, admin), + dispatcher.dispatch("baseline.list", {}, admin), + ]); + expect(events("baseline.stale")).toHaveLength(3); + await check(); + driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); + await dispatcher.dispatch("component.remove", { platform: "ios", version: "27.0" }, admin); + await vi.waitFor(() => { + expect(events("baseline.stale")).toHaveLength(4); + }); + + await dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin); + expect(events("baseline.removed")).toEqual([ + { name: "checkout", reason: "replaced", requesterId: "tok_admin", version: 1 }, + { name: "checkout", reason: "operator", requesterId: "tok_admin", version: 2 }, + ]); + + await save("again"); + await save("signed in", "login"); + expect(await dispatcher.dispatch("nuke.run", { deleteDevices: false }, admin)).toMatchObject({ + removedBaselines: [], + }); + expect((await dispatcher.dispatch("baseline.list", {}, admin)).baselines).toHaveLength(2); + driver.failOn( + "destroySource", + driver.calls.filter((call) => call.operation === "destroySource").length + 1, + new Error("simctl delete failed"), + ); + expect(await dispatcher.dispatch("nuke.run", { deleteDevices: true }, admin)).toMatchObject({ + removedBaselines: ["checkout", "login"], + }); + expect(events("baseline.removed").slice(2)).toEqual([ + { name: "checkout", reason: "nuke", version: 1 }, + { name: "login", reason: "nuke", version: 1 }, + ]); + expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); + expect(await driver.listSources()).toHaveLength(1); + expect(seen.removedRows).toEqual([0, 0, 0, 0]); + expect(seen.staleReasons).toEqual([reason, reason, reason, reason]); + }); +}); + +describe("Dispatcher: events.replay", () => { + it("events.replay with sinceTs returns what the event history returns", async () => { + const fromHistory: EventEnvelope[] = [ + { + id: "evt_7", + seq: 7, + timestamp: 50, + event: "daemon.stopping", + payload: { reason: "x" }, + module: "d", + }, + ]; + const asked: unknown[] = []; + const { dispatcher } = await buildDispatcher({ + eventHistory: { + replay: async (input) => { + asked.push(input); + return fromHistory; + }, + replayPage: async () => ({ events: [], kind: "events" }), + status: () => ({ opened: true, writing: true }), + }, + }); + + await expect( + dispatcher.dispatch("events.replay", { sinceTs: 10 }, session({ role: "admin" })), + ).resolves.toEqual(fromHistory); + expect(asked).toEqual([{ sinceTs: 10 }]); + }); +}); + +describe("Dispatcher: events.replay paging", () => { + afterEach(removeUsageRigs); + + const admin = session({ role: "admin" }); + + async function pagedRig( + rows: number, + options: Parameters[0] = {}, + ): Promise<{ + readonly rig: Awaited>; + readonly ids: string[]; + readonly dispatcher: Dispatcher; + }> { + const rig = await usageRig(options); + for (let row = 0; row < rows; row += 1) { + rig.at(T0 + row); + rig.emit("daemon.stopping", { reason: `r${row + 1}` }); + } + const { dispatcher } = await buildDispatcher({ eventHistory: rig.history }); + return { dispatcher, ids: rig.bus.replay().map((event) => event.id), rig }; + } + + const reasons = (events: readonly { readonly payload?: unknown }[]): unknown[] => + events.map((event) => (event.payload as { reason: string }).reason); + + it("events.replay with beforeId and limit pages through the worker dispatcher over a real history: the page, HISTORY_NOT_KEPT, an empty table, HISTORY_UNAVAILABLE, and 100 at most for beforeId alone", async () => { + const { dispatcher, ids } = await pagedRig(150); + const page = await dispatcher.dispatch( + "events.replay", + { beforeId: ids[4] as string, limit: 2 }, + admin, + ); + expect(reasons(page)).toEqual(["r3", "r4"]); + const unbounded = await dispatcher.dispatch( + "events.replay", + { beforeId: ids[149] as string }, + admin, + ); + expect(unbounded).toHaveLength(100); + await expect( + dispatcher.dispatch("events.replay", { beforeId: "evt_gone", limit: 2 }, admin), + ).rejects.toMatchObject({ code: "HISTORY_NOT_KEPT", details: { oldestTs: T0 } }); + + const empty = await pagedRig(0); + await expect( + empty.dispatcher.dispatch("events.replay", { beforeId: "evt_any", limit: 2 }, admin), + ).resolves.toEqual([]); + + const failing = plainProjection("failing", () => ({ + apply: () => { + throw new Error("projection broke"); + }, + prune: () => undefined, + })); + const notWriting = await pagedRig(1, { projections: [failing] }); + expect(notWriting.rig.history.status().writing).toBe(false); + const unstarted = await buildDispatcher(); + for (const { dispatcher: target } of [notWriting, unstarted]) { + await expect( + target.dispatch("events.replay", { beforeId: "evt_any", limit: 2 }, admin), + ).rejects.toMatchObject({ code: "HISTORY_UNAVAILABLE" }); + } + }); + + it("events.replay with limit alone returns the newest n events, oldest first, through the worker dispatcher", async () => { + const { dispatcher } = await pagedRig(5); + const page = await dispatcher.dispatch("events.replay", { limit: 2 }, admin); + expect(reasons(page)).toEqual(["r4", "r5"]); + }); +}); diff --git a/src/daemon/dispatcher-usage.test.ts b/src/daemon/dispatcher-usage.test.ts new file mode 100644 index 000000000..3dc5170d2 --- /dev/null +++ b/src/daemon/dispatcher-usage.test.ts @@ -0,0 +1,209 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { HOUR, removeUsageRigs, requested, T0, usageRig } from "../history/testing.js"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +const threads = vi.hoisted(() => ({ + crashOnPost: false, + posts: [] as unknown[], + workers: [] as { terminate(): Promise }[], +})); + +vi.mock("node:worker_threads", async (importOriginal) => { + const actual = await importOriginal(); + class CountingWorker extends actual.Worker { + constructor(...args: ConstructorParameters) { + super(...args); + threads.workers.push(this); + } + + override postMessage(...args: Parameters["postMessage"]>) { + threads.posts.push(args[0]); + super.postMessage(...args); + if (threads.crashOnPost) void this.terminate(); + } + } + return { ...actual, Worker: CountingWorker }; +}); + +describe("Dispatcher: usage.get", () => { + afterEach(removeUsageRigs); + + const admin = session({ role: "admin" }); + + async function historyWithRows(options: { readonly gatewayLabel?: string } = {}) { + const rig = await usageRig(); + rig.emit("lease.requested", requested("R1")); + const gate = { open: true }; + const eventHistory = { + replay: (input?: { readonly sinceTs?: number }) => rig.history.replay(input), + replayPage: (input: { readonly beforeId?: string; readonly limit: number }) => + rig.history.replayPage(input), + status: () => ({ ...rig.history.status(), opened: gate.open && rig.history.status().opened }), + }; + return { eventHistory, gate, rig, ...options }; + } + + it("the worker handler answers HISTORY_NOT_KEPT with oldestTs and the message naming when the figures start when to is 1 ms before the oldest row, and figures when to is that row's time", async () => { + const { eventHistory, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + + await expect( + dispatcher.dispatch("usage.get", { from: T0 - 2 * HOUR, to: T0 - 1 }, admin), + ).rejects.toMatchObject({ + code: "HISTORY_NOT_KEPT", + details: { oldestTs: T0 }, + message: `The history does not reach back to the end of that window; its figures start at ${new Date(T0).toISOString()}.`, + }); + await expect( + dispatcher.dispatch("usage.get", { from: T0 - 2 * HOUR, to: T0 }, admin), + ).resolves.toMatchObject({ window: { to: T0 } }); + }); + + it("the worker handler answers HISTORY_UNAVAILABLE with the message naming daemon.log while history.status().opened is false, after answering figures while it was true", async () => { + const { eventHistory, gate, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const window = { from: T0, to: T0 + HOUR }; + + await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ + totals: { requests: 1 }, + }); + gate.open = false; + const before = threads.posts.length; + + await expect(dispatcher.dispatch("usage.get", window, admin)).rejects.toMatchObject({ + code: "HISTORY_UNAVAILABLE", + message: "The history could not be opened, so no figures can be read; see daemon.log.", + }); + expect(threads.posts).toHaveLength(before); + }); + + it("the worker handler passes includeLedger to the reader: it lists the window's requests when true, leaves requests out when absent or false, and refuses a non-boolean as BAD_REQUEST", async () => { + const { eventHistory, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const window = { from: T0, to: T0 + HOUR }; + + const listed = await dispatcher.dispatch( + "usage.get", + { ...window, includeLedger: true }, + admin, + ); + const absent = await dispatcher.dispatch("usage.get", window, admin); + const off = await dispatcher.dispatch("usage.get", { ...window, includeLedger: false }, admin); + + expect(listed.requests?.map((entry) => entry.requestId)).toEqual(["R1"]); + expect(absent).not.toHaveProperty("requests"); + expect(off).not.toHaveProperty("requests"); + await expect( + dispatcher.dispatch("usage.get", { ...window, includeLedger: "yes" } as never, admin), + ).rejects.toMatchObject({ code: "BAD_REQUEST" }); + }); + + it("the worker handler answers one worker entry, itself, with its instance id and gateway.label, or no label when none is set", async () => { + const { eventHistory, rig } = await historyWithRows(); + const labelled = await buildDispatcher({ + eventHistory, + gatewayLabel: "lab-a", + historyPath: rig.historyPath, + }); + const plain = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const window = { from: T0, to: T0 + HOUR }; + + const named = await labelled.dispatcher.dispatch("usage.get", window, admin); + const anonymous = await plain.dispatcher.dispatch("usage.get", window, admin); + + expect(named.workers).toMatchObject([{ id: "instance-1", label: "lab-a", requests: 1 }]); + expect(anonymous.workers).toHaveLength(1); + expect(anonymous.workers[0]).toMatchObject({ id: "instance-1", requests: 1 }); + expect(anonymous.workers[0]).not.toHaveProperty("label"); + }); + + it("the worker handler labels a requester from the token store, and leaves the label out when the daemon has no token store", async () => { + const { eventHistory, rig } = await historyWithRows(); + const withStore = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const withoutStore = await buildDispatcher({ + eventHistory, + historyPath: rig.historyPath, + withoutTokens: true, + }); + const { record } = await withStore.tokens.create("agent", "ci-bot"); + rig.emit("lease.requested", requested("R2", { requester: record.id })); + const window = { from: T0, to: T0 + HOUR }; + + const labelled = await withStore.dispatcher.dispatch("usage.get", window, admin); + const unlabelled = await withoutStore.dispatcher.dispatch("usage.get", window, admin); + + expect(labelled.requesters).toEqual([ + { granted: 0, heldTotalMs: 0, id: "alice", rejected: 0, requests: 1 }, + { granted: 0, heldTotalMs: 0, id: record.id, label: "ci-bot", rejected: 0, requests: 1 }, + ]); + expect(unlabelled.requesters.map((entry) => entry.id)).toEqual(["alice", record.id]); + expect(unlabelled.requesters.every((entry) => !("label" in entry))).toBe(true); + }); + + it("usage.get runs two calls for one window as one read on the read thread and gives both its answer, and two windows as two reads", async () => { + const { eventHistory, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const call = (from: number, to = from + HOUR) => + dispatcher.dispatch("usage.get", { from, to }, admin); + const readsOf = async (run: () => Promise) => { + const before = threads.posts.length; + await run(); + return threads.posts.length - before; + }; + await call(T0); + const single = await readsOf(() => call(T0)); + + const [first, second] = await Promise.all([call(T0), call(T0)]); + const sameWindow = await readsOf(() => Promise.all([call(T0), call(T0)])); + const twoWindows = await readsOf(() => Promise.all([call(T0), call(T0 - HOUR)])); + const otherStart = await readsOf(() => Promise.all([call(T0), call(T0 - HOUR, T0 + HOUR)])); + const otherEnd = await readsOf(() => Promise.all([call(T0), call(T0, T0 + 2 * HOUR)])); + + expect(single).toBeGreaterThan(0); + expect(sameWindow).toBe(single); + expect(twoWindows).toBe(2 * single); + expect(otherStart).toBe(2 * single); + expect(otherEnd).toBe(2 * single); + expect(first).toEqual(second); + expect(first.totals.requests).toBe(1); + }); + + it("dispose ends the read thread", async () => { + const { eventHistory, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + await dispatcher.dispatch("usage.get", { from: T0, to: T0 + HOUR }, admin); + const thread = threads.workers.at(-1) as unknown as import("node:worker_threads").Worker; + let exited = false; + thread.on("exit", () => { + exited = true; + }); + + dispatcher.dispose(); + + await vi.waitFor(() => expect(exited).toBe(true)); + }); + + it("usage.get answers INTERNAL to a call in flight when the read thread exits, and answers the next call from a new thread", async () => { + const { eventHistory, rig } = await historyWithRows(); + const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); + const window = { from: T0, to: T0 + HOUR }; + await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ + totals: { requests: 1 }, + }); + const started = threads.workers.length; + threads.crashOnPost = true; + + try { + await expect(dispatcher.dispatch("usage.get", window, admin)).rejects.toMatchObject({ + code: "INTERNAL", + }); + } finally { + threads.crashOnPost = false; + } + + await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ + totals: { requests: 1 }, + }); + expect(threads.workers.length).toBe(started + 1); + }); +}); diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 0fa32a088..aedd3877a 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -1,15 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { FakeDriver } from "../core/testing.js"; -import type { StateDatabase } from "../state/index.js"; -import { type EventEnvelope } from "../bus/index.js"; -import { - HOUR, - plainProjection, - removeUsageRigs, - requested, - T0, - usageRig, -} from "../history/testing.js"; +import { describe, expect, it, vi } from "vitest"; import { type CatalogReader } from "../core/index.js"; import { OPERATIONS, @@ -18,39 +7,9 @@ import { WORKER_VIEW_REFRESH_INTERVAL_MS, } from "../contract/index.js"; import { FakeClock } from "../ports/index.js"; -import type { DispatchSession } from "./dispatcher.js"; -import { Dispatcher, DispatchError } from "./dispatcher.js"; +import { DispatchError } from "./dispatcher.js"; import { classifyError } from "./error-code.js"; -import { - HOST_SYSTEM, - buildDispatcher, - flushPromises, - gibibyte, - session, -} from "./test-dispatcher.js"; - -const threads = vi.hoisted(() => ({ - crashOnPost: false, - posts: [] as unknown[], - workers: [] as { terminate(): Promise }[], -})); - -vi.mock("node:worker_threads", async (importOriginal) => { - const actual = await importOriginal(); - class CountingWorker extends actual.Worker { - constructor(...args: ConstructorParameters) { - super(...args); - threads.workers.push(this); - } - - override postMessage(...args: Parameters["postMessage"]>) { - threads.posts.push(args[0]); - super.postMessage(...args); - if (threads.crashOnPost) void this.terminate(); - } - } - return { ...actual, Worker: CountingWorker }; -}); +import { HOST_SYSTEM, buildDispatcher, flushPromises, session } from "./test-dispatcher.js"; class SettableClock extends FakeClock { #offset = 0; @@ -720,867 +679,6 @@ describe("Dispatcher: lease.release of a virtual device", () => { ); }); -describe("Dispatcher: baseline.save", () => { - it("ends the held lease, saves its device as version 1 of the named baseline with the files it held, and deletes the device", async () => { - const { clock, dispatcher, driver, eventBus, registry } = await buildDispatcher(); - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/login.txt", "signed in"); - - const saved = await dispatcher.dispatch( - "baseline.save", - { leaseId: grant.lease.id, name: "checkout", note: "app 4.2" }, - session({ role: "admin" }), - ); - - expect(saved).toEqual({ - baseline: { - mode: "full", - model: "iPhone 17 Pro", - name: "checkout", - note: "app 4.2", - osVersion: "26.5", - platform: "ios", - savedAt: new Date(clock.now()).toISOString(), - sizeBytes: "signed in".length, - version: 1, - }, - leaseId: grant.lease.id, - }); - expect(registry.snapshot.leases).toEqual([]); - expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["deleted"]); - const [source] = await driver.listSources(); - expect(driver.sourceFiles(source?.id ?? "")).toEqual( - new Map([["data/login.txt", "signed in"]]), - ); - expect( - eventBus - .replay() - .filter((envelope) => envelope.event === "lease.released") - .map((envelope) => envelope.payload), - ).toMatchObject([{ leaseId: grant.lease.id, reason: "explicit" }]); - }); - - it("records the image tag the saved device carries when its lease named none, so a lease naming that tag is granted and another tag is refused", async () => { - const { dispatcher } = await buildDispatcher({ - driverOptions: { - availableOsVersions: ["34"], - defaultImageTag: "google_apis_playstore", - images: [ - { abi: "arm64-v8a", runtime: "34", tag: "google_apis" }, - { abi: "arm64-v8a", runtime: "34", tag: "google_apis_playstore" }, - ], - platform: "android", - }, - }); - const grant = await dispatcher.dispatch( - "lease.request", - { model: "Pixel 8", osVersion: "34", platform: "android" }, - session(), - ); - const admin = session({ role: "admin" }); - - const saved = await dispatcher.dispatch( - "baseline.save", - { leaseId: grant.lease.id, name: "android-test" }, - admin, - ); - const listed = await dispatcher.dispatch("baseline.list", {}, admin); - const named = { baseline: "android-test", platform: "android" }; - const granted = await dispatcher.dispatch( - "lease.request", - { ...named, imageTag: "google_apis_playstore" }, - session(), - ); - await dispatcher.dispatch("lease.release", { leaseId: granted.lease.id }, session()); - const refused = await dispatcher - .dispatch("lease.request", { ...named, imageTag: "google_apis" }, session()) - .catch((caught: unknown) => caught); - - expect(saved.baseline.imageTag).toBe("google_apis_playstore"); - expect(listed.baselines.map((entry) => entry.imageTag)).toEqual(["google_apis_playstore"]); - expect(granted.device.spec.baseline).toEqual({ name: "android-test", version: 1 }); - expect(refused).toMatchObject({ - message: - "baseline android-test is image tag google_apis_playstore, the request asks for google_apis", - }); - }); -}); - -describe("Dispatcher: baseline.save versions", () => { - it("numbers each save under one name one more than the last, one at a time or two at once, and keeps only the newest version's source", async () => { - const { dispatcher, driver, registry } = await buildDispatcher(); - const admin = session({ role: "admin" }); - const lease = async (who: string): Promise => { - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session({ principal: who }), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/who", who); - return grant.lease.id; - }; - const save = (leaseId: string) => - dispatcher.dispatch("baseline.save", { leaseId, name: "checkout" }, admin); - const keptSources = async () => - (await driver.listSources()).map((source) => driver.sourceFiles(source.id)); - - const first = await save(await lease("tok_a")); - const second = await save(await lease("tok_b")); - expect([first.baseline.version, second.baseline.version]).toEqual([1, 2]); - expect(await keptSources()).toEqual([new Map([["data/who", "tok_b"]])]); - - const [c, d] = [await lease("tok_c"), await lease("tok_d")]; - const results = await Promise.all([save(c), save(d)]); - expect(results.map((result) => result.baseline.version).sort()).toEqual([3, 4]); - const newest = - results.find((result) => result.baseline.version === 4)?.leaseId === c ? "tok_c" : "tok_d"; - expect(await keptSources()).toEqual([new Map([["data/who", newest]])]); - }); -}); - -describe("Dispatcher: baseline.save and baseline.remove queue", () => { - it("runs a remove sent during a save after it, so the remove deletes the version the save made", async () => { - const { clock, dispatcher, registry } = await buildDispatcher({ - driverOptions: { latencyMs: { saveSource: 1_000 } }, - }); - const admin = session({ role: "admin" }); - const lease = async (): Promise => - ( - await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ) - ).lease.id; - const settled = async (...pending: Promise[]): Promise => { - let open = pending.length; - for (const promise of pending) void promise.then(() => (open -= 1)); - await vi.waitFor(() => { - clock.advance(1_000); - expect(open).toBe(0); - }); - }; - const first = dispatcher.dispatch( - "baseline.save", - { leaseId: await lease(), name: "checkout" }, - admin, - ); - await settled(first); - - const second = dispatcher.dispatch( - "baseline.save", - { leaseId: await lease(), name: "checkout" }, - admin, - ); - const removed = dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin); - await settled(second, removed); - - expect(await removed).toEqual({ deletedDevices: [], name: "checkout", versions: [2] }); - expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); - expect(registry.snapshot.leases).toEqual([]); - }); -}); - -describe("Dispatcher: baseline.save and nuke", () => { - it("makes nuke wait for a save in flight and remove its baseline, and answers a save queued behind nuke's maintenance with an unknown lease since nuke ended its lease", async () => { - const { clock, dispatcher, registry } = await buildDispatcher({ - driverOptions: { latencyMs: { saveSource: 1_000 } }, - includeNuke: true, - }); - const admin = session({ role: "admin" }); - const lease = async (who: string): Promise => - ( - await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session({ principal: who }), - ) - ).lease.id; - const [held, queued] = [await lease("tok_a"), await lease("tok_b")]; - const outcome = (promise: Promise): { settled: boolean } => { - const state = { settled: false }; - void promise.then( - () => (state.settled = true), - () => (state.settled = true), - ); - return state; - }; - const saving = dispatcher.dispatch("baseline.save", { leaseId: held, name: "first" }, admin); - const savingState = outcome(saving); - await vi.waitFor(() => { - expect(registry.snapshot.devices.map((device) => device.state)).toContain("reclaiming"); - }); - - const nuking = dispatcher.dispatch("nuke.run", { deleteDevices: true }, admin); - const nukeState = outcome(nuking); - const blocked = dispatcher.dispatch( - "baseline.save", - { leaseId: queued, name: "second" }, - admin, - ); - const blockedError = blocked.then( - () => undefined, - (rejected: unknown) => rejected, - ); - await vi.waitFor(() => { - clock.advance(1_000); - expect(savingState.settled).toBe(true); - }); - const [nuked, refusal] = await Promise.all([nuking, blockedError]); - - expect(nukeState.settled).toBe(true); - expect(nuked).toMatchObject({ releasedLeaseIds: [queued], removedBaselines: ["first"] }); - expect(refusal).toMatchObject({ leaseId: queued, name: "UnknownLeaseError" }); - expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); - expect(registry.snapshot.leases).toEqual([]); - }); -}); - -describe("Dispatcher: baseline.save failure", () => { - it("answers BASELINE_SAVE_FAILED when the driver cannot save, with the lease ended, the device quarantined and no baseline recorded", async () => { - const { dispatcher, driver, eventBus, registry } = await buildDispatcher(); - const admin = session({ role: "admin" }); - const request = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; - const grant = await dispatcher.dispatch("lease.request", request, session()); - driver.failOn("saveSource", 1, new Error("disk full")); - - const error = await dispatcher - .dispatch("baseline.save", { leaseId: grant.lease.id, name: "checkout" }, admin) - .then( - () => undefined, - (rejected: unknown) => rejected, - ); - - expect(error).toMatchObject({ - code: "BASELINE_SAVE_FAILED", - details: { leaseId: grant.lease.id, name: "checkout" }, - message: `baseline checkout was not saved: disk full. Lease ${grant.lease.id} has ended.`, - }); - expect(registry.snapshot.leases).toEqual([]); - expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["quarantined"]); - expect(await driver.listSources()).toEqual([]); - expect( - eventBus - .replay() - .filter((envelope) => envelope.event === "device.quarantined") - .map((envelope) => envelope.payload), - ).toMatchObject([{ kind: "shutdown-failed" }]); - const next = await dispatcher.dispatch("lease.request", request, session()); - const saved = await dispatcher.dispatch( - "baseline.save", - { leaseId: next.lease.id, name: "checkout" }, - admin, - ); - expect(saved.baseline.version).toBe(1); - }); -}); - -describe("Dispatcher: baseline.save refusals", () => { - it("refuses a bad name or note, a non-admin, an unknown lease and a physical lease, and ends no lease", async () => { - const { dispatcher, driver, engine, registry } = await buildDispatcher({ - physicalDevices: [ - { - createdAt: 1, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: [], - id: "dev_phys", - spec: { class: "tablet", model: "iPad Pro", osVersion: "18.4" }, - state: "ready", - }, - ], - }); - await engine.convergeRunningCapacity(); - driver.setPhysicalPresence("00008110-001A2B3C4D5E", true); - const virtual = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const physical = await dispatcher.dispatch( - "lease.request", - { model: "iPad Pro", physical: true, platform: "ios" }, - session({ principal: "tok_physical" }), - ); - const admin = session({ role: "admin" }); - const accepted = { leaseId: virtual.lease.id, name: "checkout" }; - expect(OPERATIONS["baseline.save"].input.safeParse(accepted).success).toBe(true); - - const refusals: readonly [string, unknown, DispatchSession, string, string?][] = [ - ["Checkout", { ...accepted, name: "Checkout" }, admin, "BAD_REQUEST"], - ["-checkout", { ...accepted, name: "-checkout" }, admin, "BAD_REQUEST"], - ["65 characters", { ...accepted, name: "a".repeat(65) }, admin, "BAD_REQUEST"], - ["201-character note", { ...accepted, note: "n".repeat(201) }, admin, "BAD_REQUEST"], - ["note with a newline", { ...accepted, note: "two\nlines" }, admin, "BAD_REQUEST"], - ["an agent token", accepted, session({ role: "agent" }), "FORBIDDEN"], - ["unknown lease", { ...accepted, leaseId: "lse_gone" }, admin, "UNKNOWN_LEASE"], - [ - "physical lease", - { ...accepted, leaseId: physical.lease.id }, - admin, - "BAD_REQUEST", - `lease ${physical.lease.id} is on a physical device; only a simulator or emulator can be saved as a baseline`, - ], - ]; - for (const [label, input, caller, code, message] of refusals) { - const error = await dispatcher.dispatch("baseline.save", input as never, caller).then( - () => undefined, - (rejected: unknown) => rejected, - ); - expect(classifyError(error), label).toBe(code); - if (message !== undefined) expect((error as Error).message, label).toBe(message); - expect(registry.snapshot.leases.map((lease) => lease.id).sort(), label).toEqual( - [virtual.lease.id, physical.lease.id].sort(), - ); - } - }); -}); - -describe("Dispatcher: baseline.list", () => { - it("lists the newest version of each baseline by name with its spec, note, size, saved time and stale reason, cleared when the runtime is back, and an agent may list", async () => { - const { clock, dispatcher, driver, registry } = await buildDispatcher(); - const agent = session({ role: "agent" }); - const admin = session({ role: "admin" }); - expect(await dispatcher.dispatch("baseline.list", {}, agent)).toEqual({ baselines: [] }); - const save = async (name: string, content: string, note?: string): Promise => { - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", content); - await dispatcher.dispatch( - "baseline.save", - { leaseId: grant.lease.id, name, ...(note === undefined ? {} : { note }) }, - admin, - ); - }; - await save("login", "signed in", "signed in"); - await save("checkout", "v1"); - await save("checkout", "version two"); - - expect(await dispatcher.dispatch("baseline.list", {}, agent)).toEqual({ - baselines: [ - { - mode: "full", - model: "iPhone 17 Pro", - name: "checkout", - osVersion: "26.5", - platform: "ios", - savedAt: new Date(clock.now()).toISOString(), - sizeBytes: "version two".length, - version: 2, - }, - { - mode: "full", - model: "iPhone 17 Pro", - name: "login", - note: "signed in", - osVersion: "26.5", - platform: "ios", - savedAt: new Date(clock.now()).toISOString(), - sizeBytes: "signed in".length, - version: 1, - }, - ], - }); - - const sources = await driver.listSources(); - const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; - const staleNames = async (): Promise<(string | undefined)[]> => - (await dispatcher.dispatch("baseline.list", {}, agent)).baselines.map( - (baseline) => baseline.staleReason, - ); - const [loginSource] = sources; - driver.setSources( - sources.map((source) => - source === loginSource ? { ...source, staleReason: reason } : source, - ), - ); - expect(await staleNames()).toEqual([undefined, reason]); - driver.setSources(sources); - expect(await staleNames()).toEqual([undefined, undefined]); - }); -}); - -describe("Dispatcher: baseline.list without a driver", () => { - it("lists a baseline whose platform has no driver as stale, without storing the reason or recording an event", async () => { - const { dispatcher, eventBus } = await buildDispatcher({ - seedState: ({ database }) => { - database - .prepare( - "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES ('droid', 1, 'android', 'Pixel 9', '36', 'full', '2026-11-02T13:40:00.000Z', 'src-droid')", - ) - .run(); - }, - }); - const agent = session({ role: "agent" }); - - const listed = await dispatcher.dispatch("baseline.list", {}, agent); - expect(listed.baselines).toMatchObject([ - { name: "droid", staleReason: "the android driver is not available on this worker" }, - ]); - expect(eventBus.replay().filter((envelope) => envelope.event === "baseline.stale")).toEqual([]); - }); -}); - -/** why: #1207 writes a device's baseline at provision; until then a test writes the registry rows a remove reads. */ -function seedBaselineRows(state: StateDatabase): void { - const { database } = state; - const spec = ["ios", "iPhone 17 Pro", "26.5", "full"]; - for (const [name, sourceId] of [ - ["login", "src-login"], - ["checkout", "src-checkout"], - ["busy", "src-busy"], - ] as const) { - database - .prepare( - "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES (?, 1, ?, ?, ?, ?, '2026-11-02T13:40:00.000Z', ?)", - ) - .run(name, ...spec, sourceId); - } - for (const [id, fakeId, deviceState, baseline] of [ - ["dev_idle", "fake-ios-1", "shutdown", "checkout"], - ["dev_idle_b", "fake-ios-4", "shutdown", "checkout"], - ["dev_leased", "fake-ios-2", "leased", "busy"], - ["dev_quarantined", "fake-ios-3", "quarantined", "login"], - ] as const) { - database - .prepare( - "INSERT INTO devices (id, platform, physical, driver_device_id, state, model, os_version, mode, created_at, baseline_name, baseline_version) VALUES (?, 'ios', 0, ?, ?, 'iPhone 17 Pro', '26.5', 'full', 1, ?, 1)", - ) - .run(id, fakeId, deviceState, baseline); - database - .prepare("INSERT INTO fake_device_data (device_id, fake_device_id) VALUES (?, ?)") - .run(id, fakeId); - } - database - .prepare( - "INSERT INTO leases (id, device_id, requester_id, owner_id, granted_at, ttl_ms, ttl_deadline, last_renewed_at, id_chosen_by_requester) VALUES ('lse_busy', 'dev_leased', 'tok_a', 'tok_a', 1, 3600000, 3600001, 1, 0)", - ) - .run(); -} - -describe("Dispatcher: baselines beside a gateway's worker view", () => { - it("answers worker.list with the baselines its own baseline.list gives, and with none when no baseline service is wired", async () => { - const { dispatcher, driver, registry } = await buildDispatcher(); - const admin = session({ role: "admin" }); - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", "signed in"); - await dispatcher.dispatch("baseline.save", { leaseId: grant.lease.id, name: "login" }, admin); - - const { baselines } = await dispatcher.dispatch("baseline.list", {}, admin); - const { workers } = await dispatcher.dispatch("worker.list", {}, admin); - - expect(baselines.map((baseline) => baseline.name)).toEqual(["login"]); - expect(workers[0]?.baselines).toEqual(baselines); - - const bare = await buildDispatcher({ withoutBaselines: true }); - const listed = await bare.dispatcher.dispatch("worker.list", {}, admin); - expect(listed.workers[0]).not.toHaveProperty("baselines"); - }); - - it("refuses baseline.remove with a workerId, which names a gateway's worker", async () => { - const { dispatcher } = await buildDispatcher(); - - await expect( - dispatcher.dispatch( - "baseline.remove", - { name: "login", workerId: "wrk_a" }, - session({ role: "admin" }), - ), - ).rejects.toMatchObject({ - code: "BAD_REQUEST", - message: - "workerId names a gateway's worker, and this daemon is not a gateway; remove without --worker", - }); - }); -}); - -describe("Dispatcher: baseline.remove", () => { - it("removes an unused baseline with its source and the idle devices made from it, refuses one a lease uses, an unknown name and an agent", async () => { - const { dispatcher, driver, engine, registry } = await buildDispatcher({ - seedState: seedBaselineRows, - }); - const spec = { - model: "iPhone 17 Pro", - osVersion: "26.5", - physical: false, - platform: "ios", - } as const; - for (let count = 0; count < 4; count += 1) await driver.provision(spec); - driver.setSources( - ["login", "checkout", "busy"].map((name) => ({ - id: `src-${name}`, - kind: "baseline" as const, - model: "iPhone 17 Pro", - osVersion: "26.5", - })), - ); - const admin = session({ role: "admin" }); - const refused = async (name: string, caller: DispatchSession): Promise => - dispatcher.dispatch("baseline.remove", { name }, caller).then( - () => undefined, - (rejected: unknown) => rejected, - ); - const listed = async (): Promise => - (await dispatcher.dispatch("baseline.list", {}, admin)).baselines.map(({ name }) => name); - - expect(await refused("login", session({ role: "agent" }))).toMatchObject({ - code: "FORBIDDEN", - }); - expect(await refused("nothere", admin)).toMatchObject({ - code: "UNKNOWN_BASELINE", - details: { name: "nothere" }, - message: 'no baseline named "nothere" on this worker', - }); - expect(await refused("Checkout", admin)).toMatchObject({ code: "BAD_REQUEST" }); - expect(await refused("busy", admin)).toMatchObject({ - code: "BASELINE_IN_USE", - details: { leaseId: "lse_busy", name: "busy" }, - message: "baseline busy is in use by lease lse_busy", - }); - expect(await listed()).toEqual(["busy", "checkout", "login"]); - - expect(await dispatcher.dispatch("baseline.remove", { name: "login" }, admin)).toEqual({ - deletedDevices: [], - name: "login", - versions: [1], - }); - expect(engine.claimReader.isClaimed("dev_quarantined")).toBe(false); - // why: the first idle device's destroy throws; the second must still be disposed and no cleanup claim left held. - driver.failOn("destroy", 1, new Error("destroy failed")); - await expect( - dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin), - ).rejects.toThrow("destroy failed"); - expect(engine.claimReader.isClaimed("dev_idle")).toBe(false); - expect(engine.claimReader.isClaimed("dev_idle_b")).toBe(false); - expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toContainEqual([ - "dev_idle_b", - "deleted", - ]); - expect(await listed()).toEqual(["busy", "checkout"]); - expect(await dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin)).toEqual({ - deletedDevices: ["dev_idle"], - name: "checkout", - versions: [1], - }); - - expect(await listed()).toEqual(["busy"]); - expect((await driver.listSources()).map((source) => source.id)).toEqual(["src-busy"]); - expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toEqual([ - ["dev_idle", "deleted"], - ["dev_idle_b", "deleted"], - ["dev_leased", "leased"], - ["dev_quarantined", "quarantined"], - ]); - }); -}); - -describe("Dispatcher: doctor baseline sources", () => { - it("lists a baseline's source with its name and version, a ready baseline source no row names as an orphan only doctor --purge-orphans deletes, and finds a stale baseline", async () => { - const { dispatcher, driver, eventBus, registry } = await buildDispatcher(); - const admin = session({ role: "admin" }); - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", "abc"); - await dispatcher.dispatch( - "baseline.save", - { leaseId: grant.lease.id, name: "checkout" }, - admin, - ); - const [named] = await driver.listSources(); - const unnamed = { - id: "left-over", - kind: "baseline", - model: "iPhone 17 Pro", - osVersion: "26.5", - } as const; - driver.setSources([named as NonNullable, unnamed]); - - const listed = await dispatcher.dispatch("doctor.run", {}, admin); - expect(listed.findings).toEqual([]); - expect(listed.sources).toEqual([ - { - baselineName: "checkout", - baselineVersion: 1, - kind: "baseline", - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - sizeBytes: 3, - }, - { - kind: "baseline", - model: "iPhone 17 Pro", - orphan: true, - osVersion: "26.5", - platform: "ios", - }, - ]); - expect((await driver.listSources()).map((source) => source.id)).toContain("left-over"); - - const purged = await dispatcher.dispatch("doctor.run", { purgeOrphans: true }, admin); - expect(purged.sources.map((source) => source.orphan)).toEqual([undefined]); - expect((await driver.listSources()).map((source) => source.id)).not.toContain("left-over"); - - const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; - driver.setSources([{ ...(named as NonNullable), staleReason: reason }]); - await dispatcher.dispatch("doctor.run", {}, admin); - expect( - eventBus - .replay() - .filter((envelope) => envelope.event === "baseline.stale") - .map((envelope) => envelope.payload), - ).toEqual([{ name: "checkout", reason, version: 1 }]); - }); -}); - -describe("Dispatcher: baseline events", () => { - it("records baseline.saved, baseline.removed with its reason (nuke --delete-devices removes every baseline, plain nuke keeps them) and baseline.stale, once each, after the change is committed", async () => { - const { dispatcher, driver, eventBus, registry, state } = await buildDispatcher({ - driverOptions: { componentSizes: { "27.0": 1_000 }, knownModels: ["iPhone 17 Pro"] }, - includeNuke: true, - }); - const admin = session({ principal: "tok_admin", role: "admin" }); - // why: what the table held at the moment each event reached a subscriber, to prove the change is committed before the event. - const seen: { removedRows: number[]; staleReasons: unknown[] } = { - removedRows: [], - staleReasons: [], - }; - eventBus.subscribe("baseline.removed", ({ payload }) => { - const row = state.database - .prepare("SELECT COUNT(*) AS count FROM baselines WHERE name = ? AND version = ?") - .get(payload.name, payload.version); - seen.removedRows.push(Number(row?.count)); - }); - eventBus.subscribe("baseline.stale", ({ payload }) => { - const row = state.database - .prepare("SELECT stale_reason FROM baselines WHERE name = ? AND version = ?") - .get(payload.name, payload.version); - seen.staleReasons.push(row?.stale_reason); - }); - await dispatcher.dispatch("component.install", { platform: "ios", version: "27.0" }, admin); - const events = (name: string): unknown[] => - eventBus - .replay() - .filter((envelope) => envelope.event === name) - .map((envelope) => envelope.payload); - const save = async (content: string, name = "checkout"): Promise => { - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/state", content); - await dispatcher.dispatch("baseline.save", { leaseId: grant.lease.id, name }, admin); - return grant.lease.id; - }; - - const first = await save("v1"); - expect(events("baseline.saved")).toEqual([ - { - leaseId: first, - mode: "full", - model: "iPhone 17 Pro", - name: "checkout", - osVersion: "26.5", - platform: "ios", - requesterId: "tok_admin", - sizeBytes: 2, - version: 1, - }, - ]); - expect(events("device.deleted")).toMatchObject([{ initiator: "baseline-save" }]); - expect(events("baseline.removed")).toEqual([]); - - await save("v2"); - expect(events("baseline.saved")).toHaveLength(2); - expect(events("baseline.removed")).toEqual([ - { name: "checkout", reason: "replaced", requesterId: "tok_admin", version: 1 }, - ]); - - const [source] = await driver.listSources(); - const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; - const check = async (staleReason?: string): Promise => { - driver.setSources([ - { - ...(source as NonNullable), - ...(staleReason === undefined ? {} : { staleReason }), - }, - ]); - await dispatcher.dispatch("baseline.list", {}, admin); - }; - await check(reason); - await check(reason); - expect(events("baseline.stale")).toEqual([{ name: "checkout", reason, version: 2 }]); - await check(); - await check(reason); - expect(events("baseline.stale")).toHaveLength(2); - - await check(); - driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); - await Promise.all([ - dispatcher.dispatch("baseline.list", {}, admin), - dispatcher.dispatch("baseline.list", {}, admin), - ]); - expect(events("baseline.stale")).toHaveLength(3); - await check(); - driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); - await dispatcher.dispatch("component.remove", { platform: "ios", version: "27.0" }, admin); - await vi.waitFor(() => { - expect(events("baseline.stale")).toHaveLength(4); - }); - - await dispatcher.dispatch("baseline.remove", { name: "checkout" }, admin); - expect(events("baseline.removed")).toEqual([ - { name: "checkout", reason: "replaced", requesterId: "tok_admin", version: 1 }, - { name: "checkout", reason: "operator", requesterId: "tok_admin", version: 2 }, - ]); - - await save("again"); - await save("signed in", "login"); - expect(await dispatcher.dispatch("nuke.run", { deleteDevices: false }, admin)).toMatchObject({ - removedBaselines: [], - }); - expect((await dispatcher.dispatch("baseline.list", {}, admin)).baselines).toHaveLength(2); - driver.failOn( - "destroySource", - driver.calls.filter((call) => call.operation === "destroySource").length + 1, - new Error("simctl delete failed"), - ); - expect(await dispatcher.dispatch("nuke.run", { deleteDevices: true }, admin)).toMatchObject({ - removedBaselines: ["checkout", "login"], - }); - expect(events("baseline.removed").slice(2)).toEqual([ - { name: "checkout", reason: "nuke", version: 1 }, - { name: "login", reason: "nuke", version: 1 }, - ]); - expect(await dispatcher.dispatch("baseline.list", {}, admin)).toEqual({ baselines: [] }); - expect(await driver.listSources()).toHaveLength(1); - expect(seen.removedRows).toEqual([0, 0, 0, 0]); - expect(seen.staleReasons).toEqual([reason, reason, reason, reason]); - }); -}); - -describe("Dispatcher: events.replay", () => { - it("events.replay with sinceTs returns what the event history returns", async () => { - const fromHistory: EventEnvelope[] = [ - { - id: "evt_7", - seq: 7, - timestamp: 50, - event: "daemon.stopping", - payload: { reason: "x" }, - module: "d", - }, - ]; - const asked: unknown[] = []; - const { dispatcher } = await buildDispatcher({ - eventHistory: { - replay: async (input) => { - asked.push(input); - return fromHistory; - }, - replayPage: async () => ({ events: [], kind: "events" }), - status: () => ({ opened: true, writing: true }), - }, - }); - - await expect( - dispatcher.dispatch("events.replay", { sinceTs: 10 }, session({ role: "admin" })), - ).resolves.toEqual(fromHistory); - expect(asked).toEqual([{ sinceTs: 10 }]); - }); -}); - -describe("Dispatcher: events.replay paging", () => { - afterEach(removeUsageRigs); - - const admin = session({ role: "admin" }); - - async function pagedRig( - rows: number, - options: Parameters[0] = {}, - ): Promise<{ - readonly rig: Awaited>; - readonly ids: string[]; - readonly dispatcher: Dispatcher; - }> { - const rig = await usageRig(options); - for (let row = 0; row < rows; row += 1) { - rig.at(T0 + row); - rig.emit("daemon.stopping", { reason: `r${row + 1}` }); - } - const { dispatcher } = await buildDispatcher({ eventHistory: rig.history }); - return { dispatcher, ids: rig.bus.replay().map((event) => event.id), rig }; - } - - const reasons = (events: readonly { readonly payload?: unknown }[]): unknown[] => - events.map((event) => (event.payload as { reason: string }).reason); - - it("events.replay with beforeId and limit pages through the worker dispatcher over a real history: the page, HISTORY_NOT_KEPT, an empty table, HISTORY_UNAVAILABLE, and 100 at most for beforeId alone", async () => { - const { dispatcher, ids } = await pagedRig(150); - const page = await dispatcher.dispatch( - "events.replay", - { beforeId: ids[4] as string, limit: 2 }, - admin, - ); - expect(reasons(page)).toEqual(["r3", "r4"]); - const unbounded = await dispatcher.dispatch( - "events.replay", - { beforeId: ids[149] as string }, - admin, - ); - expect(unbounded).toHaveLength(100); - await expect( - dispatcher.dispatch("events.replay", { beforeId: "evt_gone", limit: 2 }, admin), - ).rejects.toMatchObject({ code: "HISTORY_NOT_KEPT", details: { oldestTs: T0 } }); - - const empty = await pagedRig(0); - await expect( - empty.dispatcher.dispatch("events.replay", { beforeId: "evt_any", limit: 2 }, admin), - ).resolves.toEqual([]); - - const failing = plainProjection("failing", () => ({ - apply: () => { - throw new Error("projection broke"); - }, - prune: () => undefined, - })); - const notWriting = await pagedRig(1, { projections: [failing] }); - expect(notWriting.rig.history.status().writing).toBe(false); - const unstarted = await buildDispatcher(); - for (const { dispatcher: target } of [notWriting, unstarted]) { - await expect( - target.dispatch("events.replay", { beforeId: "evt_any", limit: 2 }, admin), - ).rejects.toMatchObject({ code: "HISTORY_UNAVAILABLE" }); - } - }); - - it("events.replay with limit alone returns the newest n events, oldest first, through the worker dispatcher", async () => { - const { dispatcher } = await pagedRig(5); - const page = await dispatcher.dispatch("events.replay", { limit: 2 }, admin); - expect(reasons(page)).toEqual(["r4", "r5"]); - }); -}); - describe("Dispatcher: nuke.run", () => { it("rejects an agent session with FORBIDDEN, without deleting anything", async () => { const { dispatcher, registry } = await buildDispatcher({ includeNuke: true }); @@ -1734,726 +832,3 @@ describe("Dispatcher: error codes", () => { ).rejects.toMatchObject({ name: "RuntimeMissingError" }); }); }); - -describe("Dispatcher: lease.request naming a baseline", () => { - it("gives a copy of the baseline, and the grant, lease.granted and device.provisioned carry its name and version, and lease.requested, lease.rejected and lease.declined the name the request gave", async () => { - const { dispatcher, driver, eventBus, registry } = await buildDispatcher({ - capacity: { - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 3, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - strategy: "resource", - }, - driverOptions: { knownModels: ["iPhone 17 Pro"] }, - }); - const first = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - const held = registry.snapshot.virtualDevices.find((device) => device.id === first.device.id); - driver.writeDeviceFile(held?.driverDeviceId ?? "", "data/login.txt", "signed in"); - await dispatcher.dispatch( - "baseline.save", - { leaseId: first.lease.id, name: "checkout" }, - session({ role: "admin" }), - ); - - const grant = await dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform: "ios" }, - session(), - ); - - expect(grant.device.spec).toEqual({ - baseline: { name: "checkout", version: 1 }, - model: "iPhone 17 Pro", - osVersion: "26.5", - physical: false, - platform: "ios", - }); - expect(grant.device.id).not.toBe(first.device.id); - const copy = registry.snapshot.virtualDevices.find((device) => device.id === grant.device.id); - expect(driver.deviceFile(copy?.driverDeviceId ?? "", "data/login.txt")).toBe("signed in"); - expect( - eventBus - .replay() - .filter((envelope) => envelope.event === "lease.granted") - .map((envelope) => envelope.payload) - .at(-1), - ).toMatchObject({ baseline: { name: "checkout", version: 1 }, deviceId: grant.device.id }); - - const full = { baseline: "checkout", noWait: true, platform: "ios" } as const; - await expect( - dispatcher.dispatch("lease.request", { ...full, requesterId: "agent-2" }, session()), - ).rejects.toMatchObject({ - code: "NO_CAPACITY", - }); - await expect( - dispatcher.dispatch( - "lease.request", - { ...full, fleetRequestId: "req_gw1" }, - session({ isGatewayUplink: true, principal: "tok_gateway", role: "admin" }), - ), - ).rejects.toMatchObject({ code: "NO_CAPACITY" }); - const specOf = (event: string, field: "requestSpec" | "spec") => - eventBus - .replay() - .filter((envelope) => envelope.event === event) - .map((envelope) => (envelope.payload as Record)[field]) - .map((spec) => spec?.baseline) - .filter((baseline) => baseline !== undefined); - expect({ - declined: specOf("lease.declined", "requestSpec"), - provisioned: specOf("device.provisioned", "spec"), - rejected: specOf("lease.rejected", "requestSpec"), - requested: specOf("lease.requested", "requestSpec"), - }).toEqual({ - declined: ["checkout"], - provisioned: [{ name: "checkout", version: 1 }], - rejected: ["checkout"], - requested: ["checkout", "checkout", "checkout"], - }); - - let leaseId = grant.lease.id; - for (const matching of [{ model: "iphone 17 pro" }, { osVersion: ">=26" }]) { - await dispatcher.dispatch("lease.release", { leaseId }, session()); - const again = await dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform: "ios", ...matching }, - session(), - ); - expect(again.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); - leaseId = again.lease.id; - } - }); -}); - -describe("Dispatcher: lease.request naming a baseline that cannot serve it", () => { - it("fails at once, provisioning nothing, as UNKNOWN_BASELINE, BASELINE_MISMATCH naming the first differing field, BASELINE_STALE or BAD_REQUEST", async () => { - const clock = new FakeClock(1_000); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const { dispatcher, driver, registry } = await buildDispatcher({ - clock, - driverOptions: { knownModels: ["iPhone 17 Pro"] }, - extraDrivers: [android], - }); - const admin = session({ role: "admin" }); - const first = await dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ); - await dispatcher.dispatch( - "baseline.save", - { leaseId: first.lease.id, name: "checkout" }, - admin, - ); - const provisions = (): number => - driver.calls.filter((call) => call.operation === "provision").length; - const failure = async (input: Record) => { - const error = await dispatcher - .dispatch("lease.request", input as never, session()) - .catch((caught: unknown) => caught); - return error instanceof Error - ? { - code: classifyError(error), - details: (error as DispatchError).details, - message: error.message, - } - : error; - }; - const named = { baseline: "checkout", platform: "ios" }; - - const control = await dispatcher.dispatch( - "lease.request", - { ...named, mode: "full", model: "iPhone 17 Pro", osVersion: "26.5" } as never, - session(), - ); - expect(control.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); - await dispatcher.dispatch("lease.release", { leaseId: control.lease.id }, session()); - const provisioned = provisions(); - - const mismatch = (field: string, message: string) => ({ - code: "BASELINE_MISMATCH", - details: { baseline: "checkout", field }, - message, - }); - const failures = []; - for (const input of [ - { ...named, baseline: "chekout" }, - { ...named, platform: "android" }, - { ...named, model: "iPhone 16" }, - { ...named, osVersion: "25.0" }, - { ...named, mode: "slim" }, - { ...named, imageTag: "google_apis" }, - { ...named, mode: "slim", osVersion: "25.0" }, - { ...named, imageTag: "google_apis", mode: "slim" }, - ]) { - failures.push(await failure(input)); - } - expect(failures).toEqual([ - { - code: "UNKNOWN_BASELINE", - details: { name: "chekout" }, - message: 'no baseline named "chekout" on this worker', - }, - mismatch("platform", "baseline checkout is ios, the request asks for android"), - mismatch("model", "baseline checkout is iPhone 17 Pro, the request asks for iPhone 16"), - mismatch("osVersion", "baseline checkout is iOS 26.5, the request asks for 25.0"), - mismatch("mode", "baseline checkout is full, the request asks for slim"), - mismatch("imageTag", "baseline checkout is image tag none, the request asks for google_apis"), - mismatch("osVersion", "baseline checkout is iOS 26.5, the request asks for 25.0"), - mismatch("mode", "baseline checkout is full, the request asks for slim"), - ]); - - const [source] = await driver.listSources(); - const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; - driver.setSources([{ ...(source as NonNullable), staleReason: reason }]); - await dispatcher.dispatch("baseline.list", {}, admin); - expect(await failure(named)).toEqual({ - code: "BASELINE_STALE", - details: { baseline: "checkout", reason }, - message: `baseline checkout is stale: ${reason}`, - }); - expect(provisions()).toBe(provisioned); - expect(registry.snapshot.devices.filter((device) => device.state !== "deleted")).toEqual([]); - - expect(await failure({ ...named, physical: true })).toMatchObject({ - code: "BAD_REQUEST", - message: expect.stringContaining("`baseline` has no meaning for a physical device"), - }); - expect(await failure({ ...named, class: "tablet" })).toMatchObject({ - code: "BAD_REQUEST", - message: expect.stringContaining( - "a request names a baseline or a class, not both: send only one of `baseline` and `class`", - ), - }); - }); -}); - -describe("Dispatcher: a baseline lease is released", () => { - it.each([ - { platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, - { platform: "android", model: "Pixel 8", osVersion: "36" }, - ] as const)( - "the next lease naming the baseline gets a new copy with none of the previous lease's changes ($platform)", - async ({ model, osVersion, platform }) => { - const clock = new FakeClock(1_000); - const android = new FakeDriver({ availableOsVersions: ["36"], clock, platform: "android" }); - const { dispatcher, driver, registry } = await buildDispatcher({ - clock, - driverOptions: { knownModels: ["iPhone 17 Pro"] }, - extraDrivers: [android], - }); - const device = platform === "ios" ? driver : android; - const fileOf = (deviceId: string, path: string): string | undefined => - device.deviceFile( - registry.snapshot.virtualDevices.find((candidate) => candidate.id === deviceId) - ?.driverDeviceId ?? "", - path, - ); - const operator = await dispatcher.dispatch( - "lease.request", - { model, osVersion, platform }, - session(), - ); - const prepared = registry.snapshot.virtualDevices.find( - (candidate) => candidate.id === operator.device.id, - ); - device.writeDeviceFile(prepared?.driverDeviceId ?? "", "data/login.txt", "signed in"); - await dispatcher.dispatch( - "baseline.save", - { leaseId: operator.lease.id, name: "checkout" }, - session({ role: "admin" }), - ); - - const first = await dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform }, - session(), - ); - expect(fileOf(first.device.id, "data/login.txt")).toBe("signed in"); - device.writeDeviceFile( - registry.snapshot.virtualDevices.find((candidate) => candidate.id === first.device.id) - ?.driverDeviceId ?? "", - "data/session.txt", - "left behind", - ); - await dispatcher.dispatch("lease.release", { leaseId: first.lease.id }, session()); - const second = await dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform }, - session(), - ); - - expect(second.device.id).not.toBe(first.device.id); - expect(second.device.spec.baseline).toEqual({ name: "checkout", version: 1 }); - expect(fileOf(second.device.id, "data/login.txt")).toBe("signed in"); - expect(fileOf(second.device.id, "data/session.txt")).toBeUndefined(); - }, - ); -}); - -// why: an idle copy of version 1 is what a warm pool leaves behind; the registry is written as the pool's provisioner would. -async function seedIdleCopy( - driver: FakeDriver, - registry: Awaited>["registry"], - model: string, -): Promise<{ readonly id: string }> { - const [source] = await driver.listSources(); - const spec = { - baseline: { name: "checkout", version: 1 }, - model, - osVersion: "26.5", - physical: false, - platform: "ios", - } as const; - const driverDevice = await driver.provision(spec, { source: source?.id ?? "" }); - const booted = await driver.makeReady(driverDevice, { mode: "full", purpose: "prepare" }); - const registered = await registry.registerDevice({ - driverData: driverDevice.driverData, - driverDeviceId: driverDevice.deviceId, - provisionDuration: 0, - spec, - }); - return registry.transitionDevice( - registered.id, - "ready", - { event: "device.ready", payload: { bootDuration: 0, deviceId: registered.id } }, - booted.mode === undefined ? undefined : { mode: booted.mode }, - ); -} - -type ReplaceScenario = "running" | "idle" | "waiting" | "making"; - -const replaceModel = "iPhone 17 Pro"; - -async function buildReplaceHarness(scenario: ReplaceScenario) { - const clock = new FakeClock(1_000); - const oneDevice = { - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 3, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - strategy: "resource", - } as const; - const built = await buildDispatcher({ - ...(scenario === "waiting" ? { capacity: oneDevice } : {}), - clock, - driverOptions: { - knownModels: [replaceModel], - ...(scenario === "making" ? { latencyMs: { provision: 5_000 } } : {}), - }, - }); - const { dispatcher, driver, engine, eventBus, registry, state } = built; - const settle = async (pending: Promise, step = 5_000): Promise => { - let done = false; - void pending.then( - () => (done = true), - () => (done = true), - ); - while (!done) { - clock.advance(step); - await new Promise((resolve) => setImmediate(resolve)); - } - return pending; - }; - const lease = (input: Record) => - settle(dispatcher.dispatch("lease.request", { platform: "ios", ...input } as never, session())); - const driverIdOf = (deviceId: string): string => - registry.snapshot.virtualDevices.find((device) => device.id === deviceId)?.driverDeviceId ?? ""; - const saveCheckout = (leaseId: string, step?: number) => - settle( - dispatcher.dispatch( - "baseline.save", - { leaseId, name: "checkout" }, - session({ role: "admin" }), - ), - step, - ); - return { - clock, - dispatcher, - driver, - engine, - fileOf: (deviceId: string) => driver.deviceFile(driverIdOf(deviceId), "data/version.txt"), - lease, - registry, - copiesOf: (version: number): number => - registry.snapshot.virtualDevices.filter((device) => device.spec.baseline?.version === version) - .length, - removed: () => - eventBus - .replay() - .filter((envelope) => envelope.event === "baseline.removed") - .map((envelope) => envelope.payload), - saveCheckout, - saveFrom: async (requesterId: string, marker: string): Promise => { - const held = await lease({ model: replaceModel, requesterId }); - driver.writeDeviceFile(driverIdOf(held.device.id), "data/version.txt", marker); - await saveCheckout(held.lease.id); - }, - settle, - stateOf: (deviceId: string): string | undefined => - registry.snapshot.virtualDevices.find((device) => device.id === deviceId)?.state, - versions: (): number[] => - state.database - .prepare("SELECT version FROM baselines WHERE name = 'checkout' ORDER BY version") - .all() - .map((row) => Number(row.version)), - writeFile: (deviceId: string, marker: string) => - driver.writeDeviceFile(driverIdOf(deviceId), "data/version.txt", marker), - }; -} - -type ReplaceHarness = Awaited>; - -/** why: the v1 baseline exists; the scenario's lease, idle copy or waiting request is arranged around the operator's lease. */ -async function arrangeReplace(harness: ReplaceHarness, scenario: ReplaceScenario) { - await harness.saveFrom("operator", "one"); - expect(harness.versions()).toEqual([1]); - const running = - scenario === "running" - ? await harness.lease({ baseline: "checkout", requesterId: "agent-1" }) - : undefined; - const idle = - scenario === "idle" - ? await seedIdleCopy(harness.driver, harness.registry, replaceModel) - : undefined; - const operator = await harness.lease({ model: replaceModel, requesterId: "operator" }); - const waiting = - scenario === "waiting" || scenario === "making" - ? harness.dispatcher.dispatch( - "lease.request", - { baseline: "checkout", platform: "ios", requesterId: "agent-2" }, - session(), - ) - : undefined; - for (let turn = 0; waiting !== undefined && turn < 10; turn += 1) { - await new Promise((resolve) => setImmediate(resolve)); - } - return { idle, operator, running, waiting }; -} - -describe("Dispatcher: an operator replaces a baseline", () => { - it.each([ - { label: "a lease is running", case: "running", oldCopies: 1 }, - { label: "no lease is running", case: "idle", oldCopies: 1 }, - { - label: "the running lease's delete fails at its release", - case: "running", - oldCopies: 1, - failDelete: true, - }, - { label: "a request is waiting", case: "waiting", oldCopies: 0 }, - { label: "a copy is being made", case: "making", oldCopies: 1 }, - ] as const)( - "running leases keep their device, the next lease gets the new version, a waiting request too, idle old-version devices are destroyed, a copy of the old version finished after the save is deleted and never granted, and the old version goes with its last device ($label)", - async ({ case: scenario, oldCopies, ...rest }) => { - const failDelete = "failDelete" in rest; - const harness = await buildReplaceHarness(scenario); - const { idle, operator, running, waiting } = await arrangeReplace(harness, scenario); - harness.writeFile(operator.device.id, "two"); - // why: the first save in the arrangement and this lease's own save destroy a device before the idle copy. - if (failDelete) harness.driver.failOn("destroy", 3, new Error("destroy failed")); - - await harness.saveCheckout(operator.lease.id, 100); - if (running !== undefined) { - expect(harness.stateOf(running.device.id)).toBe("leased"); - expect(harness.versions()).toEqual([1, 2]); - expect(harness.removed()).toEqual([]); - } - if (idle !== undefined) expect(harness.stateOf(idle.id)).toBe("deleted"); - const next = await (waiting === undefined - ? harness.lease({ baseline: "checkout", requesterId: "agent-3" }) - : harness.settle(waiting)); - expect(next.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); - expect(harness.fileOf(next.device.id)).toBe("two"); - expect(harness.copiesOf(1)).toBe(oldCopies); - if (running !== undefined) { - await harness.dispatcher.dispatch( - "lease.release", - { leaseId: running.lease.id }, - session(), - ); - } - await new Promise((resolve) => setImmediate(resolve)); - if (failDelete && running !== undefined) { - // why: the last device of version 1 leaves through the quarantine's delete retry, and that drops the version. - expect(harness.stateOf(running.device.id)).toBe("quarantined"); - expect(harness.versions()).toEqual([1, 2]); - harness.clock.advance(60 * 60_000); - for (let turn = 0; turn < 10; turn += 1) { - await new Promise((resolve) => setImmediate(resolve)); - } - expect(harness.stateOf(running.device.id)).toBe("deleted"); - } - expect(harness.versions()).toEqual([2]); - expect(harness.removed()).toEqual([ - expect.objectContaining({ name: "checkout", reason: "replaced", version: 1 }), - ]); - expect(await harness.driver.listSources()).toHaveLength(1); - }, - ); -}); - -describe("Dispatcher: an operator replaces a baseline while an idle old copy will not delete", () => { - it("the save succeeds, the idle old-version device stays shut down, and version 1 stays with it", async () => { - const harness = await buildReplaceHarness("idle"); - const { idle, operator } = await arrangeReplace(harness, "idle"); - harness.writeFile(operator.device.id, "two"); - // why: the first save in the arrangement and this lease's own save destroy a device before the idle copy. - harness.driver.failOn("destroy", 3, new Error("destroy failed")); - - await harness.saveCheckout(operator.lease.id, 100); - - if (idle === undefined) throw new Error("the arrangement left no idle device"); - expect(harness.stateOf(idle.id)).toBe("shutdown"); - expect(harness.versions()).toEqual([1, 2]); - expect(harness.removed()).toEqual([]); - const next = await harness.lease({ baseline: "checkout", requesterId: "agent-3" }); - expect(next.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); - }); -}); - -describe("Dispatcher: two copies of one baseline version are made at once", () => { - it("keeps the old version after a replace until the second copy is registered, though the first copy is gone", async () => { - const harness = await buildReplaceHarness("making"); - const flush = async (): Promise => { - for (let turn = 0; turn < 10; turn += 1) - await new Promise((resolve) => setImmediate(resolve)); - }; - await harness.saveFrom("operator", "one"); - const operator = await harness.lease({ model: replaceModel, requesterId: "operator" }); - harness.writeFile(operator.device.id, "two"); - // why: a warm pool pass makes copies without a request, so two can be in flight for one version. - const copy = () => - harness.engine.core.provisioner.provision( - { - baseline: { name: "checkout", version: 1 }, - model: replaceModel, - osVersion: "26.5", - physical: false, - platform: "ios", - }, - { reservation: { release: () => undefined } }, - ); - - const first = copy(); - await flush(); - harness.clock.advance(2_000); - const second = copy(); - await flush(); - await harness.saveCheckout(operator.lease.id, 100); - expect(harness.versions()).toEqual([1, 2]); - - harness.clock.advance(2_900); - const firstCopy = await first; - const lifecycle = harness.engine.core.deviceLifecycle; - firstCopy.claim.release(); - const stopped = await lifecycle.shutdown(firstCopy.device, "lease-engine", "cleanup"); - await lifecycle.destroy(stopped as NonNullable, "lease-engine", "cleanup"); - await flush(); - expect(harness.stateOf(firstCopy.device.id)).toBe("deleted"); - expect(harness.versions()).toEqual([1, 2]); - expect(harness.removed()).toEqual([]); - - harness.clock.advance(2_000); - const secondCopy = await second; - expect(harness.stateOf(secondCopy.device.id)).toBe("ready"); - expect(harness.versions()).toEqual([1, 2]); - }); -}); - -describe("Dispatcher: usage.get", () => { - afterEach(removeUsageRigs); - - const admin = session({ role: "admin" }); - - async function historyWithRows(options: { readonly gatewayLabel?: string } = {}) { - const rig = await usageRig(); - rig.emit("lease.requested", requested("R1")); - const gate = { open: true }; - const eventHistory = { - replay: (input?: { readonly sinceTs?: number }) => rig.history.replay(input), - replayPage: (input: { readonly beforeId?: string; readonly limit: number }) => - rig.history.replayPage(input), - status: () => ({ ...rig.history.status(), opened: gate.open && rig.history.status().opened }), - }; - return { eventHistory, gate, rig, ...options }; - } - - it("the worker handler answers HISTORY_NOT_KEPT with oldestTs and the message naming when the figures start when to is 1 ms before the oldest row, and figures when to is that row's time", async () => { - const { eventHistory, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - - await expect( - dispatcher.dispatch("usage.get", { from: T0 - 2 * HOUR, to: T0 - 1 }, admin), - ).rejects.toMatchObject({ - code: "HISTORY_NOT_KEPT", - details: { oldestTs: T0 }, - message: `The history does not reach back to the end of that window; its figures start at ${new Date(T0).toISOString()}.`, - }); - await expect( - dispatcher.dispatch("usage.get", { from: T0 - 2 * HOUR, to: T0 }, admin), - ).resolves.toMatchObject({ window: { to: T0 } }); - }); - - it("the worker handler answers HISTORY_UNAVAILABLE with the message naming daemon.log while history.status().opened is false, after answering figures while it was true", async () => { - const { eventHistory, gate, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const window = { from: T0, to: T0 + HOUR }; - - await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ - totals: { requests: 1 }, - }); - gate.open = false; - const before = threads.posts.length; - - await expect(dispatcher.dispatch("usage.get", window, admin)).rejects.toMatchObject({ - code: "HISTORY_UNAVAILABLE", - message: "The history could not be opened, so no figures can be read; see daemon.log.", - }); - expect(threads.posts).toHaveLength(before); - }); - - it("the worker handler passes includeLedger to the reader: it lists the window's requests when true, leaves requests out when absent or false, and refuses a non-boolean as BAD_REQUEST", async () => { - const { eventHistory, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const window = { from: T0, to: T0 + HOUR }; - - const listed = await dispatcher.dispatch( - "usage.get", - { ...window, includeLedger: true }, - admin, - ); - const absent = await dispatcher.dispatch("usage.get", window, admin); - const off = await dispatcher.dispatch("usage.get", { ...window, includeLedger: false }, admin); - - expect(listed.requests?.map((entry) => entry.requestId)).toEqual(["R1"]); - expect(absent).not.toHaveProperty("requests"); - expect(off).not.toHaveProperty("requests"); - await expect( - dispatcher.dispatch("usage.get", { ...window, includeLedger: "yes" } as never, admin), - ).rejects.toMatchObject({ code: "BAD_REQUEST" }); - }); - - it("the worker handler answers one worker entry, itself, with its instance id and gateway.label, or no label when none is set", async () => { - const { eventHistory, rig } = await historyWithRows(); - const labelled = await buildDispatcher({ - eventHistory, - gatewayLabel: "lab-a", - historyPath: rig.historyPath, - }); - const plain = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const window = { from: T0, to: T0 + HOUR }; - - const named = await labelled.dispatcher.dispatch("usage.get", window, admin); - const anonymous = await plain.dispatcher.dispatch("usage.get", window, admin); - - expect(named.workers).toMatchObject([{ id: "instance-1", label: "lab-a", requests: 1 }]); - expect(anonymous.workers).toHaveLength(1); - expect(anonymous.workers[0]).toMatchObject({ id: "instance-1", requests: 1 }); - expect(anonymous.workers[0]).not.toHaveProperty("label"); - }); - - it("the worker handler labels a requester from the token store, and leaves the label out when the daemon has no token store", async () => { - const { eventHistory, rig } = await historyWithRows(); - const withStore = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const withoutStore = await buildDispatcher({ - eventHistory, - historyPath: rig.historyPath, - withoutTokens: true, - }); - const { record } = await withStore.tokens.create("agent", "ci-bot"); - rig.emit("lease.requested", requested("R2", { requester: record.id })); - const window = { from: T0, to: T0 + HOUR }; - - const labelled = await withStore.dispatcher.dispatch("usage.get", window, admin); - const unlabelled = await withoutStore.dispatcher.dispatch("usage.get", window, admin); - - expect(labelled.requesters).toEqual([ - { granted: 0, heldTotalMs: 0, id: "alice", rejected: 0, requests: 1 }, - { granted: 0, heldTotalMs: 0, id: record.id, label: "ci-bot", rejected: 0, requests: 1 }, - ]); - expect(unlabelled.requesters.map((entry) => entry.id)).toEqual(["alice", record.id]); - expect(unlabelled.requesters.every((entry) => !("label" in entry))).toBe(true); - }); - - it("usage.get runs two calls for one window as one read on the read thread and gives both its answer, and two windows as two reads", async () => { - const { eventHistory, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const call = (from: number, to = from + HOUR) => - dispatcher.dispatch("usage.get", { from, to }, admin); - const readsOf = async (run: () => Promise) => { - const before = threads.posts.length; - await run(); - return threads.posts.length - before; - }; - await call(T0); - const single = await readsOf(() => call(T0)); - - const [first, second] = await Promise.all([call(T0), call(T0)]); - const sameWindow = await readsOf(() => Promise.all([call(T0), call(T0)])); - const twoWindows = await readsOf(() => Promise.all([call(T0), call(T0 - HOUR)])); - const otherStart = await readsOf(() => Promise.all([call(T0), call(T0 - HOUR, T0 + HOUR)])); - const otherEnd = await readsOf(() => Promise.all([call(T0), call(T0, T0 + 2 * HOUR)])); - - expect(single).toBeGreaterThan(0); - expect(sameWindow).toBe(single); - expect(twoWindows).toBe(2 * single); - expect(otherStart).toBe(2 * single); - expect(otherEnd).toBe(2 * single); - expect(first).toEqual(second); - expect(first.totals.requests).toBe(1); - }); - - it("dispose ends the read thread", async () => { - const { eventHistory, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - await dispatcher.dispatch("usage.get", { from: T0, to: T0 + HOUR }, admin); - const thread = threads.workers.at(-1) as unknown as import("node:worker_threads").Worker; - let exited = false; - thread.on("exit", () => { - exited = true; - }); - - dispatcher.dispose(); - - await vi.waitFor(() => expect(exited).toBe(true)); - }); - - it("usage.get answers INTERNAL to a call in flight when the read thread exits, and answers the next call from a new thread", async () => { - const { eventHistory, rig } = await historyWithRows(); - const { dispatcher } = await buildDispatcher({ eventHistory, historyPath: rig.historyPath }); - const window = { from: T0, to: T0 + HOUR }; - await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ - totals: { requests: 1 }, - }); - const started = threads.workers.length; - threads.crashOnPost = true; - - try { - await expect(dispatcher.dispatch("usage.get", window, admin)).rejects.toMatchObject({ - code: "INTERNAL", - }); - } finally { - threads.crashOnPost = false; - } - - await expect(dispatcher.dispatch("usage.get", window, admin)).resolves.toMatchObject({ - totals: { requests: 1 }, - }); - expect(threads.workers.length).toBe(started + 1); - }); -}); From 96209f9b4b7e4da6967f975f38e1c12fb83b8be8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:27:21 +0100 Subject: [PATCH 32/83] test(daemon): split server.test.ts by topic --- src/daemon/server-claim-and-stop.test.ts | 177 +++ src/daemon/server-drivers.test.ts | 351 +++++ src/daemon/server-leases.test.ts | 287 ++++ src/daemon/server-pushes.test.ts | 506 +++++++ src/daemon/server-requests.test.ts | 368 +++++ src/daemon/server.test.ts | 1640 +--------------------- 6 files changed, 1691 insertions(+), 1638 deletions(-) create mode 100644 src/daemon/server-claim-and-stop.test.ts create mode 100644 src/daemon/server-drivers.test.ts create mode 100644 src/daemon/server-leases.test.ts create mode 100644 src/daemon/server-pushes.test.ts create mode 100644 src/daemon/server-requests.test.ts diff --git a/src/daemon/server-claim-and-stop.test.ts b/src/daemon/server-claim-and-stop.test.ts new file mode 100644 index 000000000..951beb1ee --- /dev/null +++ b/src/daemon/server-claim-and-stop.test.ts @@ -0,0 +1,177 @@ +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CryptoTokenSecrets, + MemoryFilesystem, + NodeFilesystem, + NodeIpcTransport, +} from "../ports/index.js"; +import type { DaemonServer } from "./server.js"; +import { DaemonEndpointHost } from "./connection-host.js"; +import { + adminAuth, + createClient, + createHarness, + hello, + leaseIdOf, + runningDaemons, + stopTestDaemons, + temporaryDirectories, +} from "./test-server.js"; + +afterEach(stopTestDaemons); + +describe("DaemonServer", () => { + it("recovers a stale socket file and refuses a second live daemon before running any device work", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-stale-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + await new NodeFilesystem().writeFileAtomic(socketPath, "stale"); + const first = await createHarness({ socketPath }); + let convergeCalls = 0; + const second = await createHarness({ + converge: () => { + convergeCalls += 1; + return Promise.resolve(); + }, + socketPath, + start: false, + }); + + await expect(second.daemon.start()).rejects.toMatchObject({ + name: "DaemonAlreadyRunningError", + }); + expect(convergeCalls).toBe(0); + await second.daemon.stop("failed-start"); + const client = await createClient(socketPath); + await hello(client); + await client.close(); + const firstDaemon: DaemonServer = first.daemon; + expect(firstDaemon.socketPath).toBe(socketPath); + }); + + // why: review finding S1: `AdminSecretManager`'s own doc (`src/auth`) asserts "a daemon + // that loses the start race never calls `persist()` at all ... so the file an + // already-running daemon wrote is never touched by the loser" -- untested anywhere before + // this. Mirrors the "recovers a stale socket file" test above (two harnesses racing for the + // same socket path), but gives each its own real `AdminSecretManager` over its own + // `MemoryFilesystem` so this can assert on the filesystem directly, rather than trusting a + // stub. In production both instances would target the *same* real `admin.token` path (see + // `main.ts`); separate filesystems here isolate what each instance actually did, without that + // shared path letting a wrongly-invoked `remove()` on the loser silently clean up after a + // wrongly-invoked `persist()` and mask the very bug this test exists to catch. + it("never touches admin.token on the daemon instance that loses the start race", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-stale-admin-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + await new NodeFilesystem().writeFileAtomic(socketPath, "stale"); + + const secrets = new CryptoTokenSecrets(); + const winnerFilesystem = new MemoryFilesystem(); + await createHarness({ + adminSecret: (await adminAuth(winnerFilesystem, secrets)).adminSecret, + socketPath, + }); + await expect(winnerFilesystem.readFile("/admin.token")).resolves.toContain("\n"); + + const loserFilesystem = new MemoryFilesystem(); + const second = await createHarness({ + adminSecret: (await adminAuth(loserFilesystem, secrets)).adminSecret, + socketPath, + start: false, + }); + await expect(second.daemon.start()).rejects.toMatchObject({ + name: "DaemonAlreadyRunningError", + }); + await expect(loserFilesystem.readFile("/admin.token")).rejects.toThrow(); + + await second.daemon.stop("failed-start"); + const client = await createClient(socketPath); + await hello(client); + await client.close(); + }); + + it.each([false, true])( + "preserves the replacement daemon's admin credential and still releases the endpoint when it starts as the old endpoint is released (credential removal fails: %s)", + async (removalFails) => { + const directory = await mkdtemp(join(tmpdir(), "simlock-admin-restart-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + const filesystem = new MemoryFilesystem(); + const secrets = new CryptoTokenSecrets(); + const oldSecret = await adminAuth(filesystem, secrets); + const replacementSecret = await adminAuth(filesystem, secrets); + const replacement = await createHarness({ + adminSecret: replacementSecret.adminSecret, + socketPath, + start: false, + }); + const host = new DaemonEndpointHost({ + connector: new NodeIpcTransport(), + endpoint: socketPath, + filesystem: new NodeFilesystem(), + listenerFactory: new NodeIpcTransport(), + }); + let replacementToken: string | undefined; + const old = await createHarness({ + adminSecret: oldSecret.adminSecret, + host: { + endpoint: socketPath, + start: (accept) => host.start(accept), + async stop() { + await host.stop(); + // why: endpoint release lets a new process start before the old stop resumes. + await replacement.daemon.start(); + replacementToken = await filesystem.readFile("/admin.token"); + await expect( + replacementSecret.resolveHelloRole({ credential: replacementToken.trim() }), + ).resolves.toBe("admin"); + }, + }, + socketPath, + }); + await expect( + oldSecret.resolveHelloRole({ + credential: (await filesystem.readFile("/admin.token")).trim(), + }), + ).resolves.toBe("admin"); + + if (removalFails) { + vi.spyOn(filesystem, "rm").mockRejectedValueOnce(new Error("EACCES")); + await expect(old.daemon.stop("restart")).rejects.toThrow("EACCES"); + // why: the failed stop is settled; teardown must not stop it again. + runningDaemons.splice(runningDaemons.indexOf(old.daemon), 1); + } else { + await old.daemon.stop("restart"); + } + + // why: the replacement could only start because the old stop released the endpoint. + expect(replacementToken).toBeDefined(); + await expect(filesystem.readFile("/admin.token")).resolves.toBe(replacementToken); + }, + ); + + it("gracefully stops without touching leases, persisting them with their deadlines", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + const grant = await client.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + + await expect(client.request("daemon.stop", {})).resolves.toMatchObject({ ok: true }); + + expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); + expect(harness.state.database.prepare("SELECT id FROM leases").all()).toEqual([ + { id: leaseId }, + ]); + expect(harness.eventBus.replay().map((event) => event.event)).toContain("daemon.stopping"); + expect(harness.eventBus.replay().map((event) => event.event)).not.toContain("lease.released"); + }); +}); diff --git a/src/daemon/server-drivers.test.ts b/src/daemon/server-drivers.test.ts new file mode 100644 index 000000000..73ce5cfd1 --- /dev/null +++ b/src/daemon/server-drivers.test.ts @@ -0,0 +1,351 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { type DriverRejection, RuntimeMissingError } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import { + createClient, + createHarness, + hello, + rejectionNaming, + stopTestDaemons, +} from "./test-server.js"; + +afterEach(stopTestDaemons); + +describe("DaemonServer driver rejections", () => { + it("publishes one event per platform that refused to start, after the daemon is up", async () => { + const harness = await createHarness({ + driverRejections: [ + { + event: "driver.root-rejected", + payload: { platform: "ios", reason: "wrong-instance", root: "/Devices" }, + platform: "ios", + reason: "wrong-instance", + summary: "Refusing the ios device root /Devices: it belongs to another instance", + }, + { + event: "driver.adb-server-rejected", + payload: { port: 5038, reason: "occupied" }, + platform: "android", + reason: "occupied", + summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", + }, + ], + }); + + const events = harness.eventBus.replay(); + const rejected = events.filter( + (event) => + event.event === "driver.root-rejected" || event.event === "driver.adb-server-rejected", + ); + expect(rejected).toEqual([ + expect.objectContaining({ + event: "driver.root-rejected", + module: "daemon", + payload: { platform: "ios", reason: "wrong-instance", root: "/Devices" }, + }), + expect.objectContaining({ + event: "driver.adb-server-rejected", + module: "daemon", + payload: { port: 5038, reason: "occupied" }, + }), + ]); + const started = events.find((event) => event.event === "daemon.started"); + for (const event of rejected) expect(started?.seq).toBeLessThan(event.seq); + }); + + it("names the refusal when a lease asks for a platform whose driver did not start", async () => { + const harness = await createHarness({ + driverRejections: [ + { + event: "driver.adb-server-rejected", + payload: { port: 5038, reason: "occupied" }, + platform: "android", + reason: "occupied", + summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", + }, + ], + }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("lease.request", { + requesterId: "agent-1", + model: "Pixel 8", + platform: "android", + }); + + expect(response.error?.code).toBe("NO_DRIVER"); + expect(response.error?.message).toContain("port 5038 is held by an adb server we do not own"); + }); + + it("leaves an ordinary missing platform unexplained, having nothing to explain", async () => { + const harness = await createHarness({ + driverRejections: [ + { + event: "driver.root-rejected", + payload: { platform: "ios", reason: "symlink", root: "/Devices" }, + platform: "ios", + reason: "symlink", + summary: "Refusing the ios device root /Devices: it is a symlink", + }, + ], + }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("lease.request", { + requesterId: "agent-1", + model: "Pixel 8", + platform: "android", + }); + + expect(response.error?.code).toBe("NO_DRIVER"); + expect(response.error?.message).toBe("No driver registered for platform: android"); + }); + + it.each([ + { + name: "adb", + rejection: { + event: "driver.adb-server-rejected", + payload: { port: 5038, reason: "occupied" }, + platform: "android", + reason: "occupied", + summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", + }, + tools: ["adb"], + request: { args: ["devices"], tool: "adb" }, + text: "port 5038 is held by an adb server we do not own", + }, + { + name: "devicectl", + rejection: { + event: "driver.root-rejected", + payload: { platform: "ios", reason: "symlink", root: "/Devices" }, + platform: "ios", + reason: "symlink", + summary: "Refusing the ios device root /Devices: it is a symlink", + }, + tools: ["simctl", "devicectl"], + request: { args: ["list", "devices"], tool: "devicectl" }, + text: "Refusing the ios device root /Devices: it is a symlink", + }, + ] as const)( + "names the refusal when a $name passthrough asks for the tool whose driver did not start", + async ({ rejection, request, text, tools }) => { + const harness = await createHarness({ + driverRejections: [rejectionNaming(rejection, tools)], + }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("driver.passthrough", { + args: [...request.args], + tool: request.tool, + }); + + expect(response.error?.code).toBe("UNKNOWN_PASSTHROUGH_TOOL"); + expect(response.error?.message).toContain(text); + }, + ); + + it("leaves an unknown passthrough tool unexplained when no driver claimed it", async () => { + const harness = await createHarness({ + driverRejections: [ + rejectionNaming( + { + event: "driver.root-rejected", + payload: { platform: "ios", reason: "symlink", root: "/Devices" }, + platform: "ios", + reason: "symlink", + summary: "Refusing the ios device root /Devices: it is a symlink", + }, + ["simctl", "devicectl"], + ), + ], + }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("driver.passthrough", { + args: ["devices"], + tool: "adb", + }); + + expect(response.error?.message).toBe("No driver provides a adb passthrough"); + }); + + it("refuses at compile time to pair an event with another event's payload", () => { + const rejection: DriverRejection = { + event: "driver.adb-server-rejected", + // @ts-expect-error -- `port` is a number on the wire (docs/internal/EVENTS.md). + payload: { port: "5038", reason: "occupied" }, + platform: "android", + reason: "occupied", + summary: "Refusing the Android driver: port 5038 is occupied", + }; + + expect(rejection.event).toBe("driver.adb-server-rejected"); + }); + + it("publishes nothing when every driver started", async () => { + const harness = await createHarness(); + + expect(harness.eventBus.replay().filter((event) => event.event.startsWith("driver."))).toEqual( + [], + ); + }); +}); + +describe("DaemonServer download policy", () => { + it("grants download permission under the always policy without a per-request flag", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); + const harness = await createHarness({ clock, downloads: { policy: "always" }, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + const grant = await client.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(grant.ok).toBe(true); + expect( + driver.calls + .filter((call) => call.operation === "installComponent") + .map((call) => call.arguments), + ).toEqual([["26.5"]]); + await client.close(); + }); + + it("still answers a lease request whose download names a component past the progress push's bound, sending no downloading push for it", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + // why: the driver, not the request, names the component: a request's exact osVersion is held + // within the push's bound. Every resolve before the install answers as a missing runtime does. + for (const callNumber of [1, 2]) { + driver.failOn( + "resolveSpec", + callNumber, + new RuntimeMissingError("ios", "26.5", { component: "9".repeat(65) }), + ); + } + const harness = await createHarness({ clock, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + const grant = await client.request("lease.request", { + allowDownload: true, + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(grant.ok, JSON.stringify(grant.error)).toBe(true); + const stages = client + .frames() + .filter((frame) => frame.push === "progress") + .map((frame) => (frame.payload as { progress: { stage: string } }).progress.stage); + expect(stages).toContain("provisioning"); + expect(stages).not.toContain("downloading"); + await client.close(); + }); + + it.each([ + ["asks for it", { allowDownload: true }], + ["never asked for a download", {}], + ])( + "attaches the download-policy suffix and withholds download permission under the never policy even when the request %s", + async (_name, extra) => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); + const harness = await createHarness({ clock, downloads: { policy: "never" }, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + // why: when the request never asked, the driver's own message still suggests `--allow-download`, + // which under the never policy can never help, so the suffix must still attach as the + // correction. + const response = await client.request("lease.request", { + ...extra, + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(response.ok).toBe(false); + expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); + expect(response.error?.message).toContain("downloads.policy"); + expect(driver.calls.map((call) => call.operation)).not.toContain("installComponent"); + await client.close(); + }, + ); + + it("defers to the request's own flag under the default on-request policy", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); + const harness = await createHarness({ clock, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(response.ok).toBe(false); + expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); + expect(response.error?.message).not.toContain("downloads.policy"); + expect(driver.calls.map((call) => call.operation)).not.toContain("installComponent"); + + const flagged = await client.request("lease.request", { + allowDownload: true, + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(flagged.ok).toBe(true); + expect( + driver.calls + .filter((call) => call.operation === "installComponent") + .map((call) => call.arguments), + ).toEqual([["26.5"]]); + await client.close(); + }); + + it("never attaches the download-policy suffix to an undownloadable RuntimeMissingError, even under the never policy", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); + driver.failOn( + "resolveSpec", + 1, + new RuntimeMissingError("ios", "12.0", { downloadable: false }), + ); + const harness = await createHarness({ clock, downloads: { policy: "never" }, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("lease.request", { + allowDownload: true, + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "12.0", + platform: "ios", + }); + + expect(response.ok).toBe(false); + expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); + expect(response.error?.message).not.toContain("downloads.policy"); + await client.close(); + }); +}); diff --git a/src/daemon/server-leases.test.ts b/src/daemon/server-leases.test.ts new file mode 100644 index 000000000..20eb11e95 --- /dev/null +++ b/src/daemon/server-leases.test.ts @@ -0,0 +1,287 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { DAEMON_PROTOCOL_VERSION } from "../daemon-protocol/index.js"; +import { createClient, createHarness, hello, leaseIdOf, stopTestDaemons } from "./test-server.js"; + +afterEach(stopTestDaemons); + +describe("DaemonServer lease liveness (ADR 0004)", () => { + it("keeps a lease alive only while something renews it, and expires it when nothing does", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 40 } }); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + expect((grant.payload as { lease: { ttlDeadline: number } }).lease.ttlDeadline).toBe(1_040); + + let expectedDeadline = 1_040; + for (let cycle = 0; cycle < 6; cycle += 1) { + harness.clock.advance(10); + expectedDeadline = harness.clock.now() + 40; + await expect(holder.request("lease.renew", { leaseId })).resolves.toMatchObject({ + ok: true, + payload: { id: leaseId, ttlDeadline: expectedDeadline, ttlMs: 40 }, + }); + } + expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); + + harness.clock.advance(40); + await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); + await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ + payload: { leaseId, reason: "expired" }, + push: "lease-lost", + }); + await holder.close(); + }); + + it("no longer answers lease.heartbeat at all", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect(client.request("lease.heartbeat", { nonce: 1 })).resolves.toMatchObject({ + error: { code: "UNKNOWN_REQUEST" }, + ok: false, + }); + await client.close(); + }); + + it("answers worker.list with this daemon's instance id and the version its hello reports", async () => { + const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); + const client = await createClient(harness.socketPath); + const greeting = await client.request("hello", { + clientVersion: "test", + protocolVersion: DAEMON_PROTOCOL_VERSION, + }); + expect(greeting).toMatchObject({ ok: true, payload: { version: "test" } }); + + await expect(client.request("worker.list", {})).resolves.toMatchObject({ + ok: true, + payload: { workers: [{ id: "instance-test", version: "test" }] }, + }); + await client.close(); + }); + + it("re-reads worker.list's kept catalog once this daemon's bus reports a component installed", async () => { + const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); + const client = await createClient(harness.socketPath); + await hello(client); + const catalogReads = vi.spyOn(harness.driver, "listCatalog"); + + await client.request("worker.list", {}); + await client.request("worker.list", {}); + expect(catalogReads).toHaveBeenCalledTimes(1); + + harness.eventBus.emit( + "component.installed", + { + alreadyPresent: false, + componentId: "27.0", + durationMs: 1, + platform: "ios", + version: "27.0", + }, + "test", + ); + await client.request("worker.list", {}); + + expect(catalogReads).toHaveBeenCalledTimes(2); + await client.close(); + }); + + it("disposes the worker's dispatcher, ending its bus subscriptions, when the daemon stops", async () => { + const live = new Map(); + const subscribe = EventBus.prototype.subscribe; + const spy = vi.spyOn(EventBus.prototype, "subscribe").mockImplementation(function ( + this: EventBus, + event, + handler, + ) { + const unsubscribe = subscribe.call(this, event, handler); + live.set(event, (live.get(event) ?? 0) + 1); + let ended = false; + return () => { + if (!ended) live.set(event, (live.get(event) ?? 0) - 1); + ended = true; + unsubscribe(); + }; + }); + try { + const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); + expect(live.get("component.installed")).toBeGreaterThan(0); + + await harness.daemon.stop("test"); + + expect(live.get("component.installed")).toBe(0); + } finally { + spy.mockRestore(); + } + }); + + it("answers worker.install-component and worker.update with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on or update", async () => { + const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect( + client.request("worker.install-component", { + platform: "android", + version: "35", + workers: "all", + }), + ).resolves.toMatchObject({ + error: { + code: "UNSUPPORTED_IN_WORKER_MODE", + details: { operation: "worker.install-component" }, + }, + ok: false, + }); + await expect(client.request("worker.update", { workerId: "wrk_1" })).resolves.toMatchObject({ + error: { code: "UNSUPPORTED_IN_WORKER_MODE", details: { operation: "worker.update" } }, + ok: false, + }); + await client.close(); + }); + + it("keeps a lease across a daemon restart and restores its timer from the persisted deadline", async () => { + const leaseOverrides = { defaultTtlMs: 40 }; + const first = await createHarness({ lease: leaseOverrides }); + const holder = await createClient(first.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + const deviceId = (grant.payload as { device: { id: string } }).device.id; + + first.clock.advance(10); + await expect(holder.request("lease.renew", { leaseId })).resolves.toMatchObject({ + ok: true, + payload: { ttlDeadline: 1_050 }, + }); + + const second = await createHarness({ + clock: new FakeClock(1_010), + driver: first.driver, + lease: leaseOverrides, + state: first.state, + }); + await second.engine.convergeRunningCapacity(); + + expect(second.registry.snapshot.leases).toMatchObject([{ id: leaseId, ttlDeadline: 1_050 }]); + expect(second.registry.snapshot.devices).toMatchObject([{ id: deviceId, state: "leased" }]); + + second.clock.advance(30); + await expect.poll(() => second.registry.snapshot.leases).toHaveLength(1); + second.clock.advance(10); + await expect.poll(() => second.registry.snapshot.leases).toEqual([]); + await expect.poll(() => second.registry.snapshot.devices[0]?.state).toBe("deleted"); + await holder.close(); + }); + + it("expires a lease whose deadline passed while no daemon was running, as soon as one is", async () => { + const leaseOverrides = { defaultTtlMs: 40 }; + const first = await createHarness({ lease: leaseOverrides }); + const holder = await createClient(first.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const deviceId = (grant.payload as { device: { id: string } }).device.id; + await holder.close(); + + const second = await createHarness({ + clock: new FakeClock(2_000), + driver: first.driver, + lease: leaseOverrides, + state: first.state, + }); + await second.engine.convergeRunningCapacity(); + + await expect.poll(() => second.registry.snapshot.leases).toEqual([]); + await expect + .poll(() => second.registry.snapshot.devices) + .toMatchObject([{ id: deviceId, state: "deleted" }]); + }); + + it("surfaces the stored lastRenewedAt and ttlMs in status and list --leases", async () => { + const harness = await createHarness({ lease: { defaultTtlMs: 40 } }); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + expect((grant.payload as { lease: { lastRenewedAt: number } }).lease.lastRenewedAt).toBe(1_000); + + harness.clock.advance(10); + await holder.request("lease.renew", { leaseId }); + + await expect(holder.request("status.get", {})).resolves.toMatchObject({ + ok: true, + payload: { leases: [{ id: leaseId, lastRenewedAt: 1_010, ttlMs: 40 }] }, + }); + await expect(holder.request("list.get", { kind: "leases" })).resolves.toMatchObject({ + ok: true, + payload: [{ id: leaseId, lastRenewedAt: 1_010, ttlMs: 40 }], + }); + await holder.close(); + }); +}); + +describe("DaemonServer lease request mode", () => { + it("grants a slim request on a runtime the driver can slim with mode slim", async () => { + const harness = await createHarness({ slimmableOsVersions: ["26.5"] }); + const client = await createClient(harness.socketPath); + await hello(client); + + const grant = await client.request("lease.request", { + requesterId: "agent-1", + mode: "slim", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + expect(grant.ok).toBe(true); + expect((grant.payload as { device: { mode: string } }).device.mode).toBe("slim"); + await client.close(); + }); + + it.each([ + ["full", { full: true }], + ["a mode other than slim or full", { mode: "fast" }], + ["the lease mode ADR 0004 retired", { mode: "held" }], + ])("answers a lease.request carrying %s with BAD_REQUEST", async (_label, extra) => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + ...extra, + }); + + expect(response.ok).toBe(false); + expect(response.error).toMatchObject({ code: "BAD_REQUEST" }); + await client.close(); + }); +}); diff --git a/src/daemon/server-pushes.test.ts b/src/daemon/server-pushes.test.ts new file mode 100644 index 000000000..f810981de --- /dev/null +++ b/src/daemon/server-pushes.test.ts @@ -0,0 +1,506 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { FakeClock, ScriptedProcessRunner } from "../ports/index.js"; +import { DAEMON_PROTOCOL_VERSION } from "../daemon-protocol/index.js"; +import { + BlockingConnection, + ControllableStreamingRunner, + SingleConnectionHost, + createClient, + createHarness, + flush, + hello, + leaseIdOf, + passthroughDriver, + passthroughFake, + stopTestDaemons, + waitFor, +} from "./test-server.js"; + +afterEach(stopTestDaemons); + +describe("DaemonServer", () => { + it("pushes device.exec output frames keyed by the request id, then replies with the exit code", async () => { + const clock = new FakeClock(1_000); + const driver = passthroughFake({ + availableOsVersions: ["26.5"], + clock, + passthrough: (args: readonly string[]) => ({ + args: ["--set", "/root", ...args], + command: "simctl", + env: {}, + }), + passthroughTools: ["simctl"], + platform: "ios", + }); + const harness = await createHarness({ + clock, + driver, + processRunner: new ScriptedProcessRunner([ + { + chunks: [ + { chunk: "booted", stream: "stdout" }, + { chunk: "a warning", stream: "stderr" }, + ], + match: { args: ["--set", "/root", "list", "devices"], command: "simctl" }, + result: { code: 6, stderr: "", stdout: "" }, + }, + ]), + }); + const client = await createClient(harness.socketPath); + await hello(client); + const grant = await client.request("lease.request", { + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + }); + + const response = await client.request( + "device.exec", + { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, + "exec-frame", + ); + + expect(response).toMatchObject({ ok: true, payload: { exitCode: 6 } }); + expect(client.frames().filter((frame) => frame.push === "output")).toEqual([ + { payload: { chunk: "booted", requestId: "exec-frame", stream: "stdout" }, push: "output" }, + { + payload: { chunk: "a warning", requestId: "exec-frame", stream: "stderr" }, + push: "output", + }, + ]); + + // why: ADR 0005 §19a: `started` reaches the socket exactly once, keyed by the same frame id. + // Round 5 review: this send site could be deleted with the entire suite green, because + // nothing asserted the frame ever reached a socket at all. + // + // Deliberately not asserted here: that it precedes the first `output`. It does in + // production -- the dispatcher calls `onStarted` synchronously after the spawn returns, and + // a real child's chunks arrive later from `data` events -- but `ScriptedProcessRunner` + // delivers its scripted chunks synchronously *inside* `runCommand`, so this harness sees + // them first and cannot distinguish the two orderings. Claiming the ordering here would be a + // title outrunning its body; `client.test.ts`'s wire test pins it where it is observable. + expect(client.frames().filter((frame) => frame.push === "started")).toEqual([ + { payload: { requestId: "exec-frame" }, push: "started" }, + ]); + }); + + it("pushes component.install progress frames keyed by the request id, then replies with the outcome", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + installProgress: [50], + platform: "ios", + }); + const harness = await createHarness({ clock, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + const response = await client.request( + "component.install", + { platform: "ios", version: "27.0" }, + "install-frame", + ); + + expect(response).toMatchObject({ + ok: true, + payload: { component: "27.0", outcome: "installed", platform: "ios", version: "27.0" }, + }); + expect(client.frames().filter((frame) => frame.push === "component-progress")).toEqual([ + { + payload: { progress: { stage: "downloading" }, requestId: "install-frame" }, + push: "component-progress", + }, + { + payload: { progress: { fraction: 0.5, stage: "downloading" }, requestId: "install-frame" }, + push: "component-progress", + }, + { + payload: { progress: { fraction: 1, stage: "downloading" }, requestId: "install-frame" }, + push: "component-progress", + }, + ]); + }); + + it("routes two commands on one connection by frame id, and pushes to no other connection", async () => { + const clock = new FakeClock(1_000); + const driver = passthroughDriver(clock); + const harness = await createHarness({ + clock, + driver, + processRunner: new ScriptedProcessRunner([ + { + chunks: [{ chunk: "first-command", stream: "stdout" }], + match: { args: ["--set", "/root", "list", "devices"], command: "simctl" }, + }, + { + chunks: [{ chunk: "second-command", stream: "stdout" }], + match: { args: ["--set", "/root", "list", "runtimes"], command: "simctl" }, + }, + ]), + }); + const client = await createClient(harness.socketPath); + const observer = await createClient(harness.socketPath); + await hello(client); + await hello(observer); + const grant = await client.request("lease.request", { + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + + const first = client.request( + "device.exec", + { args: ["list", "devices"], leaseId, tool: "simctl" }, + "exec-a", + ); + const second = client.request( + "device.exec", + { args: ["list", "runtimes"], leaseId, tool: "simctl" }, + "exec-b", + ); + await Promise.all([first, second]); + + expect( + client + .frames() + .filter((frame) => frame.push === "output") + .map((frame) => frame.payload), + ).toEqual([ + { chunk: "first-command", requestId: "exec-a", stream: "stdout" }, + { chunk: "second-command", requestId: "exec-b", stream: "stdout" }, + ]); + expect(observer.frames().filter((frame) => frame.push === "output")).toEqual([]); + }); + + it("stops pushing when the connection dies, and lets the command finish anyway", async () => { + const clock = new FakeClock(1_000); + const driver = passthroughDriver(clock); + const runner = new ControllableStreamingRunner(); + const harness = await createHarness({ clock, driver, processRunner: runner }); + const client = await createClient(harness.socketPath); + await hello(client); + const grant = await client.request("lease.request", { + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + }); + + void client.request( + "device.exec", + { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, + "exec-dropped", + ); + await waitFor(() => runner.handle !== undefined); + const firstPush = client.nextFrame((frame) => frame.push === "output"); + runner.handle?.emit("stdout", "before-the-drop"); + expect(await firstPush).toMatchObject({ + payload: { chunk: "before-the-drop", requestId: "exec-dropped" }, + push: "output", + }); + + await client.close(); + await flush(); + runner.handle?.emit("stdout", "after-the-drop"); + runner.handle?.finish(0); + await flush(); + expect(runner.handle?.settled).toBe(true); + + const survivor = await createClient(harness.socketPath); + await hello(survivor); + await expect(survivor.request("status.get", {})).resolves.toMatchObject({ ok: true }); + await survivor.close(); + }); + + it("keeps ConnectionPushes.output's promise unresolved while a frame is still going out, resolving it once the write drains", async () => { + // why: ADR 0005 §19e on this transport: `ConnectionPushes.output`'s returned promise is + // what `device.exec` hands the process runner as a chunk's delivery, and it is that promise + // -- not anything this test can observe about a real stream -- staying unresolved while a + // client's socket has not drained that would eventually stop the command at its pipe. + // `ControllableStreamingRunner` is a fake with no stream of its own to stop reading (its + // `emit` calls `onChunk` directly), so this proves only the promise half of the contract; + // `ports/process-runner.test.ts`'s `NodeProcessRunner` suite is where a real pipe is + // actually paused. Driven through a connection whose `write` this test settles by hand -- + // a real socket would need megabytes to fill a kernel buffer, which is the same property + // tested by luck. + const clock = new FakeClock(1_000); + const driver = passthroughDriver(clock); + const runner = new ControllableStreamingRunner(); + const connection = new BlockingConnection(); + const harness = await createHarness({ + clock, + driver, + host: new SingleConnectionHost(connection), + processRunner: runner, + }); + expect(harness.daemon.health).toBeDefined(); + + await connection.request( + { clientVersion: "test", protocolVersion: DAEMON_PROTOCOL_VERSION }, + "hello", + ); + const grant = await connection.request( + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + "lease.request", + ); + connection.releaseAll(); + + void connection.request( + { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, + "device.exec", + "exec-slow", + ); + await waitFor(() => runner.handle !== undefined); + + connection.block(); + let firstDelivered = false; + void Promise.resolve(runner.handle?.emit("stdout", "first")).then(() => { + firstDelivered = true; + }); + await flush(); + expect(firstDelivered).toBe(false); + + connection.releaseAll(); + await flush(); + expect(firstDelivered).toBe(true); + + runner.handle?.finish(0); + await flush(); + }); + + it("pushes a lease-lost notification to the holding connection when its lease's TTL backstop expires", async () => { + const harness = await createHarness(); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + const deviceId = harness.registry.snapshot.leases.find( + (lease) => lease.id === leaseId, + )?.deviceId; + + harness.clock.advance(60_000); + await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ + payload: { deviceId, leaseId, reason: "expired" }, + push: "lease-lost", + }); + await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); + + await holder.close(); + const observer = await createClient(harness.socketPath); + await hello(observer); + await expect(observer.request("status.get", {})).resolves.toMatchObject({ ok: true }); + await observer.close(); + }); + + it("pushes a lease-lost notification to the actual holding connection when another connection force-releases its lease", async () => { + const harness = await createHarness(); + const holder = await createClient(harness.socketPath); + const releaser = await createClient(harness.socketPath); + await Promise.all([hello(holder), hello(releaser)]); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + + await expect(releaser.request("lease.release", { leaseId })).resolves.toMatchObject({ + ok: true, + }); + await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ + payload: { leaseId, reason: "explicit" }, + push: "lease-lost", + }); + + await expect(holder.request("lease.release", { leaseId })).resolves.toMatchObject({ + error: { code: "UNKNOWN_LEASE" }, + ok: false, + }); + await holder.close(); + await releaser.close(); + }); + + it("drops a lease-lost push to a connection whose peer has hung up instead of leaving its failed write unhandled", async () => { + const connection = new BlockingConnection(); + const harness = await createHarness({ host: new SingleConnectionHost(connection) }); + await connection.request( + { clientVersion: "test", protocolVersion: DAEMON_PROTOCOL_VERSION }, + "hello", + ); + const grant = await connection.request( + { model: "iPhone 16", osVersion: "26.5", platform: "ios" }, + "lease.request", + ); + const unhandled: unknown[] = []; + const recordUnhandled = (reason: unknown) => unhandled.push(reason); + process.on("unhandledRejection", recordUnhandled); + try { + connection.hangUp(); + await harness.engine.release(leaseIdOf(grant), "explicit"); + await new Promise((resolve) => setImmediate(resolve)); + + expect(connection.frames.filter((frame) => frame.push === "lease-lost")).toHaveLength(1); + expect(unhandled).toEqual([]); + } finally { + process.off("unhandledRejection", recordUnhandled); + } + }); + + type EventBus = Awaited>["eventBus"]; + + it.each([ + [ + "lease-lost", + (eventBus: EventBus) => { + eventBus.emit( + "lease.expired", + { deviceId: "device-x", leaseId: "lease-x", ownerId: "test-process" }, + "test", + ); + eventBus.emit( + "lease.released", + { deviceId: "device-y", leaseId: "lease-y", ownerId: "test-process", reason: "killed" }, + "test", + ); + }, + ], + [ + "device-unhealthy/device-recovered", + (eventBus: EventBus) => { + eventBus.emit( + "device.crash-detected", + { deviceId: "device-x", leaseId: "lease-x", observed: "stopped", platform: "ios" }, + "test", + ); + eventBus.emit( + "device.recovered", + { + attempts: 1, + deviceId: "device-x", + duration: 1_000, + leaseId: "lease-x", + platform: "ios", + }, + "test", + ); + }, + ], + ])( + "ignores %s facts for leases with no currently connected holder without breaking the daemon", + async (_name, emitFacts) => { + const harness = await createHarness(); + emitFacts(harness.eventBus); + + const client = await createClient(harness.socketPath); + await hello(client); + await expect(client.request("status.get", {})).resolves.toMatchObject({ ok: true }); + await client.close(); + }, + ); + + it("pushes a device-unhealthy notification on device.crash-detected without releasing the lease", async () => { + const harness = await createHarness(); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + const deviceId = harness.registry.snapshot.leases.find( + (lease) => lease.id === leaseId, + )?.deviceId; + + harness.eventBus.emit( + "device.crash-detected", + { deviceId: deviceId as string, leaseId, observed: "stopped", platform: "ios" }, + "test", + ); + await expect( + holder.nextFrame((frame) => frame.push === "device-unhealthy"), + ).resolves.toMatchObject({ + payload: { deviceId, leaseId, reason: "crashed" }, + push: "device-unhealthy", + }); + + expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); + await holder.request("lease.release", { leaseId }); + await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); + await holder.close(); + }); + + it("pushes a device-recovered notification on device.recovered without releasing the lease", async () => { + const harness = await createHarness(); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + const deviceId = harness.registry.snapshot.leases.find( + (lease) => lease.id === leaseId, + )?.deviceId; + + harness.eventBus.emit( + "device.recovered", + { + attempts: 2, + deviceId: deviceId as string, + duration: 30_000, + leaseId, + platform: "ios", + }, + "test", + ); + await expect( + holder.nextFrame((frame) => frame.push === "device-recovered"), + ).resolves.toMatchObject({ + payload: { attempts: 2, deviceId, leaseId }, + push: "device-recovered", + }); + + expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); + await holder.request("lease.release", { leaseId }); + await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); + await holder.close(); + }); + + it("pushes a lease-lost notification carrying reason device-lost when a leased device could not be recovered", async () => { + const harness = await createHarness(); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = leaseIdOf(grant); + const deviceId = harness.registry.snapshot.leases.find( + (lease) => lease.id === leaseId, + )?.deviceId; + + harness.eventBus.emit( + "lease.released", + { deviceId: deviceId as string, leaseId, ownerId: "test-process", reason: "device-lost" }, + "test", + ); + await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ + payload: { deviceId, leaseId, reason: "device-lost" }, + push: "lease-lost", + }); + }); +}); diff --git a/src/daemon/server-requests.test.ts b/src/daemon/server-requests.test.ts new file mode 100644 index 000000000..f0007456e --- /dev/null +++ b/src/daemon/server-requests.test.ts @@ -0,0 +1,368 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { PassthroughRefusedError } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import { + createClient, + createHarness, + hello, + leaseIdOf, + passthroughFake, + stopTestDaemons, + testServerConfig, +} from "./test-server.js"; + +afterEach(stopTestDaemons); + +describe("DaemonServer", () => { + it("answers status.get with the capacity, answers list.get, config.get, cleanup.run and events.replay as ok, pushes subscribed events, and renews and releases a lease", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect(client.request("status.get", {})).resolves.toMatchObject({ + ok: true, + payload: { + capacity: { + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: { + limit: 1, + maxRunning: 1, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }, + }, + }, + }); + await expect(client.request("list.get", {})).resolves.toMatchObject({ ok: true }); + await expect(client.request("config.get", {})).resolves.toMatchObject({ ok: true }); + await expect(client.request("cleanup.run", { dryRun: true })).resolves.toMatchObject({ + ok: true, + }); + await expect(client.request("events.replay", { sinceTs: 0 })).resolves.toMatchObject({ + ok: true, + }); + await expect(client.request("events.subscribe", {})).resolves.toMatchObject({ ok: true }); + harness.eventBus.emit( + "disk.pressure-detected", + { freeBytes: 1, threshold: testServerConfig().diskPressure.freeBytesThreshold }, + "test", + ); + await expect(client.nextFrame((frame) => frame.push === "event")).resolves.toMatchObject({ + push: "event", + }); + await expect(client.request("events.unsubscribe", {})).resolves.toMatchObject({ ok: true }); + + const grant = await client.request("lease.request", { + requesterId: "agent-1", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + const leaseId = (grant.payload as { readonly lease: { readonly id: string } }).lease.id; + await expect(client.request("lease.renew", { leaseId, ttlMs: 120_000 })).resolves.toMatchObject( + { + ok: true, + }, + ); + await expect(client.request("lease.release", { leaseId })).resolves.toMatchObject({ ok: true }); + expect(harness.registry.snapshot.leases).toEqual([]); + }); + + it("routes usage.get to the dispatcher with the window the frame carries", async () => { + const harness = await createHarness({ withHistory: true }); + const client = await createClient(harness.socketPath); + await hello(client); + + const answer = await client.request("usage.get", { from: 0, to: 3_600_000 }); + + expect(answer).toMatchObject({ + ok: true, + payload: { totals: { requests: 0 }, window: { from: 0, to: 3_600_000 } }, + }); + }); + + it("serves the device catalog, omitting platforms with no registered driver", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect(client.request("catalog.get", {})).resolves.toMatchObject({ + ok: true, + payload: { + platforms: [{ defaultRuntime: "26.5", models: [], platform: "ios", runtimes: ["26.5"] }], + }, + }); + await expect(client.request("catalog.get", { platform: "ios" })).resolves.toMatchObject({ + ok: true, + payload: { platforms: [{ platform: "ios" }] }, + }); + await expect(client.request("catalog.get", { platform: "android" })).resolves.toMatchObject({ + ok: true, + payload: { platforms: [] }, + }); + await expect(client.request("catalog.get", { platform: "foo" })).resolves.toMatchObject({ + error: { code: "BAD_REQUEST" }, + ok: false, + }); + }); + + it("resolves a passthrough to its driver's command without running it here", async () => { + const clock = new FakeClock(1_000); + const driver = passthroughFake({ + availableOsVersions: ["26.5"], + clock, + passthrough: (args) => ({ + args: ["simctl", "--set", "/root", ...args], + command: "xcrun", + env: {}, + }), + passthroughTools: ["simctl"], + platform: "ios", + }); + const harness = await createHarness({ clock, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect( + client.request("driver.passthrough", { args: ["list", "devices"], tool: "simctl" }), + ).resolves.toMatchObject({ + ok: true, + payload: { args: ["simctl", "--set", "/root", "list", "devices"], command: "xcrun" }, + }); + }); + + it("gives a refused verb its own code, so the CLI can render it as a usage error", async () => { + const clock = new FakeClock(1_000); + const driver = passthroughFake({ + availableOsVersions: ["26.5"], + clock, + passthrough: () => { + throw new PassthroughRefusedError("simctl", "Refusing it; use `simlock release` instead."); + }, + passthroughTools: ["simctl"], + platform: "ios", + }); + const harness = await createHarness({ clock, driver }); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect( + client.request("driver.passthrough", { args: ["delete", "ABCD"], tool: "simctl" }), + ).resolves.toMatchObject({ + error: { code: "PASSTHROUGH_REFUSED", message: expect.stringContaining("simlock release") }, + ok: false, + }); + }); + + it.each([[{ args: "list", tool: "simctl" }], [{ args: ["list"] }]])( + "rejects a malformed passthrough payload: %s", + async (payload) => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect(client.request("driver.passthrough", payload)).resolves.toMatchObject({ + error: { code: "BAD_REQUEST" }, + ok: false, + }); + }, + ); + + it("distinguishes an unknown passthrough tool from a malformed payload", async () => { + const harness = await createHarness(); + const client = await createClient(harness.socketPath); + await hello(client); + + await expect( + client.request("driver.passthrough", { args: ["devices"], tool: "adb" }), + ).resolves.toMatchObject({ + error: { code: "UNKNOWN_PASSTHROUGH_TOOL" }, + ok: false, + }); + }); + + it("answers device.add and device.remove over the socket from the daemon's enrollment", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + clock, + physicalDevices: [ + { apps: ["a"], ids: ["UDID-1"], model: "iPhone 15 Pro", osVersion: "18.4" }, + ], + platform: "ios", + }); + const harness = await createHarness({ clock, driver, includeEnrollment: true }); + const client = await createClient(harness.socketPath); + await hello(client); + + const added = await client.request("device.add", { id: "UDID-1", platform: "ios" }, "add-1"); + const removed = await client.request("device.remove", { id: "UDID-1" }, "remove-1"); + + expect(added).toMatchObject({ + ok: true, + payload: { appCount: 1, driverDeviceId: "UDID-1", platform: "ios", reenrolled: false }, + }); + expect(removed).toMatchObject({ + ok: true, + payload: { driverDeviceId: "UDID-1", id: "dev_1" }, + }); + }); +}); + +describe("DaemonServer decorations", () => { + it("surfaces a derived transitionAgeMs for a mid-transition device in status and list --devices", async () => { + const harness = await createHarness(); + const device = await harness.registry.registerDevice({ + driverData: { fakeDeviceId: "driver_provisioning" }, + driverDeviceId: "driver_provisioning", + provisionDuration: 0, + spec: { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" }, + }); + harness.clock.advance(5_000); + + const observer = await createClient(harness.socketPath); + await hello(observer); + await expect(observer.request("status.get", {})).resolves.toMatchObject({ + ok: true, + payload: { devices: [{ id: device.id, transitionAgeMs: 5_000 }] }, + }); + await expect(observer.request("list.get", { kind: "devices" })).resolves.toMatchObject({ + ok: true, + payload: [{ id: device.id, transitionAgeMs: 5_000 }], + }); + await observer.close(); + }); + + describe("operational logging", () => { + it("drains the backgrounded reclaims before disposing, on a graceful stop", async () => { + const order: string[] = []; + let finishSettle!: () => void; + const settling = new Promise((resolve) => { + finishSettle = resolve; + }); + const harness = await createHarness({ + dispose: () => { + order.push("dispose"); + }, + settle: async () => { + order.push("settle-start"); + await settling; + order.push("settle-end"); + }, + }); + const holder = await createClient(harness.socketPath); + await hello(holder); + const grant = await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + await holder.request("lease.release", { leaseId: leaseIdOf(grant) }); + expect(harness.registry.snapshot.leases).toHaveLength(0); + + const stopping = harness.daemon.stop("test-drain"); + await expect.poll(() => order).toEqual(["settle-start"]); + + finishSettle(); + await stopping; + // why: Disposal last: a reclaim that settles into a purge failure arms a quarantine + // retry timer, and cancelling before the drain would strand it armed. + expect(order).toEqual(["settle-start", "settle-end", "dispose"]); + }); + + it("starts the physical-device watcher with the daemon and disposes it on stop before draining settle", async () => { + const order: string[] = []; + const harness = await createHarness({ + physicalDeviceWatcher: { + dispose: () => { + order.push("watcher-dispose"); + }, + start: () => { + order.push("watcher-start"); + }, + }, + settle: async () => { + order.push("settle"); + }, + }); + + await harness.daemon.stop("test-watcher"); + + expect(order).toEqual(["watcher-start", "watcher-dispose", "settle"]); + }); + + it("waits for an asynchronous disposal before reporting the daemon stopped", async () => { + let disposed = false; + let finishDispose!: () => void; + const disposing = new Promise((resolve) => { + finishDispose = resolve; + }); + const harness = await createHarness({ + dispose: async () => { + await disposing; + disposed = true; + }, + }); + + const stopping = harness.daemon.stop("test-async-dispose"); + await expect.poll(() => disposed).toBe(false); + + // why: A driver's release is asynchronous -- Android's reaps the adb server it started -- + // and a stop that returned before it finished would report a shutdown that had not + // happened, leaving the next daemon to find the port still held. + finishDispose(); + await stopping; + expect(disposed).toBe(true); + }); + + it("stops an auxiliary frontend before releasing held leases and draining settle", async () => { + const order: string[] = []; + const harness = await createHarness({ + dispose: () => { + order.push("dispose"); + }, + settle: async () => { + order.push("settle"); + }, + stopAuxiliary: async () => { + order.push("stopAuxiliary"); + }, + }); + const holder = await createClient(harness.socketPath); + await hello(holder); + await holder.request("lease.request", { + requesterId: "holder", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }); + + await harness.daemon.stop("test-stop-auxiliary"); + + expect(order).toEqual(["stopAuxiliary", "settle", "dispose"]); + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); + + it("calls beginStop once, before stopAuxiliary, when a stop is asked for", async () => { + const order: string[] = []; + const harness = await createHarness({ + beginStop: () => { + order.push("beginStop"); + }, + stopAuxiliary: async () => { + order.push("stopAuxiliary"); + }, + }); + + await harness.daemon.stop("test-begin-stop"); + await harness.daemon.stop("again"); + + expect(order).toEqual(["beginStop", "stopAuxiliary"]); + }); + }); +}); diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index bedde8501..db4cfc643 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -1,46 +1,14 @@ -import { mkdtemp } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { FakeDriver } from "../core/testing.js"; -import { EventBus } from "../bus/index.js"; -import { - type DriverRejection, - PassthroughRefusedError, - RuntimeMissingError, -} from "../core/index.js"; +import { afterEach, describe, expect, it } from "vitest"; import { PROTOCOL_VERSION_RANGE } from "../contract/index.js"; -import { - CryptoTokenSecrets, - FakeClock, - JsonLinesLogger, - MemoryFilesystem, - MemoryLogSink, - NodeFilesystem, - NodeIpcTransport, - ScriptedProcessRunner, -} from "../ports/index.js"; +import { FakeClock, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; import { DAEMON_PROTOCOL_VERSION } from "../daemon-protocol/index.js"; -import type { DaemonServer } from "./server.js"; -import { DaemonEndpointHost } from "./connection-host.js"; import { - BlockingConnection, - ControllableStreamingRunner, - SingleConnectionHost, - adminAuth, createClient, createHarness, flush, hello, leaseIdOf, - passthroughDriver, - passthroughFake, - rejectionNaming, - runningDaemons, stopTestDaemons, - temporaryDirectories, - testServerConfig, - waitFor, } from "./test-server.js"; afterEach(stopTestDaemons); @@ -330,1608 +298,4 @@ describe("DaemonServer", () => { ok: true, }); }); - - it("answers status.get with the capacity, answers list.get, config.get, cleanup.run and events.replay as ok, pushes subscribed events, and renews and releases a lease", async () => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect(client.request("status.get", {})).resolves.toMatchObject({ - ok: true, - payload: { - capacity: { - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: { - limit: 1, - maxRunning: 1, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }, - }, - }, - }); - await expect(client.request("list.get", {})).resolves.toMatchObject({ ok: true }); - await expect(client.request("config.get", {})).resolves.toMatchObject({ ok: true }); - await expect(client.request("cleanup.run", { dryRun: true })).resolves.toMatchObject({ - ok: true, - }); - await expect(client.request("events.replay", { sinceTs: 0 })).resolves.toMatchObject({ - ok: true, - }); - await expect(client.request("events.subscribe", {})).resolves.toMatchObject({ ok: true }); - harness.eventBus.emit( - "disk.pressure-detected", - { freeBytes: 1, threshold: testServerConfig().diskPressure.freeBytesThreshold }, - "test", - ); - await expect(client.nextFrame((frame) => frame.push === "event")).resolves.toMatchObject({ - push: "event", - }); - await expect(client.request("events.unsubscribe", {})).resolves.toMatchObject({ ok: true }); - - const grant = await client.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = (grant.payload as { readonly lease: { readonly id: string } }).lease.id; - await expect(client.request("lease.renew", { leaseId, ttlMs: 120_000 })).resolves.toMatchObject( - { - ok: true, - }, - ); - await expect(client.request("lease.release", { leaseId })).resolves.toMatchObject({ ok: true }); - expect(harness.registry.snapshot.leases).toEqual([]); - }); - - it("routes usage.get to the dispatcher with the window the frame carries", async () => { - const harness = await createHarness({ withHistory: true }); - const client = await createClient(harness.socketPath); - await hello(client); - - const answer = await client.request("usage.get", { from: 0, to: 3_600_000 }); - - expect(answer).toMatchObject({ - ok: true, - payload: { totals: { requests: 0 }, window: { from: 0, to: 3_600_000 } }, - }); - }); - - it("serves the device catalog, omitting platforms with no registered driver", async () => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect(client.request("catalog.get", {})).resolves.toMatchObject({ - ok: true, - payload: { - platforms: [{ defaultRuntime: "26.5", models: [], platform: "ios", runtimes: ["26.5"] }], - }, - }); - await expect(client.request("catalog.get", { platform: "ios" })).resolves.toMatchObject({ - ok: true, - payload: { platforms: [{ platform: "ios" }] }, - }); - await expect(client.request("catalog.get", { platform: "android" })).resolves.toMatchObject({ - ok: true, - payload: { platforms: [] }, - }); - await expect(client.request("catalog.get", { platform: "foo" })).resolves.toMatchObject({ - error: { code: "BAD_REQUEST" }, - ok: false, - }); - }); - - it("resolves a passthrough to its driver's command without running it here", async () => { - const clock = new FakeClock(1_000); - const driver = passthroughFake({ - availableOsVersions: ["26.5"], - clock, - passthrough: (args) => ({ - args: ["simctl", "--set", "/root", ...args], - command: "xcrun", - env: {}, - }), - passthroughTools: ["simctl"], - platform: "ios", - }); - const harness = await createHarness({ clock, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect( - client.request("driver.passthrough", { args: ["list", "devices"], tool: "simctl" }), - ).resolves.toMatchObject({ - ok: true, - payload: { args: ["simctl", "--set", "/root", "list", "devices"], command: "xcrun" }, - }); - }); - - it("gives a refused verb its own code, so the CLI can render it as a usage error", async () => { - const clock = new FakeClock(1_000); - const driver = passthroughFake({ - availableOsVersions: ["26.5"], - clock, - passthrough: () => { - throw new PassthroughRefusedError("simctl", "Refusing it; use `simlock release` instead."); - }, - passthroughTools: ["simctl"], - platform: "ios", - }); - const harness = await createHarness({ clock, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect( - client.request("driver.passthrough", { args: ["delete", "ABCD"], tool: "simctl" }), - ).resolves.toMatchObject({ - error: { code: "PASSTHROUGH_REFUSED", message: expect.stringContaining("simlock release") }, - ok: false, - }); - }); - - it.each([[{ args: "list", tool: "simctl" }], [{ args: ["list"] }]])( - "rejects a malformed passthrough payload: %s", - async (payload) => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect(client.request("driver.passthrough", payload)).resolves.toMatchObject({ - error: { code: "BAD_REQUEST" }, - ok: false, - }); - }, - ); - - it("distinguishes an unknown passthrough tool from a malformed payload", async () => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect( - client.request("driver.passthrough", { args: ["devices"], tool: "adb" }), - ).resolves.toMatchObject({ - error: { code: "UNKNOWN_PASSTHROUGH_TOOL" }, - ok: false, - }); - }); - - it("pushes device.exec output frames keyed by the request id, then replies with the exit code", async () => { - const clock = new FakeClock(1_000); - const driver = passthroughFake({ - availableOsVersions: ["26.5"], - clock, - passthrough: (args: readonly string[]) => ({ - args: ["--set", "/root", ...args], - command: "simctl", - env: {}, - }), - passthroughTools: ["simctl"], - platform: "ios", - }); - const harness = await createHarness({ - clock, - driver, - processRunner: new ScriptedProcessRunner([ - { - chunks: [ - { chunk: "booted", stream: "stdout" }, - { chunk: "a warning", stream: "stderr" }, - ], - match: { args: ["--set", "/root", "list", "devices"], command: "simctl" }, - result: { code: 6, stderr: "", stdout: "" }, - }, - ]), - }); - const client = await createClient(harness.socketPath); - await hello(client); - const grant = await client.request("lease.request", { - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - }); - - const response = await client.request( - "device.exec", - { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, - "exec-frame", - ); - - expect(response).toMatchObject({ ok: true, payload: { exitCode: 6 } }); - expect(client.frames().filter((frame) => frame.push === "output")).toEqual([ - { payload: { chunk: "booted", requestId: "exec-frame", stream: "stdout" }, push: "output" }, - { - payload: { chunk: "a warning", requestId: "exec-frame", stream: "stderr" }, - push: "output", - }, - ]); - - // why: ADR 0005 §19a: `started` reaches the socket exactly once, keyed by the same frame id. - // Round 5 review: this send site could be deleted with the entire suite green, because - // nothing asserted the frame ever reached a socket at all. - // - // Deliberately not asserted here: that it precedes the first `output`. It does in - // production -- the dispatcher calls `onStarted` synchronously after the spawn returns, and - // a real child's chunks arrive later from `data` events -- but `ScriptedProcessRunner` - // delivers its scripted chunks synchronously *inside* `runCommand`, so this harness sees - // them first and cannot distinguish the two orderings. Claiming the ordering here would be a - // title outrunning its body; `client.test.ts`'s wire test pins it where it is observable. - expect(client.frames().filter((frame) => frame.push === "started")).toEqual([ - { payload: { requestId: "exec-frame" }, push: "started" }, - ]); - }); - - it("pushes component.install progress frames keyed by the request id, then replies with the outcome", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - installProgress: [50], - platform: "ios", - }); - const harness = await createHarness({ clock, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request( - "component.install", - { platform: "ios", version: "27.0" }, - "install-frame", - ); - - expect(response).toMatchObject({ - ok: true, - payload: { component: "27.0", outcome: "installed", platform: "ios", version: "27.0" }, - }); - expect(client.frames().filter((frame) => frame.push === "component-progress")).toEqual([ - { - payload: { progress: { stage: "downloading" }, requestId: "install-frame" }, - push: "component-progress", - }, - { - payload: { progress: { fraction: 0.5, stage: "downloading" }, requestId: "install-frame" }, - push: "component-progress", - }, - { - payload: { progress: { fraction: 1, stage: "downloading" }, requestId: "install-frame" }, - push: "component-progress", - }, - ]); - }); - - it("answers device.add and device.remove over the socket from the daemon's enrollment", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - clock, - physicalDevices: [ - { apps: ["a"], ids: ["UDID-1"], model: "iPhone 15 Pro", osVersion: "18.4" }, - ], - platform: "ios", - }); - const harness = await createHarness({ clock, driver, includeEnrollment: true }); - const client = await createClient(harness.socketPath); - await hello(client); - - const added = await client.request("device.add", { id: "UDID-1", platform: "ios" }, "add-1"); - const removed = await client.request("device.remove", { id: "UDID-1" }, "remove-1"); - - expect(added).toMatchObject({ - ok: true, - payload: { appCount: 1, driverDeviceId: "UDID-1", platform: "ios", reenrolled: false }, - }); - expect(removed).toMatchObject({ - ok: true, - payload: { driverDeviceId: "UDID-1", id: "dev_1" }, - }); - }); - - it("routes two commands on one connection by frame id, and pushes to no other connection", async () => { - const clock = new FakeClock(1_000); - const driver = passthroughDriver(clock); - const harness = await createHarness({ - clock, - driver, - processRunner: new ScriptedProcessRunner([ - { - chunks: [{ chunk: "first-command", stream: "stdout" }], - match: { args: ["--set", "/root", "list", "devices"], command: "simctl" }, - }, - { - chunks: [{ chunk: "second-command", stream: "stdout" }], - match: { args: ["--set", "/root", "list", "runtimes"], command: "simctl" }, - }, - ]), - }); - const client = await createClient(harness.socketPath); - const observer = await createClient(harness.socketPath); - await hello(client); - await hello(observer); - const grant = await client.request("lease.request", { - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - - const first = client.request( - "device.exec", - { args: ["list", "devices"], leaseId, tool: "simctl" }, - "exec-a", - ); - const second = client.request( - "device.exec", - { args: ["list", "runtimes"], leaseId, tool: "simctl" }, - "exec-b", - ); - await Promise.all([first, second]); - - expect( - client - .frames() - .filter((frame) => frame.push === "output") - .map((frame) => frame.payload), - ).toEqual([ - { chunk: "first-command", requestId: "exec-a", stream: "stdout" }, - { chunk: "second-command", requestId: "exec-b", stream: "stdout" }, - ]); - expect(observer.frames().filter((frame) => frame.push === "output")).toEqual([]); - }); - - it("stops pushing when the connection dies, and lets the command finish anyway", async () => { - const clock = new FakeClock(1_000); - const driver = passthroughDriver(clock); - const runner = new ControllableStreamingRunner(); - const harness = await createHarness({ clock, driver, processRunner: runner }); - const client = await createClient(harness.socketPath); - await hello(client); - const grant = await client.request("lease.request", { - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - }); - - void client.request( - "device.exec", - { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, - "exec-dropped", - ); - await waitFor(() => runner.handle !== undefined); - const firstPush = client.nextFrame((frame) => frame.push === "output"); - runner.handle?.emit("stdout", "before-the-drop"); - expect(await firstPush).toMatchObject({ - payload: { chunk: "before-the-drop", requestId: "exec-dropped" }, - push: "output", - }); - - await client.close(); - await flush(); - runner.handle?.emit("stdout", "after-the-drop"); - runner.handle?.finish(0); - await flush(); - expect(runner.handle?.settled).toBe(true); - - const survivor = await createClient(harness.socketPath); - await hello(survivor); - await expect(survivor.request("status.get", {})).resolves.toMatchObject({ ok: true }); - await survivor.close(); - }); - - it("keeps ConnectionPushes.output's promise unresolved while a frame is still going out, resolving it once the write drains", async () => { - // why: ADR 0005 §19e on this transport: `ConnectionPushes.output`'s returned promise is - // what `device.exec` hands the process runner as a chunk's delivery, and it is that promise - // -- not anything this test can observe about a real stream -- staying unresolved while a - // client's socket has not drained that would eventually stop the command at its pipe. - // `ControllableStreamingRunner` is a fake with no stream of its own to stop reading (its - // `emit` calls `onChunk` directly), so this proves only the promise half of the contract; - // `ports/process-runner.test.ts`'s `NodeProcessRunner` suite is where a real pipe is - // actually paused. Driven through a connection whose `write` this test settles by hand -- - // a real socket would need megabytes to fill a kernel buffer, which is the same property - // tested by luck. - const clock = new FakeClock(1_000); - const driver = passthroughDriver(clock); - const runner = new ControllableStreamingRunner(); - const connection = new BlockingConnection(); - const harness = await createHarness({ - clock, - driver, - host: new SingleConnectionHost(connection), - processRunner: runner, - }); - expect(harness.daemon.health).toBeDefined(); - - await connection.request( - { clientVersion: "test", protocolVersion: DAEMON_PROTOCOL_VERSION }, - "hello", - ); - const grant = await connection.request( - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - "lease.request", - ); - connection.releaseAll(); - - void connection.request( - { args: ["list", "devices"], leaseId: leaseIdOf(grant), tool: "simctl" }, - "device.exec", - "exec-slow", - ); - await waitFor(() => runner.handle !== undefined); - - connection.block(); - let firstDelivered = false; - void Promise.resolve(runner.handle?.emit("stdout", "first")).then(() => { - firstDelivered = true; - }); - await flush(); - expect(firstDelivered).toBe(false); - - connection.releaseAll(); - await flush(); - expect(firstDelivered).toBe(true); - - runner.handle?.finish(0); - await flush(); - }); - - it("recovers a stale socket file and refuses a second live daemon before running any device work", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-stale-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - await new NodeFilesystem().writeFileAtomic(socketPath, "stale"); - const first = await createHarness({ socketPath }); - let convergeCalls = 0; - const second = await createHarness({ - converge: () => { - convergeCalls += 1; - return Promise.resolve(); - }, - socketPath, - start: false, - }); - - await expect(second.daemon.start()).rejects.toMatchObject({ - name: "DaemonAlreadyRunningError", - }); - expect(convergeCalls).toBe(0); - await second.daemon.stop("failed-start"); - const client = await createClient(socketPath); - await hello(client); - await client.close(); - const firstDaemon: DaemonServer = first.daemon; - expect(firstDaemon.socketPath).toBe(socketPath); - }); - - // why: review finding S1: `AdminSecretManager`'s own doc (`src/auth`) asserts "a daemon - // that loses the start race never calls `persist()` at all ... so the file an - // already-running daemon wrote is never touched by the loser" -- untested anywhere before - // this. Mirrors the "recovers a stale socket file" test above (two harnesses racing for the - // same socket path), but gives each its own real `AdminSecretManager` over its own - // `MemoryFilesystem` so this can assert on the filesystem directly, rather than trusting a - // stub. In production both instances would target the *same* real `admin.token` path (see - // `main.ts`); separate filesystems here isolate what each instance actually did, without that - // shared path letting a wrongly-invoked `remove()` on the loser silently clean up after a - // wrongly-invoked `persist()` and mask the very bug this test exists to catch. - it("never touches admin.token on the daemon instance that loses the start race", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-stale-admin-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - await new NodeFilesystem().writeFileAtomic(socketPath, "stale"); - - const secrets = new CryptoTokenSecrets(); - const winnerFilesystem = new MemoryFilesystem(); - await createHarness({ - adminSecret: (await adminAuth(winnerFilesystem, secrets)).adminSecret, - socketPath, - }); - await expect(winnerFilesystem.readFile("/admin.token")).resolves.toContain("\n"); - - const loserFilesystem = new MemoryFilesystem(); - const second = await createHarness({ - adminSecret: (await adminAuth(loserFilesystem, secrets)).adminSecret, - socketPath, - start: false, - }); - await expect(second.daemon.start()).rejects.toMatchObject({ - name: "DaemonAlreadyRunningError", - }); - await expect(loserFilesystem.readFile("/admin.token")).rejects.toThrow(); - - await second.daemon.stop("failed-start"); - const client = await createClient(socketPath); - await hello(client); - await client.close(); - }); - - it.each([false, true])( - "preserves the replacement daemon's admin credential and still releases the endpoint when it starts as the old endpoint is released (credential removal fails: %s)", - async (removalFails) => { - const directory = await mkdtemp(join(tmpdir(), "simlock-admin-restart-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - const filesystem = new MemoryFilesystem(); - const secrets = new CryptoTokenSecrets(); - const oldSecret = await adminAuth(filesystem, secrets); - const replacementSecret = await adminAuth(filesystem, secrets); - const replacement = await createHarness({ - adminSecret: replacementSecret.adminSecret, - socketPath, - start: false, - }); - const host = new DaemonEndpointHost({ - connector: new NodeIpcTransport(), - endpoint: socketPath, - filesystem: new NodeFilesystem(), - listenerFactory: new NodeIpcTransport(), - }); - let replacementToken: string | undefined; - const old = await createHarness({ - adminSecret: oldSecret.adminSecret, - host: { - endpoint: socketPath, - start: (accept) => host.start(accept), - async stop() { - await host.stop(); - // why: endpoint release lets a new process start before the old stop resumes. - await replacement.daemon.start(); - replacementToken = await filesystem.readFile("/admin.token"); - await expect( - replacementSecret.resolveHelloRole({ credential: replacementToken.trim() }), - ).resolves.toBe("admin"); - }, - }, - socketPath, - }); - await expect( - oldSecret.resolveHelloRole({ - credential: (await filesystem.readFile("/admin.token")).trim(), - }), - ).resolves.toBe("admin"); - - if (removalFails) { - vi.spyOn(filesystem, "rm").mockRejectedValueOnce(new Error("EACCES")); - await expect(old.daemon.stop("restart")).rejects.toThrow("EACCES"); - // why: the failed stop is settled; teardown must not stop it again. - runningDaemons.splice(runningDaemons.indexOf(old.daemon), 1); - } else { - await old.daemon.stop("restart"); - } - - // why: the replacement could only start because the old stop released the endpoint. - expect(replacementToken).toBeDefined(); - await expect(filesystem.readFile("/admin.token")).resolves.toBe(replacementToken); - }, - ); - - it("gracefully stops without touching leases, persisting them with their deadlines", async () => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - const grant = await client.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - - await expect(client.request("daemon.stop", {})).resolves.toMatchObject({ ok: true }); - - expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); - expect(harness.state.database.prepare("SELECT id FROM leases").all()).toEqual([ - { id: leaseId }, - ]); - expect(harness.eventBus.replay().map((event) => event.event)).toContain("daemon.stopping"); - expect(harness.eventBus.replay().map((event) => event.event)).not.toContain("lease.released"); - }); - - it("pushes a lease-lost notification to the holding connection when its lease's TTL backstop expires", async () => { - const harness = await createHarness(); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - const deviceId = harness.registry.snapshot.leases.find( - (lease) => lease.id === leaseId, - )?.deviceId; - - harness.clock.advance(60_000); - await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ - payload: { deviceId, leaseId, reason: "expired" }, - push: "lease-lost", - }); - await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); - - await holder.close(); - const observer = await createClient(harness.socketPath); - await hello(observer); - await expect(observer.request("status.get", {})).resolves.toMatchObject({ ok: true }); - await observer.close(); - }); - - it("pushes a lease-lost notification to the actual holding connection when another connection force-releases its lease", async () => { - const harness = await createHarness(); - const holder = await createClient(harness.socketPath); - const releaser = await createClient(harness.socketPath); - await Promise.all([hello(holder), hello(releaser)]); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - - await expect(releaser.request("lease.release", { leaseId })).resolves.toMatchObject({ - ok: true, - }); - await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ - payload: { leaseId, reason: "explicit" }, - push: "lease-lost", - }); - - await expect(holder.request("lease.release", { leaseId })).resolves.toMatchObject({ - error: { code: "UNKNOWN_LEASE" }, - ok: false, - }); - await holder.close(); - await releaser.close(); - }); - - it("drops a lease-lost push to a connection whose peer has hung up instead of leaving its failed write unhandled", async () => { - const connection = new BlockingConnection(); - const harness = await createHarness({ host: new SingleConnectionHost(connection) }); - await connection.request( - { clientVersion: "test", protocolVersion: DAEMON_PROTOCOL_VERSION }, - "hello", - ); - const grant = await connection.request( - { model: "iPhone 16", osVersion: "26.5", platform: "ios" }, - "lease.request", - ); - const unhandled: unknown[] = []; - const recordUnhandled = (reason: unknown) => unhandled.push(reason); - process.on("unhandledRejection", recordUnhandled); - try { - connection.hangUp(); - await harness.engine.release(leaseIdOf(grant), "explicit"); - await new Promise((resolve) => setImmediate(resolve)); - - expect(connection.frames.filter((frame) => frame.push === "lease-lost")).toHaveLength(1); - expect(unhandled).toEqual([]); - } finally { - process.off("unhandledRejection", recordUnhandled); - } - }); - - type EventBus = Awaited>["eventBus"]; - - it.each([ - [ - "lease-lost", - (eventBus: EventBus) => { - eventBus.emit( - "lease.expired", - { deviceId: "device-x", leaseId: "lease-x", ownerId: "test-process" }, - "test", - ); - eventBus.emit( - "lease.released", - { deviceId: "device-y", leaseId: "lease-y", ownerId: "test-process", reason: "killed" }, - "test", - ); - }, - ], - [ - "device-unhealthy/device-recovered", - (eventBus: EventBus) => { - eventBus.emit( - "device.crash-detected", - { deviceId: "device-x", leaseId: "lease-x", observed: "stopped", platform: "ios" }, - "test", - ); - eventBus.emit( - "device.recovered", - { - attempts: 1, - deviceId: "device-x", - duration: 1_000, - leaseId: "lease-x", - platform: "ios", - }, - "test", - ); - }, - ], - ])( - "ignores %s facts for leases with no currently connected holder without breaking the daemon", - async (_name, emitFacts) => { - const harness = await createHarness(); - emitFacts(harness.eventBus); - - const client = await createClient(harness.socketPath); - await hello(client); - await expect(client.request("status.get", {})).resolves.toMatchObject({ ok: true }); - await client.close(); - }, - ); - - it("pushes a device-unhealthy notification on device.crash-detected without releasing the lease", async () => { - const harness = await createHarness(); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - const deviceId = harness.registry.snapshot.leases.find( - (lease) => lease.id === leaseId, - )?.deviceId; - - harness.eventBus.emit( - "device.crash-detected", - { deviceId: deviceId as string, leaseId, observed: "stopped", platform: "ios" }, - "test", - ); - await expect( - holder.nextFrame((frame) => frame.push === "device-unhealthy"), - ).resolves.toMatchObject({ - payload: { deviceId, leaseId, reason: "crashed" }, - push: "device-unhealthy", - }); - - expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); - await holder.request("lease.release", { leaseId }); - await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); - await holder.close(); - }); - - it("pushes a device-recovered notification on device.recovered without releasing the lease", async () => { - const harness = await createHarness(); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - const deviceId = harness.registry.snapshot.leases.find( - (lease) => lease.id === leaseId, - )?.deviceId; - - harness.eventBus.emit( - "device.recovered", - { - attempts: 2, - deviceId: deviceId as string, - duration: 30_000, - leaseId, - platform: "ios", - }, - "test", - ); - await expect( - holder.nextFrame((frame) => frame.push === "device-recovered"), - ).resolves.toMatchObject({ - payload: { attempts: 2, deviceId, leaseId }, - push: "device-recovered", - }); - - expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); - await holder.request("lease.release", { leaseId }); - await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); - await holder.close(); - }); - - it("pushes a lease-lost notification carrying reason device-lost when a leased device could not be recovered", async () => { - const harness = await createHarness(); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - const deviceId = harness.registry.snapshot.leases.find( - (lease) => lease.id === leaseId, - )?.deviceId; - - harness.eventBus.emit( - "lease.released", - { deviceId: deviceId as string, leaseId, ownerId: "test-process", reason: "device-lost" }, - "test", - ); - await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ - payload: { deviceId, leaseId, reason: "device-lost" }, - push: "lease-lost", - }); - }); -}); - -describe("DaemonServer driver rejections", () => { - it("publishes one event per platform that refused to start, after the daemon is up", async () => { - const harness = await createHarness({ - driverRejections: [ - { - event: "driver.root-rejected", - payload: { platform: "ios", reason: "wrong-instance", root: "/Devices" }, - platform: "ios", - reason: "wrong-instance", - summary: "Refusing the ios device root /Devices: it belongs to another instance", - }, - { - event: "driver.adb-server-rejected", - payload: { port: 5038, reason: "occupied" }, - platform: "android", - reason: "occupied", - summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", - }, - ], - }); - - const events = harness.eventBus.replay(); - const rejected = events.filter( - (event) => - event.event === "driver.root-rejected" || event.event === "driver.adb-server-rejected", - ); - expect(rejected).toEqual([ - expect.objectContaining({ - event: "driver.root-rejected", - module: "daemon", - payload: { platform: "ios", reason: "wrong-instance", root: "/Devices" }, - }), - expect.objectContaining({ - event: "driver.adb-server-rejected", - module: "daemon", - payload: { port: 5038, reason: "occupied" }, - }), - ]); - const started = events.find((event) => event.event === "daemon.started"); - for (const event of rejected) expect(started?.seq).toBeLessThan(event.seq); - }); - - it("names the refusal when a lease asks for a platform whose driver did not start", async () => { - const harness = await createHarness({ - driverRejections: [ - { - event: "driver.adb-server-rejected", - payload: { port: 5038, reason: "occupied" }, - platform: "android", - reason: "occupied", - summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", - }, - ], - }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("lease.request", { - requesterId: "agent-1", - model: "Pixel 8", - platform: "android", - }); - - expect(response.error?.code).toBe("NO_DRIVER"); - expect(response.error?.message).toContain("port 5038 is held by an adb server we do not own"); - }); - - it("leaves an ordinary missing platform unexplained, having nothing to explain", async () => { - const harness = await createHarness({ - driverRejections: [ - { - event: "driver.root-rejected", - payload: { platform: "ios", reason: "symlink", root: "/Devices" }, - platform: "ios", - reason: "symlink", - summary: "Refusing the ios device root /Devices: it is a symlink", - }, - ], - }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("lease.request", { - requesterId: "agent-1", - model: "Pixel 8", - platform: "android", - }); - - expect(response.error?.code).toBe("NO_DRIVER"); - expect(response.error?.message).toBe("No driver registered for platform: android"); - }); - - it.each([ - { - name: "adb", - rejection: { - event: "driver.adb-server-rejected", - payload: { port: 5038, reason: "occupied" }, - platform: "android", - reason: "occupied", - summary: "Refusing the Android driver: port 5038 is held by an adb server we do not own", - }, - tools: ["adb"], - request: { args: ["devices"], tool: "adb" }, - text: "port 5038 is held by an adb server we do not own", - }, - { - name: "devicectl", - rejection: { - event: "driver.root-rejected", - payload: { platform: "ios", reason: "symlink", root: "/Devices" }, - platform: "ios", - reason: "symlink", - summary: "Refusing the ios device root /Devices: it is a symlink", - }, - tools: ["simctl", "devicectl"], - request: { args: ["list", "devices"], tool: "devicectl" }, - text: "Refusing the ios device root /Devices: it is a symlink", - }, - ] as const)( - "names the refusal when a $name passthrough asks for the tool whose driver did not start", - async ({ rejection, request, text, tools }) => { - const harness = await createHarness({ - driverRejections: [rejectionNaming(rejection, tools)], - }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("driver.passthrough", { - args: [...request.args], - tool: request.tool, - }); - - expect(response.error?.code).toBe("UNKNOWN_PASSTHROUGH_TOOL"); - expect(response.error?.message).toContain(text); - }, - ); - - it("leaves an unknown passthrough tool unexplained when no driver claimed it", async () => { - const harness = await createHarness({ - driverRejections: [ - rejectionNaming( - { - event: "driver.root-rejected", - payload: { platform: "ios", reason: "symlink", root: "/Devices" }, - platform: "ios", - reason: "symlink", - summary: "Refusing the ios device root /Devices: it is a symlink", - }, - ["simctl", "devicectl"], - ), - ], - }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("driver.passthrough", { - args: ["devices"], - tool: "adb", - }); - - expect(response.error?.message).toBe("No driver provides a adb passthrough"); - }); - - it("refuses at compile time to pair an event with another event's payload", () => { - const rejection: DriverRejection = { - event: "driver.adb-server-rejected", - // @ts-expect-error -- `port` is a number on the wire (docs/internal/EVENTS.md). - payload: { port: "5038", reason: "occupied" }, - platform: "android", - reason: "occupied", - summary: "Refusing the Android driver: port 5038 is occupied", - }; - - expect(rejection.event).toBe("driver.adb-server-rejected"); - }); - - it("publishes nothing when every driver started", async () => { - const harness = await createHarness(); - - expect(harness.eventBus.replay().filter((event) => event.event.startsWith("driver."))).toEqual( - [], - ); - }); -}); - -describe("DaemonServer lease liveness (ADR 0004)", () => { - it("keeps a lease alive only while something renews it, and expires it when nothing does", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 40 } }); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - expect((grant.payload as { lease: { ttlDeadline: number } }).lease.ttlDeadline).toBe(1_040); - - let expectedDeadline = 1_040; - for (let cycle = 0; cycle < 6; cycle += 1) { - harness.clock.advance(10); - expectedDeadline = harness.clock.now() + 40; - await expect(holder.request("lease.renew", { leaseId })).resolves.toMatchObject({ - ok: true, - payload: { id: leaseId, ttlDeadline: expectedDeadline, ttlMs: 40 }, - }); - } - expect(harness.registry.snapshot.leases).toMatchObject([{ id: leaseId }]); - - harness.clock.advance(40); - await expect.poll(() => harness.registry.snapshot.leases).toEqual([]); - await expect(holder.nextFrame((frame) => frame.push === "lease-lost")).resolves.toMatchObject({ - payload: { leaseId, reason: "expired" }, - push: "lease-lost", - }); - await holder.close(); - }); - - it("no longer answers lease.heartbeat at all", async () => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect(client.request("lease.heartbeat", { nonce: 1 })).resolves.toMatchObject({ - error: { code: "UNKNOWN_REQUEST" }, - ok: false, - }); - await client.close(); - }); - - it("answers worker.list with this daemon's instance id and the version its hello reports", async () => { - const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); - const client = await createClient(harness.socketPath); - const greeting = await client.request("hello", { - clientVersion: "test", - protocolVersion: DAEMON_PROTOCOL_VERSION, - }); - expect(greeting).toMatchObject({ ok: true, payload: { version: "test" } }); - - await expect(client.request("worker.list", {})).resolves.toMatchObject({ - ok: true, - payload: { workers: [{ id: "instance-test", version: "test" }] }, - }); - await client.close(); - }); - - it("re-reads worker.list's kept catalog once this daemon's bus reports a component installed", async () => { - const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); - const client = await createClient(harness.socketPath); - await hello(client); - const catalogReads = vi.spyOn(harness.driver, "listCatalog"); - - await client.request("worker.list", {}); - await client.request("worker.list", {}); - expect(catalogReads).toHaveBeenCalledTimes(1); - - harness.eventBus.emit( - "component.installed", - { - alreadyPresent: false, - componentId: "27.0", - durationMs: 1, - platform: "ios", - version: "27.0", - }, - "test", - ); - await client.request("worker.list", {}); - - expect(catalogReads).toHaveBeenCalledTimes(2); - await client.close(); - }); - - it("disposes the worker's dispatcher, ending its bus subscriptions, when the daemon stops", async () => { - const live = new Map(); - const subscribe = EventBus.prototype.subscribe; - const spy = vi.spyOn(EventBus.prototype, "subscribe").mockImplementation(function ( - this: EventBus, - event, - handler, - ) { - const unsubscribe = subscribe.call(this, event, handler); - live.set(event, (live.get(event) ?? 0) + 1); - let ended = false; - return () => { - if (!ended) live.set(event, (live.get(event) ?? 0) - 1); - ended = true; - unsubscribe(); - }; - }); - try { - const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); - expect(live.get("component.installed")).toBeGreaterThan(0); - - await harness.daemon.stop("test"); - - expect(live.get("component.installed")).toBe(0); - } finally { - spy.mockRestore(); - } - }); - - it("answers worker.install-component and worker.update with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on or update", async () => { - const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); - const client = await createClient(harness.socketPath); - await hello(client); - - await expect( - client.request("worker.install-component", { - platform: "android", - version: "35", - workers: "all", - }), - ).resolves.toMatchObject({ - error: { - code: "UNSUPPORTED_IN_WORKER_MODE", - details: { operation: "worker.install-component" }, - }, - ok: false, - }); - await expect(client.request("worker.update", { workerId: "wrk_1" })).resolves.toMatchObject({ - error: { code: "UNSUPPORTED_IN_WORKER_MODE", details: { operation: "worker.update" } }, - ok: false, - }); - await client.close(); - }); - - it("keeps a lease across a daemon restart and restores its timer from the persisted deadline", async () => { - const leaseOverrides = { defaultTtlMs: 40 }; - const first = await createHarness({ lease: leaseOverrides }); - const holder = await createClient(first.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - const deviceId = (grant.payload as { device: { id: string } }).device.id; - - first.clock.advance(10); - await expect(holder.request("lease.renew", { leaseId })).resolves.toMatchObject({ - ok: true, - payload: { ttlDeadline: 1_050 }, - }); - - const second = await createHarness({ - clock: new FakeClock(1_010), - driver: first.driver, - lease: leaseOverrides, - state: first.state, - }); - await second.engine.convergeRunningCapacity(); - - expect(second.registry.snapshot.leases).toMatchObject([{ id: leaseId, ttlDeadline: 1_050 }]); - expect(second.registry.snapshot.devices).toMatchObject([{ id: deviceId, state: "leased" }]); - - second.clock.advance(30); - await expect.poll(() => second.registry.snapshot.leases).toHaveLength(1); - second.clock.advance(10); - await expect.poll(() => second.registry.snapshot.leases).toEqual([]); - await expect.poll(() => second.registry.snapshot.devices[0]?.state).toBe("deleted"); - await holder.close(); - }); - - it("expires a lease whose deadline passed while no daemon was running, as soon as one is", async () => { - const leaseOverrides = { defaultTtlMs: 40 }; - const first = await createHarness({ lease: leaseOverrides }); - const holder = await createClient(first.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const deviceId = (grant.payload as { device: { id: string } }).device.id; - await holder.close(); - - const second = await createHarness({ - clock: new FakeClock(2_000), - driver: first.driver, - lease: leaseOverrides, - state: first.state, - }); - await second.engine.convergeRunningCapacity(); - - await expect.poll(() => second.registry.snapshot.leases).toEqual([]); - await expect - .poll(() => second.registry.snapshot.devices) - .toMatchObject([{ id: deviceId, state: "deleted" }]); - }); - - it("surfaces the stored lastRenewedAt and ttlMs in status and list --leases", async () => { - const harness = await createHarness({ lease: { defaultTtlMs: 40 } }); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - const leaseId = leaseIdOf(grant); - expect((grant.payload as { lease: { lastRenewedAt: number } }).lease.lastRenewedAt).toBe(1_000); - - harness.clock.advance(10); - await holder.request("lease.renew", { leaseId }); - - await expect(holder.request("status.get", {})).resolves.toMatchObject({ - ok: true, - payload: { leases: [{ id: leaseId, lastRenewedAt: 1_010, ttlMs: 40 }] }, - }); - await expect(holder.request("list.get", { kind: "leases" })).resolves.toMatchObject({ - ok: true, - payload: [{ id: leaseId, lastRenewedAt: 1_010, ttlMs: 40 }], - }); - await holder.close(); - }); -}); - -describe("DaemonServer decorations", () => { - it("surfaces a derived transitionAgeMs for a mid-transition device in status and list --devices", async () => { - const harness = await createHarness(); - const device = await harness.registry.registerDevice({ - driverData: { fakeDeviceId: "driver_provisioning" }, - driverDeviceId: "driver_provisioning", - provisionDuration: 0, - spec: { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" }, - }); - harness.clock.advance(5_000); - - const observer = await createClient(harness.socketPath); - await hello(observer); - await expect(observer.request("status.get", {})).resolves.toMatchObject({ - ok: true, - payload: { devices: [{ id: device.id, transitionAgeMs: 5_000 }] }, - }); - await expect(observer.request("list.get", { kind: "devices" })).resolves.toMatchObject({ - ok: true, - payload: [{ id: device.id, transitionAgeMs: 5_000 }], - }); - await observer.close(); - }); - - describe("operational logging", () => { - it("drains the backgrounded reclaims before disposing, on a graceful stop", async () => { - const order: string[] = []; - let finishSettle!: () => void; - const settling = new Promise((resolve) => { - finishSettle = resolve; - }); - const harness = await createHarness({ - dispose: () => { - order.push("dispose"); - }, - settle: async () => { - order.push("settle-start"); - await settling; - order.push("settle-end"); - }, - }); - const holder = await createClient(harness.socketPath); - await hello(holder); - const grant = await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - await holder.request("lease.release", { leaseId: leaseIdOf(grant) }); - expect(harness.registry.snapshot.leases).toHaveLength(0); - - const stopping = harness.daemon.stop("test-drain"); - await expect.poll(() => order).toEqual(["settle-start"]); - - finishSettle(); - await stopping; - // why: Disposal last: a reclaim that settles into a purge failure arms a quarantine - // retry timer, and cancelling before the drain would strand it armed. - expect(order).toEqual(["settle-start", "settle-end", "dispose"]); - }); - - it("starts the physical-device watcher with the daemon and disposes it on stop before draining settle", async () => { - const order: string[] = []; - const harness = await createHarness({ - physicalDeviceWatcher: { - dispose: () => { - order.push("watcher-dispose"); - }, - start: () => { - order.push("watcher-start"); - }, - }, - settle: async () => { - order.push("settle"); - }, - }); - - await harness.daemon.stop("test-watcher"); - - expect(order).toEqual(["watcher-start", "watcher-dispose", "settle"]); - }); - - it("waits for an asynchronous disposal before reporting the daemon stopped", async () => { - let disposed = false; - let finishDispose!: () => void; - const disposing = new Promise((resolve) => { - finishDispose = resolve; - }); - const harness = await createHarness({ - dispose: async () => { - await disposing; - disposed = true; - }, - }); - - const stopping = harness.daemon.stop("test-async-dispose"); - await expect.poll(() => disposed).toBe(false); - - // why: A driver's release is asynchronous -- Android's reaps the adb server it started -- - // and a stop that returned before it finished would report a shutdown that had not - // happened, leaving the next daemon to find the port still held. - finishDispose(); - await stopping; - expect(disposed).toBe(true); - }); - - it("stops an auxiliary frontend before releasing held leases and draining settle", async () => { - const order: string[] = []; - const harness = await createHarness({ - dispose: () => { - order.push("dispose"); - }, - settle: async () => { - order.push("settle"); - }, - stopAuxiliary: async () => { - order.push("stopAuxiliary"); - }, - }); - const holder = await createClient(harness.socketPath); - await hello(holder); - await holder.request("lease.request", { - requesterId: "holder", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - await harness.daemon.stop("test-stop-auxiliary"); - - expect(order).toEqual(["stopAuxiliary", "settle", "dispose"]); - expect(harness.registry.snapshot.leases).toHaveLength(1); - }); - - it("calls beginStop once, before stopAuxiliary, when a stop is asked for", async () => { - const order: string[] = []; - const harness = await createHarness({ - beginStop: () => { - order.push("beginStop"); - }, - stopAuxiliary: async () => { - order.push("stopAuxiliary"); - }, - }); - - await harness.daemon.stop("test-begin-stop"); - await harness.daemon.stop("again"); - - expect(order).toEqual(["beginStop", "stopAuxiliary"]); - }); - }); -}); - -describe("DaemonServer download policy", () => { - it("grants download permission under the always policy without a per-request flag", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); - const harness = await createHarness({ clock, downloads: { policy: "always" }, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - const grant = await client.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(grant.ok).toBe(true); - expect( - driver.calls - .filter((call) => call.operation === "installComponent") - .map((call) => call.arguments), - ).toEqual([["26.5"]]); - await client.close(); - }); - - it("still answers a lease request whose download names a component past the progress push's bound, sending no downloading push for it", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - // why: the driver, not the request, names the component: a request's exact osVersion is held - // within the push's bound. Every resolve before the install answers as a missing runtime does. - for (const callNumber of [1, 2]) { - driver.failOn( - "resolveSpec", - callNumber, - new RuntimeMissingError("ios", "26.5", { component: "9".repeat(65) }), - ); - } - const harness = await createHarness({ clock, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - const grant = await client.request("lease.request", { - allowDownload: true, - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(grant.ok, JSON.stringify(grant.error)).toBe(true); - const stages = client - .frames() - .filter((frame) => frame.push === "progress") - .map((frame) => (frame.payload as { progress: { stage: string } }).progress.stage); - expect(stages).toContain("provisioning"); - expect(stages).not.toContain("downloading"); - await client.close(); - }); - - it.each([ - ["asks for it", { allowDownload: true }], - ["never asked for a download", {}], - ])( - "attaches the download-policy suffix and withholds download permission under the never policy even when the request %s", - async (_name, extra) => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); - const harness = await createHarness({ clock, downloads: { policy: "never" }, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - // why: when the request never asked, the driver's own message still suggests `--allow-download`, - // which under the never policy can never help, so the suffix must still attach as the - // correction. - const response = await client.request("lease.request", { - ...extra, - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(response.ok).toBe(false); - expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); - expect(response.error?.message).toContain("downloads.policy"); - expect(driver.calls.map((call) => call.operation)).not.toContain("installComponent"); - await client.close(); - }, - ); - - it("defers to the request's own flag under the default on-request policy", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); - const harness = await createHarness({ clock, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(response.ok).toBe(false); - expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); - expect(response.error?.message).not.toContain("downloads.policy"); - expect(driver.calls.map((call) => call.operation)).not.toContain("installComponent"); - - const flagged = await client.request("lease.request", { - allowDownload: true, - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(flagged.ok).toBe(true); - expect( - driver.calls - .filter((call) => call.operation === "installComponent") - .map((call) => call.arguments), - ).toEqual([["26.5"]]); - await client.close(); - }); - - it("never attaches the download-policy suffix to an undownloadable RuntimeMissingError, even under the never policy", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }); - driver.failOn( - "resolveSpec", - 1, - new RuntimeMissingError("ios", "12.0", { downloadable: false }), - ); - const harness = await createHarness({ clock, downloads: { policy: "never" }, driver }); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("lease.request", { - allowDownload: true, - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "12.0", - platform: "ios", - }); - - expect(response.ok).toBe(false); - expect(response.error).toMatchObject({ code: "RUNTIME_MISSING" }); - expect(response.error?.message).not.toContain("downloads.policy"); - await client.close(); - }); -}); - -describe("DaemonServer lease request mode", () => { - it("grants a slim request on a runtime the driver can slim with mode slim", async () => { - const harness = await createHarness({ slimmableOsVersions: ["26.5"] }); - const client = await createClient(harness.socketPath); - await hello(client); - - const grant = await client.request("lease.request", { - requesterId: "agent-1", - mode: "slim", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - - expect(grant.ok).toBe(true); - expect((grant.payload as { device: { mode: string } }).device.mode).toBe("slim"); - await client.close(); - }); - - it.each([ - ["full", { full: true }], - ["a mode other than slim or full", { mode: "fast" }], - ["the lease mode ADR 0004 retired", { mode: "held" }], - ])("answers a lease.request carrying %s with BAD_REQUEST", async (_label, extra) => { - const harness = await createHarness(); - const client = await createClient(harness.socketPath); - await hello(client); - - const response = await client.request("lease.request", { - requesterId: "agent-1", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - ...extra, - }); - - expect(response.ok).toBe(false); - expect(response.error).toMatchObject({ code: "BAD_REQUEST" }); - await client.close(); - }); }); From e37c8853f884ee1b976b9889ab915d08432e4279 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:22:45 +0100 Subject: [PATCH 33/83] test(drivers): split index.test.ts by topic --- src/drivers/android/index-boot.test.ts | 650 +++++++ src/drivers/android/index-launch.test.ts | 286 +++ .../android/index-profiles-components.test.ts | 415 ++++ .../android/index-screen-capture.test.ts | 284 +++ src/drivers/android/index.test.ts | 1696 +---------------- src/drivers/android/test-index.ts | 25 + 6 files changed, 1712 insertions(+), 1644 deletions(-) create mode 100644 src/drivers/android/index-boot.test.ts create mode 100644 src/drivers/android/index-launch.test.ts create mode 100644 src/drivers/android/index-profiles-components.test.ts create mode 100644 src/drivers/android/index-screen-capture.test.ts create mode 100644 src/drivers/android/test-index.ts diff --git a/src/drivers/android/index-boot.test.ts b/src/drivers/android/index-boot.test.ts new file mode 100644 index 000000000..98e362c16 --- /dev/null +++ b/src/drivers/android/index-boot.test.ts @@ -0,0 +1,650 @@ +import { describe, expect, it, vi } from "vitest"; +import { promiseState } from "../../test-support/promise-state.js"; +import { + FakeClock, + FakeProcessSupervisor, + FakeTcpProbe, + MemoryFilesystem, + ScriptedProcessRunner, + type ProcessResult, + type ProcessRunner, +} from "../../ports/index.js"; +import { AndroidDriver, SdkMissingError } from "./index.js"; +import { + androidFilesystem, + avdDirectory, + binaries, + createdFromPackage, + taggedHarness as taggedHarnessWith, + EmulatorExitingRunner, + instanceId, + simlockHome, + memoryAndroidSourceDatabase, + pixelDevices, + processResult, + adbServerPort, + adbServerPid, + scopedEnv, + provisionedHarness, + createDriver, + recordRunningAdbServer, + EmulatorHost, + RecordingRunner, + consolePortOf, + copyPathExpectation, +} from "./testing.js"; +import { sdk, home, buildDriver } from "./test-index.js"; + +const scopedOptions = { env: scopedEnv } as const; + +const taggedHarness = ( + ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never +) => taggedHarnessWith(buildDriver, ...args); + +/** why: how many times the emulator has been asked whether it finished booting. */ +function bootPolls(runner: ScriptedProcessRunner): number { + return runner.calls.filter((call) => call.args.at(-1) === "sys.boot_completed").length; +} + +describe("AndroidDriver", () => { + it("discovers ANDROID_HOME before the other SDK locations and rejects a missing SDK", async () => { + const filesystem = await androidFilesystem(); + const runner = new EmulatorExitingRunner([]); + + await recordRunningAdbServer(filesystem); + const driver = await AndroidDriver.create({ + clock: new FakeClock(), + database: await memoryAndroidSourceDatabase(), + driverConfig: {}, + env: { + ANDROID_HOME: sdk, + ANDROID_SDK_ROOT: "/ignored-sdk", + }, + filesystem, + homeDirectory: home, + hostAbi: "arm64-v8a", + instanceId, + processRunner: runner, + processSupervisor: new FakeProcessSupervisor([adbServerPid]), + simlockHome, + tcpProbe: new FakeTcpProbe([adbServerPort]), + }); + + expect(driver.sdkPath).toBe(sdk); + expect(driver.deviceRoot).toBe(avdDirectory); + await expect( + AndroidDriver.create({ + clock: new FakeClock(), + database: await memoryAndroidSourceDatabase(), + driverConfig: {}, + env: {}, + filesystem: new MemoryFilesystem(), + homeDirectory: home, + hostAbi: "arm64-v8a", + instanceId, + processRunner: runner, + processSupervisor: new FakeProcessSupervisor(), + simlockHome, + tcpProbe: new FakeTcpProbe(), + }), + ).rejects.toBeInstanceOf(SdkMissingError); + }); + + it("discovers versioned command-line tools before obsolete legacy tools", async () => { + const filesystem = await androidFilesystem(); + const versionedTools = `${sdk}/cmdline-tools/19.0/bin`; + await filesystem.rm(`${sdk}/cmdline-tools/latest`); + await filesystem.mkdirp(versionedTools); + await filesystem.writeFileAtomic(`${versionedTools}/avdmanager`, "binary"); + await filesystem.writeFileAtomic(`${versionedTools}/sdkmanager`, "binary"); + + const legacyTools = `${sdk}/tools/bin`; + await filesystem.mkdirp(legacyTools); + await filesystem.writeFileAtomic(`${legacyTools}/avdmanager`, "binary"); + await filesystem.writeFileAtomic(`${legacyTools}/sdkmanager`, "binary"); + + const runner = new EmulatorExitingRunner([ + processResult(`${versionedTools}/avdmanager`, ["list", "device"], pixelDevices), + ]); + const driver = await createDriver(filesystem, runner); + + await expect(driver.resolveSpec({ model: "Pixel 5", platform: "android" })).resolves.toEqual({ + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + }); + + describe("image tag", () => { + it("resolves a request naming an installed tag to a spec with that tag, and creates the device from that image", async () => { + const { driver, runner } = await taggedHarness([ + ["34", "google_apis", "arm64-v8a"], + ["34", "google_apis_playstore", "arm64-v8a"], + ]); + + const spec = await driver.resolveSpec({ + imageTag: "google_apis_playstore", + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + await driver.provision(spec); + + expect(spec).toEqual({ + physical: false, + imageTag: "google_apis_playstore", + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + expect(createdFromPackage(runner)).toBe( + "system-images;android-34;google_apis_playstore;arm64-v8a", + ); + }); + + it("resolves a request naming no tag to the same google_apis image as before, with no tag on the spec", async () => { + const { driver, runner } = await taggedHarness([ + ["34", "default", "arm64-v8a"], + ["34", "google_apis_playstore", "arm64-v8a"], + ["34", "google_apis", "arm64-v8a"], + ]); + + const spec = await driver.resolveSpec({ model: "Pixel 5", platform: "android" }); + await driver.provision(spec); + + expect(spec).toEqual({ + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + expect(createdFromPackage(runner)).toBe("system-images;android-34;google_apis;arm64-v8a"); + }); + }); + + it("boots concurrent devices on different even ports and skips ports adb shows taken", async () => { + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + const filesystem = await androidFilesystem(); + const host = new EmulatorHost(filesystem); + // why: Another process's emulator holds the first port in the range. This daemon reserved + // nothing for it, so only `adb devices` can show that it is taken. + const elsewhere = await createDriver(filesystem, host.runner(), { ids: ["zero"] }); + await elsewhere.resolveSpec(spec); + const taken = await elsewhere.makeReady(await elsewhere.provision(spec)); + const runner = new RecordingRunner(host.runner()); + const first = await createDriver(filesystem, runner, { ids: ["first"] }); + const second = await createDriver(filesystem, runner, { ids: ["second"] }); + await Promise.all([first.resolveSpec(spec), second.resolveSpec(spec)]); + const [firstDevice, secondDevice] = await Promise.all([ + first.provision(spec), + second.provision(spec), + ]); + + const [firstReady, secondReady] = await Promise.all([ + first.makeReady(firstDevice), + second.makeReady(secondDevice), + ]); + + expect({ + first: host.avdAnswering(firstReady.address), + second: host.avdAnswering(secondReady.address), + taken: host.avdAnswering(taken.address), + }).toEqual({ first: "simlock_first", second: "simlock_second", taken: "simlock_zero" }); + expect( + [taken, firstReady, secondReady].map((device) => consolePortOf(device.address) % 2), + ).toEqual([0, 0, 0]); + // why: Occupancy is read from Simlock's own server, not the machine's shared one: an + // unscoped `adb devices` polls 5037, which reports the user's emulators and none of + // Simlock's, so every console port would read free and collide on the next boot. + const occupancyReads = runner.calls.filter((call) => call.args[0] === "devices"); + expect(occupancyReads.length).toBeGreaterThan(0); + for (const read of occupancyReads) { + expect(read).toEqual({ args: ["devices"], command: binaries.adb, options: scopedOptions }); + } + }); + + // why: Another Simlock instance's emulator is announced only to that instance's adb server, so + // its own `adb devices` lists nothing here; only the machine's TCP ports show it (#257). + it.each([ + ["its console port", 5586], + ["the adb port above its console port", 5587], + ["its gRPC port", 8586], + ])( + "skips a console port when something listens on %s that its own adb server does not list", + async (_, listeningPort) => { + const filesystem = await androidFilesystem(); + const host = new EmulatorHost(filesystem); + const driver = await createDriver(filesystem, host.runner(), { + ids: ["one"], + tcpProbe: new FakeTcpProbe([adbServerPort, listeningPort]), + }); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + await driver.resolveSpec(spec); + + const device = await driver.makeReady(await driver.provision(spec)); + + expect(device.driverData).toMatchObject({ avdName: "simlock", port: 5588 }); + }, + ); + + // why: The gRPC port follows the reserved console port, so two parallel leases never share one. + it("puts the gRPC endpoint at the reserved console port plus 3000", async () => { + const filesystem = await androidFilesystem(); + const host = new EmulatorHost(filesystem); + const launches: (readonly string[])[] = []; + const hostRunner = host.runner(); + const runner: ProcessRunner = { + ...hostRunner, + spawn: (command, args, options) => { + launches.push(args); + return hostRunner.spawn(command, args, options); + }, + }; + const driver = await createDriver(filesystem, runner, { + ids: ["one"], + tcpProbe: new FakeTcpProbe([adbServerPort, 5586]), + }); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + await driver.resolveSpec(spec); + + await driver.makeReady(await driver.provision(spec)); + + expect(launches[0]).toEqual( + expect.arrayContaining(["-port", "5588", "-grpc", "8588", "-grpc-use-token"]), + ); + }); + + it("starts the seed's boots and the copy's boot with -crash-report-mode never and a crash database under the Simlock root, so a stale crash dump asks no one and the user's shared database is never read", async () => { + const filesystem = await androidFilesystem(); + const host = new EmulatorHost(filesystem); + const launches: (readonly string[])[] = []; + const crashDatabases: (string | undefined)[] = []; + const hostRunner = host.runner(); + const runner: ProcessRunner = { + ...hostRunner, + spawn: (command, args, options) => { + launches.push(args); + crashDatabases.push(options?.env?.ANDROID_EMU_CRASH_REPORTING_DATABASE); + return hostRunner.spawn(command, args, options); + }, + }; + const driver = await createDriver(filesystem, runner, { + ids: ["one"], + tcpProbe: new FakeTcpProbe([adbServerPort]), + }); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + await driver.resolveSpec(spec); + + await driver.makeReady(await driver.provision(spec)); + + expect(crashDatabases).toEqual([ + `${avdDirectory}/emu-crash.db`, + `${avdDirectory}/emu-crash.db`, + `${avdDirectory}/emu-crash.db`, + ]); + const crashReportMode = (args: readonly string[]) => { + const flag = args.indexOf("-crash-report-mode"); + return flag === -1 ? undefined : args[flag + 1]; + }; + expect( + launches.map((args) => ({ boot: args.at(-1), crashReportMode: crashReportMode(args) })), + ).toEqual([ + { boot: "-no-snapshot-load", crashReportMode: "never" }, + { boot: "-no-snapshot-load", crashReportMode: "never" }, + { boot: "simlock_boot", crashReportMode: "never" }, + ]); + }); + + // why: emulator 36.1.9's `-help` lists every flag Simlock passes except -crash-report-mode, and it exits at `unknown option: -crash-report-mode` (#1199). + const usage = [ + "Android Emulator usage: emulator [options] [-qemu args]", + " options:", + " -avd use a specific android virtual device", + " -port TCP port that will be used for the console", + " -no-snapshot-save do not auto-save to snapshot on exit: abandon changed state", + " -no-snapshot-load do not auto-start from snapshot: perform a full boot", + " -snapshot name of snapshot within storage file for auto-start and auto-save", + " -grpc TCP ports used for the gRPC bridge", + " -grpc-use-token Use the emulator console token for gRPC authentication.", + "", + ].join("\n"); + + it.each([ + { + probe: "-help does not list it", + help: (): Promise => Promise.resolve({ code: 0, stderr: "", stdout: usage }), + }, + { + probe: "-help fails or times out", + help: (): Promise => Promise.reject(new Error("emulator -help timed out")), + }, + ])( + "starts the seed's boots and the copy's boot without -crash-report-mode when the installed emulator's $probe, keeping every other launch flag (#1297)", + async ({ help }) => { + const filesystem = await androidFilesystem(); + const host = new EmulatorHost(filesystem); + const launches: (readonly string[])[] = []; + const hostRunner = host.runner(); + const runner: ProcessRunner = { + ...hostRunner, + run: (command, args, options) => + command === binaries.emulator && args[0] === "-help" + ? help() + : hostRunner.run(command, args, options), + spawn: (command, args, options) => { + launches.push(args); + return hostRunner.spawn(command, args, options); + }, + }; + const driver = await createDriver(filesystem, runner, { + ids: ["one"], + tcpProbe: new FakeTcpProbe([adbServerPort]), + }); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + await driver.resolveSpec(spec); + + await driver.makeReady(await driver.provision(spec)); + + expect( + launches.map((args) => ({ + boot: args.at(-1), + crashReportMode: args.includes("-crash-report-mode"), + })), + ).toEqual([ + { boot: "-no-snapshot-load", crashReportMode: false }, + { boot: "-no-snapshot-load", crashReportMode: false }, + { boot: "simlock_boot", crashReportMode: false }, + ]); + for (const args of launches) { + expect(args).toEqual( + expect.arrayContaining(["-port", "-no-snapshot-save", "-grpc", "-grpc-use-token"]), + ); + } + }, + ); + + /** + * why: The half of the daemon-stop fix that argv cannot show. `stdio: "ignore"` alone does not + * release the event loop -- the ChildProcess handle does -- so without the `unref` a + * `daemon stop` logged "Daemon stopped" and then sat there for as long as any emulator ran. + * An emulator is meant to outlive the daemon (`DeviceTable#reattachRunningEmulators` adopts it after a + * restart) and is reaped through adb and by pid, never by awaiting its exit. + */ + it("releases the event loop for the emulator it starts, so a stopped daemon can exit", async () => { + const harness = await provisionedHarness(); + + await harness.driver.makeReady(harness.device); + + // why: The launches specifically -- `emulator` is also invoked as a short-lived query + // (`-list-avds` and friends) through `run`, which waits on the child and must stay + // referenced. Only a boot is detached. + const launches = harness.runner.calls + .map((call, index) => ({ call, index })) + .filter(({ call }) => call.command === binaries.emulator && call.args.includes("-avd")); + expect(launches.length, "no emulator was launched").toBeGreaterThan(0); + + for (const { index } of launches) { + expect(harness.runner.handles[index]?.unreffed, `launch at call ${String(index)}`).toBe(true); + } + expect(harness.runner.handles.filter((handle) => handle.unreffed)).toHaveLength( + launches.length, + ); + }); + + /** + * why: #237: a timer the driver leaves armed holds the event loop open, so a daemon that logged + * "Daemon stopped" stays alive until it fires. The shutdown the test asks for sees the + * emulator exit, so it has nothing left to wait for. + */ + it("leaves no timer armed once the shutdown has seen its emulator exit", async () => { + const harness = await provisionedHarness({ + afterwards: [ + copyPathExpectation("emulator-5586"), + processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), + ], + }); + const ready = await harness.driver.makeReady(harness.device); + const launch = harness.runner.calls.findLastIndex( + (call) => call.command === binaries.emulator && call.args.includes("-avd"), + ); + const running = harness.runner.handles[launch]; + expect(running, "no running emulator").toBeDefined(); + + const shutdown = harness.driver.shutdown(ready); + running?.kill("SIGTERM"); + await shutdown; + + expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); + }); + + it("kills an emulator that has not exited 180 s after a shutdown, whatever the boot timeout is", async () => { + const harness = await provisionedHarness({ + afterwards: [ + copyPathExpectation("emulator-5586"), + processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), + ], + bootTimeoutMs: 600_000, + }); + const ready = await harness.driver.makeReady(harness.device); + const launch = harness.runner.calls.findLastIndex( + (call) => call.command === binaries.emulator && call.args.includes("-avd"), + ); + const running = harness.runner.handles[launch]; + if (running === undefined) throw new Error("no running emulator"); + const kill = vi.spyOn(running, "kill"); + + // why: The emulator ignores `emu kill`: only the timeout can end the wait. + harness.runner.ignoresEmuKill = true; + const armedBefore = harness.clock.pendingTimerCount; + const shutdown = harness.driver.shutdown(ready); + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(armedBefore)); + expect(kill).not.toHaveBeenCalled(); + harness.clock.advance(180_000); + await shutdown; + + expect(kill).toHaveBeenCalledWith("SIGKILL"); + expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); + }); + + it("leaves no timer armed when waiting for the emulator to exit fails", async () => { + const harness = await provisionedHarness({ + afterwards: [ + copyPathExpectation("emulator-5586"), + processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), + ], + }); + const ready = await harness.driver.makeReady(harness.device); + const launch = harness.runner.calls.findLastIndex( + (call) => call.command === binaries.emulator && call.args.includes("-avd"), + ); + const running = harness.runner.handles[launch]; + if (running === undefined) throw new Error("no running emulator"); + vi.spyOn(running, "wait").mockRejectedValue(new Error("spawn error")); + + await expect(harness.driver.shutdown(ready)).rejects.toThrow("spawn error"); + + expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); + }); + + it("fails the boot as soon as the emulator exits, naming the emulator's own error line", async () => { + const harness = await provisionedHarness({ bootCompleted: "0\n" }); + // why: Emulator 36.1.9's real output for a flag it does not know (#1199); it then exits 1. + const output = [ + "INFO | Android emulator version 36.1.9.0 (build_id 13823996) (CL:N/A)", + "unknown option: -crash-report-mode", + "please use -help for a list of valid options", + ]; + const originalSpawn = harness.runner.spawn.bind(harness.runner); + vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { + const handle = originalSpawn(command, args, options); + if (command !== binaries.emulator || !args.includes("-avd")) return handle; + handle.kill("SIGKILL"); + // why: whichever way the driver keeps the emulator's output (a log file or a pipe), it gets these lines. + const written = + typeof options?.stdio === "object" + ? harness.filesystem.writeFileAtomic(options.stdio.file, `${output.join("\n")}\n`) + : Promise.resolve(); + async function* lines(): AsyncGenerator { + yield* output; + } + return { + pid: handle.pid, + stderr: lines(), + stdout: lines(), + kill: () => undefined, + unref: () => undefined, + wait: async () => { + await written; + return { code: 1, stderr: "", stdout: `${output.join("\n")}\n` }; + }, + }; + }); + const armedBefore = harness.clock.pendingTimerCount; + + let outcome: unknown; + harness.driver.makeReady(harness.device).then( + () => (outcome = "ready"), + (error: unknown) => (outcome = error), + ); + // why: the clock never moves, so a driver that is still waiting for adb has armed its poll timer. + await vi.waitFor(() => + expect(outcome !== undefined || harness.clock.pendingTimerCount > armedBefore).toBe(true), + ); + + expect(outcome, "still waiting for the boot after the emulator exited").toMatchObject({ + message: expect.stringContaining("unknown option: -crash-report-mode"), + }); + }); + + it("sends the emulator's output to /emulator.log, mode 0600, truncated at each start", async () => { + const harness = await provisionedHarness(); + const logPath = `${avdDirectory}/simlock_one/simlock.avd/emulator.log`; + await harness.filesystem.writeFileAtomic(logPath, "left by an earlier start\n", { + mode: 0o644, + }); + + await harness.driver.makeReady(harness.device); + + const launch = harness.runner.calls.findLast( + (call) => call.command === binaries.emulator && call.args.includes("-avd"), + ); + expect(launch?.options.stdio).toEqual({ file: logPath }); + await expect(harness.filesystem.readFile(logPath)).resolves.toBe(""); + expect((await harness.filesystem.lstat(logPath)).mode & 0o777).toBe(0o600); + }); + + it("an Android boot waits android.bootTimeoutMs, then kills the emulator and fails with BootTimeoutError", async () => { + for (const { key, waitsMs } of [ + { key: undefined, waitsMs: 300_000 }, + { key: 600_000, waitsMs: 600_000 }, + ]) { + const harness = await provisionedHarness({ + bootCompleted: "0\n", + ...(key === undefined ? {} : { bootTimeoutMs: key }), + }); + const kills: string[] = []; + const originalSpawn = harness.runner.spawn.bind(harness.runner); + vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { + const handle = originalSpawn(command, args, options); + const kill = handle.kill.bind(handle); + return { + pid: handle.pid, + stderr: handle.stderr, + stdout: handle.stdout, + kill(signal) { + kills.push(signal ?? "SIGTERM"); + kill(signal); + }, + unref: () => handle.unref(), + wait: () => handle.wait(), + }; + }); + + // why: the script answers one poll; a boot that outlasts minutes keeps being told it is not done. + const scripted = harness.runner.run.bind(harness.runner); + vi.spyOn(harness.runner, "run").mockImplementation((command, args, options) => + args.at(-1) === "sys.boot_completed" + ? Promise.resolve({ code: 0, stderr: "", stdout: "0\n" }) + : scripted(command, args, options), + ); + + const ready = harness.driver.makeReady(harness.device); + const state = promiseState(ready); + const failed = expect(ready).rejects.toMatchObject({ name: "BootTimeoutError" }); + // why: each poll arms the next delay only after the previous one fired, so the clock moves in steps. + for (let elapsed = 0; elapsed < waitsMs - 1; elapsed += 1_000) { + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(Math.min(1_000, waitsMs - 1 - elapsed)); + for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); + } + expect(state.state, `still booting just before ${String(waitsMs)} ms`).toBe("pending"); + expect(kills).toEqual([]); + + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(1); + await failed; + expect(kills).toContain("SIGKILL"); + } + }); + + it("retries adb while the emulator is starting", async () => { + const harness = await provisionedHarness({ initialAdbFailures: 1 }); + const polledBefore = bootPolls(harness.runner); + const ready = harness.driver.makeReady(harness.device); + + await vi.waitFor(() => expect(bootPolls(harness.runner)).toBeGreaterThan(polledBefore)); + harness.clock.advance(2_000); + + await expect(ready).resolves.toMatchObject({ address: "emulator-5586" }); + }); + + it("is not ready until `pm path android` exits 0 and prints a package", async () => { + const harness = await provisionedHarness({ + unsettledPmAnswers: [ + { code: 1, stderr: "cmd: Can't find service: package", stdout: "" }, + { code: 0, stderr: "", stdout: "Error: could not access the Package Manager\n" }, + { code: 1, stderr: "", stdout: "package:/system/framework/framework-res.apk\n" }, + ], + }); + const pmCallsBefore = harness.runner.calls.filter((call) => call.args.includes("pm")).length; + const ready = harness.driver.makeReady(harness.device); + const settled = vi.fn(); + void ready.then(settled, settled); + const pmCalls = () => + harness.runner.calls.filter((call) => call.args.includes("pm")).length - pmCallsBefore; + + for (const asked of [1, 2, 3]) { + await vi.waitFor(() => expect(pmCalls()).toBe(asked)); + expect(settled, `ready after ${asked} unanswered pm`).not.toHaveBeenCalled(); + harness.clock.advance(2_000); + } + + await expect(ready).resolves.toMatchObject({ address: "emulator-5586" }); + }); +}); diff --git a/src/drivers/android/index-launch.test.ts b/src/drivers/android/index-launch.test.ts new file mode 100644 index 000000000..f0caa5caa --- /dev/null +++ b/src/drivers/android/index-launch.test.ts @@ -0,0 +1,286 @@ +import { describe, expect, it } from "vitest"; +import { type Driver } from "../../core/index.js"; +import { MemoryFilesystem } from "../../ports/index.js"; +import { type AndroidDriverDiagnostic, type AndroidEmulatorLaunchOptions } from "./index.js"; +import { + androidFilesystem, + avdDirectory, + binaries, + markWriteExpectation, + EmulatorExitingRunner, + seedEmulatorLine, + pixelDevices, + processResult, + adbServerPort, + scopedEnv, + PHYSICAL_DEVICE, + emulatorLaunchArgs, + provisionedHarness, + createDriver, + copyPathExpectation, +} from "./testing.js"; +import { sdk, home, VIRTUAL_DEVICE, libraryHarness } from "./test-index.js"; + +const seedLaunch = ["-avd", "simlock", ...seedEmulatorLine]; + +/** + * why: Fails every `readFile` of `failingPath` with a non-ENOENT error while `armed`, so a test can + * assert a caller rethrows it instead of treating it as an absent file -- then disarm to read + * the path back and confirm nothing overwrote it in the meantime. + */ +class ReadFailureFilesystem extends MemoryFilesystem { + armed = true; + + constructor( + private readonly failingPath: string, + private readonly error: Error, + freeDiskBytes?: number, + ) { + super(freeDiskBytes); + } + + override async readFile(path: string): Promise { + if (this.armed && path === this.failingPath) { + throw this.error; + } + return super.readFile(path); + } +} + +describe("AndroidDriver", () => { + describe("android.emulator launch options", () => { + const defaults: AndroidEmulatorLaunchOptions = { + audio: true, + bootAnimation: true, + gpu: "auto", + headless: false, + }; + it("launches with the same arguments as before these options existed under the default config", async () => { + for (const emulator of [undefined, defaults]) { + const harness = await provisionedHarness(emulator === undefined ? {} : { emulator }); + + await harness.driver.makeReady(harness.device); + + expect(emulatorLaunchArgs(harness.runner)).toEqual([ + [...seedLaunch, "-no-snapshot-load"], + [...seedLaunch, "-no-snapshot-load"], + [...seedLaunch, "-snapshot", "simlock_boot"], + ]); + } + }); + + it("adds -no-window for headless, -no-audio for audio: false, and -no-boot-anim for bootAnimation: false", async () => { + const flags = ["-no-window", "-no-audio", "-no-boot-anim"]; + const harness = await provisionedHarness({ + emulator: { ...defaults, audio: false, bootAnimation: false, headless: true }, + emulatorFlags: flags, + }); + + await harness.driver.makeReady(harness.device); + + const launches = emulatorLaunchArgs(harness.runner); + expect(launches).toHaveLength(3); + for (const args of launches) { + expect(args.slice(0, 10 + flags.length)).toEqual([...seedLaunch, ...flags]); + } + }); + + it("passes a gpu mode as -gpu , and passes nothing for auto", async () => { + const harness = await provisionedHarness({ + emulator: { ...defaults, gpu: "swiftshader_indirect" }, + emulatorFlags: ["-gpu", "swiftshader_indirect"], + }); + await harness.driver.makeReady(harness.device); + expect(emulatorLaunchArgs(harness.runner)[0]).toEqual([ + ...seedLaunch, + "-gpu", + "swiftshader_indirect", + "-no-snapshot-load", + ]); + + const auto = await provisionedHarness({ emulator: { ...defaults, gpu: "auto" } }); + await auto.driver.makeReady(auto.device); + for (const args of emulatorLaunchArgs(auto.runner)) { + expect(args).not.toContain("-gpu"); + } + }); + }); + + it("deletes only the copy's folder, keeps the seed it came from, and stops nothing for a copy that never booted", async () => { + const harness = await provisionedHarness(); + const callsBefore = harness.runner.calls.length; + + await expect(harness.driver.destroy(harness.device)).resolves.toBeUndefined(); + + expect(harness.runner.calls.slice(callsBefore)).toEqual([]); + expect(await harness.filesystem.exists(harness.copyHome)).toBe(false); + expect(await harness.filesystem.exists(`${harness.seedHome}/simlock.avd`)).toBe(true); + }); + + it("reports conservative cold-boot estimates through the Driver contract", async () => { + const driver: Driver = await createDriver( + await androidFilesystem(), + new EmulatorExitingRunner([]), + ); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + + expect(driver.estimate({ operation: "provision" }, spec)).toBe(150_000); + expect(driver.estimate({ operation: "boot" }, spec)).toBe(70_000); + }); + + it("prices reclaim by the strategy the clean level selects", async () => { + const driver: Driver = await createDriver( + await androidFilesystem(), + new EmulatorExitingRunner([]), + ); + const spec = { + physical: false, + model: "Pixel 5", + osVersion: "34", + platform: "android", + } as const; + + // why: Both measured on an M3 Pro / Pixel 8 / API 35. The gap is real: a `wipe` reclaim defers + // the wipe to the next `makeReady`; a reclaim that also boots the device back to warm + // spends a cold wipe boot in `reclaiming` rather than a shutdown. + expect(driver.estimate({ clean: "standard", operation: "reclaim" }, spec)).toBe(6_000); + expect(driver.estimate({ clean: "full", operation: "reclaim" }, spec)).toBe(32_000); + }); + + describe("avdmanager list device reads", () => { + it("seeds a library provision from the `avdmanager list device` read the catalog made", async () => { + const harness = await libraryHarness({ listing: pixelDevices }); + + await harness.driver.listCatalog(); + await harness.provision("Pixel 9"); + + expect(harness.seedDeviceId()).toBe("pixel_5"); + expect( + harness.runner.calls.filter((call) => call.args.join(" ") === "list device"), + ).toHaveLength(1); + }); + + it.each([ + { + env: { ANDROID_USER_HOME: "/user-home" }, + path: "/user-home/devices.xml", + situation: "ANDROID_USER_HOME", + }, + { env: {}, path: `${home}/.android/devices.xml`, situation: "its home directory" }, + ])( + "hands the catalog its scoped environment, its SDK root, its diagnostics callback and the devices.xml under $situation", + async ({ env, path }) => { + const addOnDevicesXml = `${sdk}/add-ons/addon-acme/devices.xml`; + const filesystem = await androidFilesystem(); + await filesystem.mkdirp(path.slice(0, path.lastIndexOf("/"))); + await filesystem.writeFileAtomic(path, "not xml at all {{{"); + await filesystem.mkdirp(`${sdk}/add-ons/addon-acme`); + await filesystem.writeFileAtomic(addOnDevicesXml, ""); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ]); + const diagnostics: AndroidDriverDiagnostic[] = []; + const driver = await createDriver(filesystem, runner, { + extraEnv: env, + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }); + + await driver.listCatalog(); + await filesystem.writeFileAtomic(addOnDevicesXml, ""); + await driver.listCatalog(); + + const listRuns = runner.calls.filter((call) => call.args.join(" ") === "list device"); + expect(listRuns.map((call) => call.options.env)).toEqual([ + { ...scopedEnv, ...env }, + { ...scopedEnv, ...env }, + ]); + expect(diagnostics).toContainEqual( + expect.objectContaining({ kind: "device-profile-source-unreadable", path }), + ); + }, + ); + }); + + it("answers virtualLeaseEnvironment with simlock's adb port, and leaseEnvironment for a virtual device equals it", async () => { + const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); + + expect(driver.virtualLeaseEnvironment()).toEqual({ + ANDROID_ADB_SERVER_PORT: String(adbServerPort), + }); + expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual(driver.virtualLeaseEnvironment()); + }); + + it("hands a lease holder of a physical device its serial and the default adb server", async () => { + const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); + + expect(driver.leaseEnvironment(PHYSICAL_DEVICE)).toEqual({ + ANDROID_ADB_SERVER_PORT: "5037", + ANDROID_SERIAL: "R58M123ABC", + }); + }); + + it("answers uninstall for the reclaim strategy of a physical device, and refuses one for an emulator, which is never reclaimed", async () => { + const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); + + expect(() => driver.reclaimStrategy(VIRTUAL_DEVICE, { clean: "standard" })).toThrow( + "virtual devices are not reclaimed", + ); + expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "standard" })).toBe("uninstall"); + expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "full" })).toBe("uninstall"); + }); + + it("rewrites the mark on makeReady's early-return branch without duplicating the config.ini line", async () => { + const harness = await provisionedHarness({ + afterwards: [ + // why: Second makeReady call: `state.handle` is still set from the first call, so this takes + // the early-return branch -- it must still wait for readiness and rewrite the mark. + processResult( + binaries.adb, + ["-s", "emulator-5586", "shell", "getprop", "sys.boot_completed"], + "1\n", + ), + processResult( + binaries.adb, + ["-s", "emulator-5586", "shell", "getprop", "init.svc.bootanim"], + "", + ), + copyPathExpectation("emulator-5586"), + processResult( + binaries.adb, + ["-s", "emulator-5586", "shell", "pm", "path", "android"], + "package:/system/framework/framework-res.apk\n", + ), + markWriteExpectation("emulator-5586", "device-4"), + ], + }); + + await harness.driver.makeReady(harness.device); + await harness.driver.makeReady(harness.device); + + const config = await harness.filesystem.readFile(`${harness.copyHome}/simlock.avd/config.ini`); + expect(config.split(/\r?\n/).filter((line) => line.startsWith("simlock.mark="))).toEqual([ + "simlock.mark=device-4", + ]); + expect(config).toContain("hw.ramSize=2048"); + }); + + it("rethrows a non-missing-file config.ini read error from the mark write instead of clobbering it as empty", async () => { + const configPath = `${avdDirectory}/simlock_one/simlock.avd/config.ini`; + const permissionError = Object.assign(new Error("EACCES: permission denied"), { + code: "EACCES", + }); + const failureFilesystem = new ReadFailureFilesystem(configPath, permissionError); + const harness = await provisionedHarness({ filesystem: failureFilesystem }); + + await expect(harness.driver.makeReady(harness.device)).rejects.toBe(permissionError); + + failureFilesystem.armed = false; + await expect(harness.filesystem.readFile(configPath)).resolves.toBe("hw.ramSize=2048\n"); + }); +}); diff --git a/src/drivers/android/index-profiles-components.test.ts b/src/drivers/android/index-profiles-components.test.ts new file mode 100644 index 000000000..90eab99d9 --- /dev/null +++ b/src/drivers/android/index-profiles-components.test.ts @@ -0,0 +1,415 @@ +import { describe, expect, it } from "vitest"; +import { AndroidDriver, AndroidLicenseNotAcceptedError } from "./index.js"; +import { type LibraryDeviceProfile } from "./profiles/index.js"; +import { + androidFilesystem, + binaries, + fitHarness as fitHarnessWith, + seedConfig, + EmulatorExitingRunner, + customDeviceXml, + customFitXml, + type FitImage, + InstallReflectingProcessRunner, + pixelDevices, + processResult, + testLibraryProfile, + twoPixelDevices, + userListing, + writeDevicesXml, + provisionedHarness, + createDriver, +} from "./testing.js"; +import { sdk, home, buildDriver, libraryHarness } from "./test-index.js"; + +const fitHarness = ( + ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never +) => fitHarnessWith(buildDriver, ...args); + +function installFor( + driver: AndroidDriver, + component: string, + options: { onProgress?: (progress: unknown) => void; signal?: AbortSignal } = {}, +) { + return driver.installComponent(component, { + onProgress: options.onProgress ?? (() => undefined), + signal: options.signal ?? new AbortController().signal, + }); +} + +describe("AndroidDriver", () => { + describe("device-profile-hidden advisories", () => { + const hiddenMessage = (xmlName: string, libraryName: string) => + `"${xmlName}" in devices.xml is not used: Simlock's own ${libraryName} profile answers to that name`; + + // why: the fake adb server answers no version, so the physical handler adds its own advisory. + async function hiddenProfileAdvisories(driver: AndroidDriver) { + return (await driver.advisories()).filter( + (advisory) => advisory.code === "device-profile-hidden", + ); + } + + async function advisoriesFor(devicesXml: string | undefined, listing = pixelDevices) { + const filesystem = await androidFilesystem(); + if (devicesXml !== undefined) await writeDevicesXml(filesystem, devicesXml); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], listing), + ]); + const driver = await createDriver(filesystem, runner); + return { advisories: await hiddenProfileAdvisories(driver), runner }; + } + + it("reports `device-profile-hidden` for a `devices.xml` profile named like a library profile, with the message naming both", async () => { + const { advisories } = await advisoriesFor(customDeviceXml("Pixel 9", 4096)); + + expect(advisories).toEqual([ + { code: "device-profile-hidden", message: hiddenMessage("Pixel 9", "Pixel 9") }, + ]); + }); + + it("reports one matching a library profile's other name, in any letter case, naming the library profile's display name", async () => { + const { advisories } = await advisoriesFor( + customDeviceXml("pixel_9", 4096) + customDeviceXml("PIXEL 9", 4096), + ); + + expect(advisories).toEqual([ + { code: "device-profile-hidden", message: hiddenMessage("pixel_9", "Pixel 9") }, + { code: "device-profile-hidden", message: hiddenMessage("PIXEL 9", "Pixel 9") }, + ]); + }); + + it("reports two hidden profiles as two advisories, in `devices.xml` order", async () => { + const { advisories } = await advisoriesFor( + customDeviceXml("Pixel 9", 4096) + customDeviceXml("Pixel 8", 4096), + ); + + expect(advisories.map((advisory) => advisory.message)).toEqual([ + hiddenMessage("Pixel 9", "Pixel 9"), + hiddenMessage("Pixel 8", "Pixel 8"), + ]); + }); + + it("reports nothing for a `devices.xml` profile no library profile answers to", async () => { + const { advisories } = await advisoriesFor(customDeviceXml("My Tablet", 4096)); + + expect(advisories).toEqual([]); + }); + + it("reports nothing for a library name the SDK also lists", async () => { + const sdkListing = `Available devices:\nid: 0 or "pixel_9"\n Name: Pixel 9\n OEM : Google\n`; + + const { advisories } = await advisoriesFor(undefined, sdkListing); + + expect(advisories).toEqual([]); + }); + + it("reports one advisory per profile when `avdmanager list device` also lists it with `OEM : User`, and runs no `avdmanager`", async () => { + const userListing = `Available devices:\nid: 0 or "Pixel 9"\n Name: Pixel 9\n OEM : User\n`; + + const { advisories, runner } = await advisoriesFor( + customDeviceXml("Pixel 9", 4096), + userListing, + ); + + expect(advisories).toEqual([ + { code: "device-profile-hidden", message: hiddenMessage("Pixel 9", "Pixel 9") }, + ]); + expect(runner.calls.map((call) => call.command)).not.toContain(binaries.avdmanager); + }); + + it("reports nothing when `devices.xml` is missing or unreadable", async () => { + const missing = await advisoriesFor(undefined); + const filesystem = await androidFilesystem(); + await filesystem.mkdirp(`${home}/.android`); + await filesystem.writeFileAtomic(`${home}/.android/devices.xml`, "not xml at all {{{"); + const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); + + expect(missing.advisories).toEqual([]); + expect(await hiddenProfileAdvisories(driver)).toEqual([]); + }); + + it("keeps the physical handler's advisories, before the new ones", async () => { + const base = await androidFilesystem(); + await writeDevicesXml(base, customDeviceXml("Pixel 9", 4096)); + let broken = false; + const filesystem = new Proxy(base, { + get(target, property) { + if (property === "stat" && broken) { + return async () => { + throw new Error("EACCES"); + }; + } + const value = Reflect.get(target, property) as unknown; + return typeof value === "function" ? (value as () => unknown).bind(target) : value; + }, + }); + const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); + broken = true; + + const advisories = await driver.advisories(); + + expect(advisories.map((advisory) => advisory.code)).toEqual([ + "adb-protocol-mismatch", + "device-profile-hidden", + ]); + }); + + it("reports no hidden profiles for a driver built with injected `deviceProfileSources`", async () => { + const filesystem = await androidFilesystem(); + await writeDevicesXml(filesystem, customDeviceXml("Pixel 9", 4096)); + const driver = await createDriver(filesystem, new EmulatorExitingRunner([]), { + deviceProfileSources: [], + }); + + expect(await hiddenProfileAdvisories(driver)).toEqual([]); + }); + }); + + describe("device-profile sources", () => { + it("resolves a devices.xml-only model and applies its properties to config.ini", async () => { + const harness = await libraryHarness({ devicesXml: customDeviceXml("Custom A", 4096) }); + + const spec = await harness.resolve("Custom A"); + expect(spec).toEqual({ + physical: false, + model: "Custom A", + osVersion: "35", + platform: "android", + }); + + const device = await harness.driver.provision(spec); + + const config = await harness.filesystem.readFile(harness.configPath); + expect(config).toContain("hw.device.name=Custom A"); + expect(config).toContain("hw.ramSize=4096"); + // why: The avdmanager `-d` seed uses a built-in device only to skip the interactive prompt -- + // it must never leak into the resolved spec or the applied hardware properties. + expect(config).not.toContain("pixel_5"); + expect(device.driverData).toMatchObject({ avdName: "simlock" }); + }); + }); + + describe("library device profiles", () => { + it.each([ + { spelling: "Pixel 9", situation: "avdmanager does not list pixel_9", listing: pixelDevices }, + { spelling: "Pixel 9", situation: "avdmanager lists pixel_9 too", listing: twoPixelDevices }, + { + spelling: "Pixel 9", + situation: 'devices.xml defines a "Pixel 9"', + listing: pixelDevices, + devicesXml: customDeviceXml("Pixel 9", 4096), + ramSize: "2G", + }, + { spelling: "pixel_9", situation: "spelled as its avdmanager id", listing: pixelDevices }, + { spelling: "PIXEL 9", situation: "spelled in upper case", listing: pixelDevices }, + ])( + 'resolves "$spelling" to the library Pixel 9 when $situation', + async ({ spelling, listing, devicesXml, ramSize }) => { + const harness = await libraryHarness({ listing, ...(devicesXml ? { devicesXml } : {}) }); + + const spec = await harness.resolve(spelling); + await harness.driver.provision(spec); + + expect(spec).toEqual({ + model: "Pixel 9", + osVersion: "35", + physical: false, + platform: "android", + }); + const config = await harness.config(); + expect(config.get("hw.lcd.height")).toBe("2424"); + expect(config.get("hw.lcd.density")).toBe("420"); + if (ramSize !== undefined) expect(config.get("hw.ramSize")).toBe(ramSize); + }, + ); + + it("resolves a model the library does not have through the SDK, then devices.xml, as today", async () => { + const sdkHarness = await libraryHarness({ + devicesXml: customDeviceXml("Custom A", 4096), + listing: pixelDevices, + }); + const sdkSpec = await sdkHarness.resolve("Pixel 5"); + await sdkHarness.driver.provision(sdkSpec); + + expect(sdkSpec.model).toBe("Pixel 5"); + expect(sdkHarness.seedDeviceId()).toBe("pixel_5"); + expect((await sdkHarness.config()).has("hw.lcd.height")).toBe(false); + + const userHarness = await libraryHarness({ + devicesXml: customDeviceXml("Custom A", 4096), + listing: pixelDevices, + }); + const userSpec = await userHarness.resolve("Custom A"); + await userHarness.driver.provision(userSpec); + + expect(userSpec.model).toBe("Custom A"); + expect((await userHarness.config()).get("hw.ramSize")).toBe("4096"); + }); + + it("captures the config hash after the library's keys are written", async () => { + const hashOf = async (profile: LibraryDeviceProfile, seed: string) => { + const harness = await libraryHarness({ profiles: [profile], seed }); + const device = await harness.provision("Test Phone"); + return (device.driverData as { configHash: string }).configHash; + }; + const ram = (value: string) => + testLibraryProfile({ hardware: { "hw.lcd.height": "100", "hw.ramSize": value } }); + + const small = await hashOf(ram("2G"), seedConfig([])); + const large = await hashOf(ram("4G"), seedConfig([])); + const largeOverOtherSeed = await hashOf(ram("4G"), seedConfig(["hw.ramSize=512M"])); + + expect(small).not.toBe(large); + expect(largeOverOtherSeed).toBe(large); + }); + + it("refuses NUL in a `simlock.mark` value, through the one shared check", async () => { + const markedConfig = (harness: Awaited>) => + harness.filesystem.readFile(`${harness.copyHome}/simlock.avd/config.ini`); + const accepted = await provisionedHarness({ + firstMarkToken: "tok-en", + ids: ["one", "two", "tok-en"], + }); + await accepted.driver.makeReady(accepted.device); + expect(await markedConfig(accepted)).toContain("simlock.mark=tok-en"); + + const refused = await provisionedHarness({ + firstMarkToken: "tok\u0000en", + ids: ["one", "two", "tok\u0000en"], + }); + + await expect(refused.driver.makeReady(refused.device)).rejects.toThrow(/Refusing to merge/); + expect(await markedConfig(refused)).not.toContain("simlock.mark="); + }); + }); + + describe("Android SDK license handling", () => { + const licenseNotAcceptedOutput = + "Warning: License for package Android SDK Platform 35 not accepted.\n\n" + + "1 package(s) were skipped due to license issues. Please accept the license(s) and try " + + "again.\nTo resolve, run: sdkmanager --licenses\n"; + + it("accepts licenses through piped confirmation and retries the install once when the flag is on", async () => { + const filesystem = await androidFilesystem(); + const runner = new InstallReflectingProcessRunner( + [ + { + match: { + args: ["--install", "system-images;android-35;google_apis;arm64-v8a"], + command: binaries.sdkmanager, + }, + result: { code: 1, stderr: "", stdout: licenseNotAcceptedOutput }, + }, + processResult(binaries.sdkmanager, ["--licenses"], "All licenses accepted.\n"), + processResult(binaries.sdkmanager, [ + "--install", + "system-images;android-35;google_apis;arm64-v8a", + ]), + ], + filesystem, + ); + const driver = await createDriver(filesystem, runner, { acceptAndroidLicenses: true }); + + await expect(installFor(driver, "35")).resolves.toMatchObject({ + outcome: "installed", + version: "35", + }); + + const licensesCall = runner.calls.find( + (call) => call.command === binaries.sdkmanager && call.args[0] === "--licenses", + ); + expect(licensesCall?.options.input).toBe("y\n".repeat(100)); + expect(runner.calls.filter((call) => call.args[0] === "--install")).toHaveLength(2); + }); + }); + + describe("component install", () => { + it("refuses an unaccepted licence with AndroidLicenseNotAcceptedError, and runs no sdkmanager --licenses, when the config does not accept them", async () => { + const filesystem = await androidFilesystem(); + const runner = new EmulatorExitingRunner([ + { + match: { + args: ["--install", "system-images;android-35;google_apis;arm64-v8a"], + command: binaries.sdkmanager, + }, + result: { + code: 1, + stderr: "", + stdout: "Warning: License for package Android SDK Platform 35 not accepted.\n", + }, + }, + ]); + const driver = await createDriver(filesystem, runner); + + await expect(installFor(driver, "35")).rejects.toBeInstanceOf(AndroidLicenseNotAcceptedError); + expect(runner.calls.some((call) => call.args[0] === "--licenses")).toBe(false); + }); + + it("reports the revisions of the tool packages under the SDK root it was given", async () => { + const filesystem = await androidFilesystem(); + await filesystem.writeFileAtomic( + `${sdk}/emulator/source.properties`, + "Pkg.Revision=35.4.9\n", + ); + const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); + + await expect(driver.toolVersions()).resolves.toEqual([ + { name: "emulator", version: "35.4.9" }, + ]); + }); + + it("states 2 GiB on the SDK root as the footprint of one install", async () => { + const filesystem = await androidFilesystem(); + const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); + + expect(driver.componentFootprint).toEqual({ bytes: 2 * 1024 ** 3, path: sdk }); + }); + }); +}); + +/** + * why: A driver whose adb server is already running and recorded, so `create` adopts it: that is + * the one start-up path that spawns nothing, which keeps the scripted process expectations + * in every test about the behaviour the test is actually for. The start, reap, and refusal + * paths are covered against the supervisor itself in `adb-server.test.ts`, and end to end + * in this file's own `AndroidDriver.create` block. + */ +describe("AndroidDriver defaultModels", () => { + it("lists the built-in Android preference list per class, newest first, with phone starting at Pixel 9 and tablet holding Medium Tablet", async () => { + const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); + + expect(driver.defaultModels).toEqual({ + auto: ["Automotive (1080p landscape)", "Automotive (1024p landscape)"], + desktop: ["Medium Desktop", "Large Desktop"], + phone: ["Pixel 9", "Pixel 8", "Pixel 7", "Pixel 6", "Medium Phone"], + tablet: ["Medium Tablet"], + tv: ["Television (1080p)", "Television (4K)"], + watch: ["Wear OS Large Round", "Wear OS Small Round"], + }); + }); +}); + +describe("AndroidDriver custom profile fit from devices.xml", () => { + it("resolveSpec with --os 30 and provision refuse a custom profile whose api-level is 31-", async () => { + const images: readonly FitImage[] = [ + ["30", "google_apis", "arm64-v8a"], + ["31", "google_apis", "arm64-v8a"], + ]; + const { createdPackage, driver, refusal, resolve } = await fitHarness(images, { + devicesXml: customFitXml({ apiLevel: "31-", name: "My Phone" }), + listing: userListing(["My Phone"]), + }); + await expect(resolve("My Phone", "31")).resolves.toMatchObject({ osVersion: "31" }); + + expect(await refusal("My Phone", "30")).toMatchObject({ name: "RuntimeMissingError" }); + await expect( + driver.provision({ + model: "My Phone", + osVersion: "30", + physical: false, + platform: "android", + }), + ).rejects.toMatchObject({ name: "RuntimeMissingError" }); + expect(createdPackage()).toBeUndefined(); + }); +}); diff --git a/src/drivers/android/index-screen-capture.test.ts b/src/drivers/android/index-screen-capture.test.ts new file mode 100644 index 000000000..40743bcea --- /dev/null +++ b/src/drivers/android/index-screen-capture.test.ts @@ -0,0 +1,284 @@ +import { describe, expect, it } from "vitest"; +import { + FakeClock, + FakeTcpProbe, + FakeTcpStreamConnector, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, +} from "../../ports/index.js"; +import { + androidFilesystem, + binaries, + EmulatorExitingRunner, + adbServerPort, + scopedEnv, + createDriver, + emptyLines, +} from "./testing.js"; +import { sdk } from "./test-index.js"; + +describe("AndroidDriver screen capture", () => { + const serverPath = "/simlock/vendor/scrcpy-server-v5.0.1"; + const running = { + address: "emulator-5586", + deviceId: "simlock_one", + driverData: { avdName: "simlock_one", configHash: "h", port: 5586, serial: "emulator-5586" }, + physical: false, + } as const; + const settings = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; + + it("is offered, with the device input, only when both the server path and a TCP stream connector are given", async () => { + const both = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + expect(both.screenCapture).toBeDefined(); + expect(both.input).toBeDefined(); + const noPath = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + expect(noPath.screenCapture).toBeUndefined(); + expect(noPath.input).toBeUndefined(); + const noConnector = await createDriver( + await androidFilesystem(), + new EmulatorExitingRunner([]), + { + scrcpyServerPath: serverPath, + }, + ); + expect(noConnector.screenCapture).toBeUndefined(); + expect(noConnector.input).toBeUndefined(); + }); + + it("pushes the server with this driver's adb, its scoped environment and the device's serial", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp("/simlock/vendor"); + await filesystem.writeFileAtomic(serverPath, "jar"); + const runner = new EmulatorExitingRunner([ + { + match: { + args: ["-s", "emulator-5586", "push", serverPath, /simlock-scrcpy-/], + command: binaries.adb, + }, + result: { code: 1, stderr: "error: closed", stdout: "" }, + }, + { + match: { + args: ["-s", "emulator-5586", "shell", "rm", "-f", /simlock-scrcpy-/], + command: binaries.adb, + }, + }, + ]); + const driver = await createDriver(filesystem, runner, { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + await expect( + driver.screenCapture?.open(running, settings, new AbortController().signal), + ).rejects.toThrow(/error: closed/); + expect(runner.calls[0]?.options.env).toEqual(scopedEnv); + expect(runner.calls[1]?.options.env).toEqual(scopedEnv); + }); + + it("refuses an emulator that is not running, before running anything", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp("/simlock/vendor"); + await filesystem.writeFileAtomic(serverPath, "jar"); + const runner = new EmulatorExitingRunner([]); + const driver = await createDriver(filesystem, runner, { + scrcpyServerPath: serverPath, + tcpStreamConnector: new FakeTcpStreamConnector(), + }); + const stopped = { ...running, driverData: { ...running.driverData, port: 0, serial: "" } }; + await expect( + driver.screenCapture?.open(stopped, settings, new AbortController().signal), + ).rejects.toThrow(/not running/); + expect(runner.calls).toEqual([]); + }); + + it("routes each device's capture to its own adb server: a phone's commands run adb -P 5037 and its socket connects to 5037, an emulator's use Simlock's port", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp("/simlock/vendor"); + await filesystem.writeFileAtomic(serverPath, "jar"); + const phone = { + address: "R58M123ABC", + deviceId: "R58M123ABC", + driverData: {}, + enrolledApps: [], + physical: true, + } as const; + const clock = new FakeClock(1_000); + // why: the shared server answers the version check with Simlock's own protocol, so the phone's + // target is open and its commands are expected to go out. + const probe = new FakeTcpProbe([5038, 5037]); + probe.replyWith("OKAY00040029"); + const runner = new CaptureRoutingRunner(); + const tcp = new FakeTcpStreamConnector(); + const driver = await createDriver(filesystem, runner, { + clock, + scrcpyServerPath: serverPath, + tcpProbe: probe, + tcpStreamConnector: tcp, + }); + + // why: each open is stopped while it waits on its socket's first reply — which is exactly the + // moment this test owns: every command has gone out through the device's own server, the socket + // has been aimed at the device, and a stop runs nothing further. + const phoneStop = new AbortController(); + const phoneOpening = driver.screenCapture?.open(phone, settings, phoneStop.signal); + if (phoneOpening === undefined) throw new Error("the driver offers no screen capture"); + // why: the rejection is asserted below; marking it handled here keeps a rejection that arrives + // while this test is still reading the socket from being reported as unhandled. + phoneOpening.catch(() => undefined); + await waitForSocket(clock, tcp, 1); + expect(tcp.connections[0]?.port).toBe(5037); + expect(tcp.connections[0]?.peer.writtenText()).toContain("host:transport:R58M123ABC"); + phoneStop.abort(); + await expect(driveOpen(clock, phoneOpening)).rejects.toThrow(/closed the connection/); + + const emulatorStop = new AbortController(); + const emulatorOpening = driver.screenCapture?.open(running, settings, emulatorStop.signal); + if (emulatorOpening === undefined) throw new Error("the driver offers no screen capture"); + emulatorOpening.catch(() => undefined); + await waitForSocket(clock, tcp, 2); + expect(tcp.connections[1]?.port).toBe(adbServerPort); + expect(tcp.connections[1]?.peer.writtenText()).toContain("host:transport:emulator-5586"); + emulatorStop.abort(); + await expect(driveOpen(clock, emulatorOpening)).rejects.toThrow(/closed the connection/); + + // why: every command the phone's session ran aims at the shared server — `adb -P 5037` with + // that server's env, Simlock's port in it and the three keys that could aim adb elsewhere + // removed — and every command the emulator's session ran carries this driver's scoped env, + // which points at Simlock's own port. Which commands a phone's session runs (its screen read + // included) is the capture's and the handler's own claim, proved in their own files. + const commands = runner.calls.filter((call) => call.args[0] !== "version"); + const phoneCommands = commands.filter((call) => call.args[0] === "-P"); + const emulatorCommands = commands.filter((call) => call.args[0] === "-s"); + expect(phoneCommands.length).toBeGreaterThanOrEqual(3); + for (const call of phoneCommands) { + expect(call.args.slice(0, 3)).toEqual(["-P", "5037", "-s"]); + expect(call.args[3]).toBe("R58M123ABC"); + expect(call.options.env).toEqual({ + ANDROID_ADB_SERVER_PORT: "5037", + ANDROID_HOME: sdk, + }); + } + expect( + phoneCommands.map((call) => + call.args[4] === "push" + ? "push" + : call.args[5] === "rm" + ? "rm" + : call.args[5]?.startsWith("CLASSPATH=") + ? "server" + : String(call.args[4]), + ), + ).toEqual(expect.arrayContaining(["push", "server", "rm"])); + expect(emulatorCommands.map((call) => call.args.slice(0, 3))).toEqual([ + ["-s", "emulator-5586", "push"], + ["-s", "emulator-5586", "shell"], + ["-s", "emulator-5586", "shell"], + ]); + for (const call of emulatorCommands) { + expect(call.options.env).toEqual(scopedEnv); + } + }); +}); + +// why: flushes work until the n-th raw connection has been made, advancing the fake clock so the +// capture's own bounds cannot strand it; an open that died first fails on a named assertion. +async function waitForSocket( + clock: FakeClock, + tcp: FakeTcpStreamConnector, + count: number, +): Promise { + for (let spent = 0; spent < 60_000 && tcp.connections.length < count; spent += 50) { + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + clock.advance(50); + } + expect(tcp.connections.length, "the capture reached its socket").toBe(count); +} + +// why: awaits an open with the fake clock turning, so its own 15 s bound can fire while the test waits. +async function driveOpen(clock: FakeClock, opening: Promise): Promise { + let done = false; + void opening.then( + () => (done = true), + () => (done = true), + ); + for (let spent = 0; spent < 60_000 && !done; spent += 50) { + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + clock.advance(50); + } + return await opening; +} + +/** + * why: answers every adb call a capture makes by what it is rather than by its place in a queue, so + * the routing this test reads is proven the same way whether or not the capture reads the phone's + * screen state first -- the server's shell hangs until the capture's own close kills it, as a live + * scrcpy server does. + */ +class CaptureRoutingRunner implements ProcessRunner { + readonly calls: { readonly args: readonly string[]; readonly options: ProcessRunOptions }[] = []; + #nextPid = 1; + + run( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): Promise { + return this.spawn(command, args, options).wait(); + } + + spawn(_command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { + this.calls.push({ args, options }); + const rest = args[0] === "-P" ? args.slice(2) : [...args]; + const line = rest.join(" "); + if (line === "version") { + return endedHandle(this.#nextPid++, "Android Debug Bridge version 1.0.41\nVersion 35.0.2\n"); + } + if (line.includes("dumpsys power")) { + return endedHandle(this.#nextPid++, " mWakefulness=Awake\n"); + } + if (/shell CLASSPATH=\S*app_process/.test(line)) { + return hangingHandle(this.#nextPid++); + } + return endedHandle(this.#nextPid++, ""); + } + + spawnStreaming(..._args: Parameters): never { + throw new Error("a screen capture spawns no streaming process"); + } +} + +function endedHandle(pid: number, stdout: string): ProcessHandle { + return { + kill: () => undefined, + pid, + stderr: emptyLines(), + stdout: emptyLines(), + unref: () => undefined, + wait: async () => ({ code: 0, stderr: "", stdout }), + }; +} + +function hangingHandle(pid: number): ProcessHandle { + let settle: (() => void) | undefined; + const ended = new Promise((resolve) => (settle = resolve)); + return { + kill: () => settle?.(), + pid, + stderr: emptyLines(), + stdout: emptyLines(), + unref: () => undefined, + wait: async () => { + await ended; + return { code: null, stderr: "", stdout: "" }; + }, + }; +} diff --git a/src/drivers/android/index.test.ts b/src/drivers/android/index.test.ts index 15914288c..aafcae338 100644 --- a/src/drivers/android/index.test.ts +++ b/src/drivers/android/index.test.ts @@ -1,1536 +1,70 @@ import { describe, expect, it, vi } from "vitest"; -import { promiseState } from "../../test-support/promise-state.js"; import { ComponentInUseError, ComponentNotOwnedError, OWNED_ROOT_MARKER_FILE, OwnedRootError, - type Driver, } from "../../core/index.js"; +import { FakeClock, FakeProcessSupervisor, FakeTcpProbe } from "../../ports/index.js"; +import { AdbServerUnavailableError, AndroidDriver } from "./index.js"; import { - FakeClock, - FakeProcessSupervisor, - FakeTcpProbe, - FakeTcpStreamConnector, - MemoryFilesystem, - ScriptedProcessRunner, - type ProcessHandle, - type ProcessResult, - type ProcessRunner, - type ProcessRunOptions, -} from "../../ports/index.js"; -import { - AdbServerUnavailableError, - AndroidDriver, - AndroidLicenseNotAcceptedError, - SdkMissingError, - type AndroidDriverDiagnostic, - type AndroidEmulatorLaunchOptions, -} from "./index.js"; -import { type LibraryDeviceProfile } from "./profiles/index.js"; -import { - ANDROID_SDK_ROOT, - androidFilesystem, - avdDirectory, - binaries, - type BuildProvisioner, - createdFromPackage, - fitHarness as fitHarnessWith, - libraryHarness as libraryHarnessWith, - markWriteExpectation, - seedConfig, - taggedHarness as taggedHarnessWith, - EmulatorExitingRunner, - instanceId, - seedEmulatorLine, - simlockHome, - customDeviceXml, - customFitXml, - type FitImage, - InstallReflectingProcessRunner, - memoryAndroidSourceDatabase, - pixelDevices, - processResult, - testLibraryProfile, - twoPixelDevices, - userAvd, - userListing, - writeDevicesXml, - adbServerPort, - adbServerPid, - adbRecordPath, - scopedEnv, - PHYSICAL_DEVICE, - emulatorLaunchArgs, - provisionedHarness, - createDriver, - recordRunningAdbServer, - EmulatorHost, - RecordingRunner, - consolePortOf, - ok, - emptyLines, - copyPathExpectation, - WorkingDirectoryFilesystem, - ANDROID_HOME_DIRECTORY, -} from "./testing.js"; - -const sdk = ANDROID_SDK_ROOT; -const home = ANDROID_HOME_DIRECTORY; - -const scopedOptions = { env: scopedEnv } as const; - -const VIRTUAL_DEVICE = { - address: "", - deviceId: "device", - driverData: {}, - physical: false, -} as const; - -/** why: the harnesses the seed tests share with `seeds.test.ts` build whatever makes the seed; here it is the whole driver. */ -const buildDriver: BuildProvisioner = (filesystem, processRunner, options) => - createDriver(filesystem, processRunner, options); - -const fitHarness = ( - ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never -) => fitHarnessWith(buildDriver, ...args); - -const libraryHarness = (options?: Parameters[1]) => - libraryHarnessWith(buildDriver, options); - -const taggedHarness = ( - ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never -) => taggedHarnessWith(buildDriver, ...args); - -describe("AndroidDriver", () => { - it("discovers ANDROID_HOME before the other SDK locations and rejects a missing SDK", async () => { - const filesystem = await androidFilesystem(); - const runner = new EmulatorExitingRunner([]); - - await recordRunningAdbServer(filesystem); - const driver = await AndroidDriver.create({ - clock: new FakeClock(), - database: await memoryAndroidSourceDatabase(), - driverConfig: {}, - env: { - ANDROID_HOME: sdk, - ANDROID_SDK_ROOT: "/ignored-sdk", - }, - filesystem, - homeDirectory: home, - hostAbi: "arm64-v8a", - instanceId, - processRunner: runner, - processSupervisor: new FakeProcessSupervisor([adbServerPid]), - simlockHome, - tcpProbe: new FakeTcpProbe([adbServerPort]), - }); - - expect(driver.sdkPath).toBe(sdk); - expect(driver.deviceRoot).toBe(avdDirectory); - await expect( - AndroidDriver.create({ - clock: new FakeClock(), - database: await memoryAndroidSourceDatabase(), - driverConfig: {}, - env: {}, - filesystem: new MemoryFilesystem(), - homeDirectory: home, - hostAbi: "arm64-v8a", - instanceId, - processRunner: runner, - processSupervisor: new FakeProcessSupervisor(), - simlockHome, - tcpProbe: new FakeTcpProbe(), - }), - ).rejects.toBeInstanceOf(SdkMissingError); - }); - - it("discovers versioned command-line tools before obsolete legacy tools", async () => { - const filesystem = await androidFilesystem(); - const versionedTools = `${sdk}/cmdline-tools/19.0/bin`; - await filesystem.rm(`${sdk}/cmdline-tools/latest`); - await filesystem.mkdirp(versionedTools); - await filesystem.writeFileAtomic(`${versionedTools}/avdmanager`, "binary"); - await filesystem.writeFileAtomic(`${versionedTools}/sdkmanager`, "binary"); - - const legacyTools = `${sdk}/tools/bin`; - await filesystem.mkdirp(legacyTools); - await filesystem.writeFileAtomic(`${legacyTools}/avdmanager`, "binary"); - await filesystem.writeFileAtomic(`${legacyTools}/sdkmanager`, "binary"); - - const runner = new EmulatorExitingRunner([ - processResult(`${versionedTools}/avdmanager`, ["list", "device"], pixelDevices), - ]); - const driver = await createDriver(filesystem, runner); - - await expect(driver.resolveSpec({ model: "Pixel 5", platform: "android" })).resolves.toEqual({ - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - }); - - describe("image tag", () => { - it("resolves a request naming an installed tag to a spec with that tag, and creates the device from that image", async () => { - const { driver, runner } = await taggedHarness([ - ["34", "google_apis", "arm64-v8a"], - ["34", "google_apis_playstore", "arm64-v8a"], - ]); - - const spec = await driver.resolveSpec({ - imageTag: "google_apis_playstore", - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - await driver.provision(spec); - - expect(spec).toEqual({ - physical: false, - imageTag: "google_apis_playstore", - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - expect(createdFromPackage(runner)).toBe( - "system-images;android-34;google_apis_playstore;arm64-v8a", - ); - }); - - it("resolves a request naming no tag to the same google_apis image as before, with no tag on the spec", async () => { - const { driver, runner } = await taggedHarness([ - ["34", "default", "arm64-v8a"], - ["34", "google_apis_playstore", "arm64-v8a"], - ["34", "google_apis", "arm64-v8a"], - ]); - - const spec = await driver.resolveSpec({ model: "Pixel 5", platform: "android" }); - await driver.provision(spec); - - expect(spec).toEqual({ - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - expect(createdFromPackage(runner)).toBe("system-images;android-34;google_apis;arm64-v8a"); - }); - }); - - it("boots concurrent devices on different even ports and skips ports adb shows taken", async () => { - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - const filesystem = await androidFilesystem(); - const host = new EmulatorHost(filesystem); - // why: Another process's emulator holds the first port in the range. This daemon reserved - // nothing for it, so only `adb devices` can show that it is taken. - const elsewhere = await createDriver(filesystem, host.runner(), { ids: ["zero"] }); - await elsewhere.resolveSpec(spec); - const taken = await elsewhere.makeReady(await elsewhere.provision(spec)); - const runner = new RecordingRunner(host.runner()); - const first = await createDriver(filesystem, runner, { ids: ["first"] }); - const second = await createDriver(filesystem, runner, { ids: ["second"] }); - await Promise.all([first.resolveSpec(spec), second.resolveSpec(spec)]); - const [firstDevice, secondDevice] = await Promise.all([ - first.provision(spec), - second.provision(spec), - ]); - - const [firstReady, secondReady] = await Promise.all([ - first.makeReady(firstDevice), - second.makeReady(secondDevice), - ]); - - expect({ - first: host.avdAnswering(firstReady.address), - second: host.avdAnswering(secondReady.address), - taken: host.avdAnswering(taken.address), - }).toEqual({ first: "simlock_first", second: "simlock_second", taken: "simlock_zero" }); - expect( - [taken, firstReady, secondReady].map((device) => consolePortOf(device.address) % 2), - ).toEqual([0, 0, 0]); - // why: Occupancy is read from Simlock's own server, not the machine's shared one: an - // unscoped `adb devices` polls 5037, which reports the user's emulators and none of - // Simlock's, so every console port would read free and collide on the next boot. - const occupancyReads = runner.calls.filter((call) => call.args[0] === "devices"); - expect(occupancyReads.length).toBeGreaterThan(0); - for (const read of occupancyReads) { - expect(read).toEqual({ args: ["devices"], command: binaries.adb, options: scopedOptions }); - } - }); - - // why: Another Simlock instance's emulator is announced only to that instance's adb server, so - // its own `adb devices` lists nothing here; only the machine's TCP ports show it (#257). - it.each([ - ["its console port", 5586], - ["the adb port above its console port", 5587], - ["its gRPC port", 8586], - ])( - "skips a console port when something listens on %s that its own adb server does not list", - async (_, listeningPort) => { - const filesystem = await androidFilesystem(); - const host = new EmulatorHost(filesystem); - const driver = await createDriver(filesystem, host.runner(), { - ids: ["one"], - tcpProbe: new FakeTcpProbe([adbServerPort, listeningPort]), - }); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - await driver.resolveSpec(spec); - - const device = await driver.makeReady(await driver.provision(spec)); - - expect(device.driverData).toMatchObject({ avdName: "simlock", port: 5588 }); - }, - ); - - // why: The gRPC port follows the reserved console port, so two parallel leases never share one. - it("puts the gRPC endpoint at the reserved console port plus 3000", async () => { - const filesystem = await androidFilesystem(); - const host = new EmulatorHost(filesystem); - const launches: (readonly string[])[] = []; - const hostRunner = host.runner(); - const runner: ProcessRunner = { - ...hostRunner, - spawn: (command, args, options) => { - launches.push(args); - return hostRunner.spawn(command, args, options); - }, - }; - const driver = await createDriver(filesystem, runner, { - ids: ["one"], - tcpProbe: new FakeTcpProbe([adbServerPort, 5586]), - }); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - await driver.resolveSpec(spec); - - await driver.makeReady(await driver.provision(spec)); - - expect(launches[0]).toEqual( - expect.arrayContaining(["-port", "5588", "-grpc", "8588", "-grpc-use-token"]), - ); - }); - - it("starts the seed's boots and the copy's boot with -crash-report-mode never and a crash database under the Simlock root, so a stale crash dump asks no one and the user's shared database is never read", async () => { - const filesystem = await androidFilesystem(); - const host = new EmulatorHost(filesystem); - const launches: (readonly string[])[] = []; - const crashDatabases: (string | undefined)[] = []; - const hostRunner = host.runner(); - const runner: ProcessRunner = { - ...hostRunner, - spawn: (command, args, options) => { - launches.push(args); - crashDatabases.push(options?.env?.ANDROID_EMU_CRASH_REPORTING_DATABASE); - return hostRunner.spawn(command, args, options); - }, - }; - const driver = await createDriver(filesystem, runner, { - ids: ["one"], - tcpProbe: new FakeTcpProbe([adbServerPort]), - }); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - await driver.resolveSpec(spec); - - await driver.makeReady(await driver.provision(spec)); - - expect(crashDatabases).toEqual([ - `${avdDirectory}/emu-crash.db`, - `${avdDirectory}/emu-crash.db`, - `${avdDirectory}/emu-crash.db`, - ]); - const crashReportMode = (args: readonly string[]) => { - const flag = args.indexOf("-crash-report-mode"); - return flag === -1 ? undefined : args[flag + 1]; - }; - expect( - launches.map((args) => ({ boot: args.at(-1), crashReportMode: crashReportMode(args) })), - ).toEqual([ - { boot: "-no-snapshot-load", crashReportMode: "never" }, - { boot: "-no-snapshot-load", crashReportMode: "never" }, - { boot: "simlock_boot", crashReportMode: "never" }, - ]); - }); - - // why: emulator 36.1.9's `-help` lists every flag Simlock passes except -crash-report-mode, and it exits at `unknown option: -crash-report-mode` (#1199). - const usage = [ - "Android Emulator usage: emulator [options] [-qemu args]", - " options:", - " -avd use a specific android virtual device", - " -port TCP port that will be used for the console", - " -no-snapshot-save do not auto-save to snapshot on exit: abandon changed state", - " -no-snapshot-load do not auto-start from snapshot: perform a full boot", - " -snapshot name of snapshot within storage file for auto-start and auto-save", - " -grpc TCP ports used for the gRPC bridge", - " -grpc-use-token Use the emulator console token for gRPC authentication.", - "", - ].join("\n"); - - it.each([ - { - probe: "-help does not list it", - help: (): Promise => Promise.resolve({ code: 0, stderr: "", stdout: usage }), - }, - { - probe: "-help fails or times out", - help: (): Promise => Promise.reject(new Error("emulator -help timed out")), - }, - ])( - "starts the seed's boots and the copy's boot without -crash-report-mode when the installed emulator's $probe, keeping every other launch flag (#1297)", - async ({ help }) => { - const filesystem = await androidFilesystem(); - const host = new EmulatorHost(filesystem); - const launches: (readonly string[])[] = []; - const hostRunner = host.runner(); - const runner: ProcessRunner = { - ...hostRunner, - run: (command, args, options) => - command === binaries.emulator && args[0] === "-help" - ? help() - : hostRunner.run(command, args, options), - spawn: (command, args, options) => { - launches.push(args); - return hostRunner.spawn(command, args, options); - }, - }; - const driver = await createDriver(filesystem, runner, { - ids: ["one"], - tcpProbe: new FakeTcpProbe([adbServerPort]), - }); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - await driver.resolveSpec(spec); - - await driver.makeReady(await driver.provision(spec)); - - expect( - launches.map((args) => ({ - boot: args.at(-1), - crashReportMode: args.includes("-crash-report-mode"), - })), - ).toEqual([ - { boot: "-no-snapshot-load", crashReportMode: false }, - { boot: "-no-snapshot-load", crashReportMode: false }, - { boot: "simlock_boot", crashReportMode: false }, - ]); - for (const args of launches) { - expect(args).toEqual( - expect.arrayContaining(["-port", "-no-snapshot-save", "-grpc", "-grpc-use-token"]), - ); - } - }, - ); - - /** - * why: The half of the daemon-stop fix that argv cannot show. `stdio: "ignore"` alone does not - * release the event loop -- the ChildProcess handle does -- so without the `unref` a - * `daemon stop` logged "Daemon stopped" and then sat there for as long as any emulator ran. - * An emulator is meant to outlive the daemon (`DeviceTable#reattachRunningEmulators` adopts it after a - * restart) and is reaped through adb and by pid, never by awaiting its exit. - */ - it("releases the event loop for the emulator it starts, so a stopped daemon can exit", async () => { - const harness = await provisionedHarness(); - - await harness.driver.makeReady(harness.device); - - // why: The launches specifically -- `emulator` is also invoked as a short-lived query - // (`-list-avds` and friends) through `run`, which waits on the child and must stay - // referenced. Only a boot is detached. - const launches = harness.runner.calls - .map((call, index) => ({ call, index })) - .filter(({ call }) => call.command === binaries.emulator && call.args.includes("-avd")); - expect(launches.length, "no emulator was launched").toBeGreaterThan(0); - - for (const { index } of launches) { - expect(harness.runner.handles[index]?.unreffed, `launch at call ${String(index)}`).toBe(true); - } - expect(harness.runner.handles.filter((handle) => handle.unreffed)).toHaveLength( - launches.length, - ); - }); - - /** - * why: #237: a timer the driver leaves armed holds the event loop open, so a daemon that logged - * "Daemon stopped" stays alive until it fires. The shutdown the test asks for sees the - * emulator exit, so it has nothing left to wait for. - */ - it("leaves no timer armed once the shutdown has seen its emulator exit", async () => { - const harness = await provisionedHarness({ - afterwards: [ - copyPathExpectation("emulator-5586"), - processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), - ], - }); - const ready = await harness.driver.makeReady(harness.device); - const launch = harness.runner.calls.findLastIndex( - (call) => call.command === binaries.emulator && call.args.includes("-avd"), - ); - const running = harness.runner.handles[launch]; - expect(running, "no running emulator").toBeDefined(); - - const shutdown = harness.driver.shutdown(ready); - running?.kill("SIGTERM"); - await shutdown; - - expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); - }); - - it("kills an emulator that has not exited 180 s after a shutdown, whatever the boot timeout is", async () => { - const harness = await provisionedHarness({ - afterwards: [ - copyPathExpectation("emulator-5586"), - processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), - ], - bootTimeoutMs: 600_000, - }); - const ready = await harness.driver.makeReady(harness.device); - const launch = harness.runner.calls.findLastIndex( - (call) => call.command === binaries.emulator && call.args.includes("-avd"), - ); - const running = harness.runner.handles[launch]; - if (running === undefined) throw new Error("no running emulator"); - const kill = vi.spyOn(running, "kill"); - - // why: The emulator ignores `emu kill`: only the timeout can end the wait. - harness.runner.ignoresEmuKill = true; - const armedBefore = harness.clock.pendingTimerCount; - const shutdown = harness.driver.shutdown(ready); - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(armedBefore)); - expect(kill).not.toHaveBeenCalled(); - harness.clock.advance(180_000); - await shutdown; - - expect(kill).toHaveBeenCalledWith("SIGKILL"); - expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); - }); - - it("leaves no timer armed when waiting for the emulator to exit fails", async () => { - const harness = await provisionedHarness({ - afterwards: [ - copyPathExpectation("emulator-5586"), - processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), - ], - }); - const ready = await harness.driver.makeReady(harness.device); - const launch = harness.runner.calls.findLastIndex( - (call) => call.command === binaries.emulator && call.args.includes("-avd"), - ); - const running = harness.runner.handles[launch]; - if (running === undefined) throw new Error("no running emulator"); - vi.spyOn(running, "wait").mockRejectedValue(new Error("spawn error")); - - await expect(harness.driver.shutdown(ready)).rejects.toThrow("spawn error"); - - expect(harness.clock.pendingTimerCount, "timers still armed after shutdown").toBe(0); - }); - - it("fails the boot as soon as the emulator exits, naming the emulator's own error line", async () => { - const harness = await provisionedHarness({ bootCompleted: "0\n" }); - // why: Emulator 36.1.9's real output for a flag it does not know (#1199); it then exits 1. - const output = [ - "INFO | Android emulator version 36.1.9.0 (build_id 13823996) (CL:N/A)", - "unknown option: -crash-report-mode", - "please use -help for a list of valid options", - ]; - const originalSpawn = harness.runner.spawn.bind(harness.runner); - vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { - const handle = originalSpawn(command, args, options); - if (command !== binaries.emulator || !args.includes("-avd")) return handle; - handle.kill("SIGKILL"); - // why: whichever way the driver keeps the emulator's output (a log file or a pipe), it gets these lines. - const written = - typeof options?.stdio === "object" - ? harness.filesystem.writeFileAtomic(options.stdio.file, `${output.join("\n")}\n`) - : Promise.resolve(); - async function* lines(): AsyncGenerator { - yield* output; - } - return { - pid: handle.pid, - stderr: lines(), - stdout: lines(), - kill: () => undefined, - unref: () => undefined, - wait: async () => { - await written; - return { code: 1, stderr: "", stdout: `${output.join("\n")}\n` }; - }, - }; - }); - const armedBefore = harness.clock.pendingTimerCount; - - let outcome: unknown; - harness.driver.makeReady(harness.device).then( - () => (outcome = "ready"), - (error: unknown) => (outcome = error), - ); - // why: the clock never moves, so a driver that is still waiting for adb has armed its poll timer. - await vi.waitFor(() => - expect(outcome !== undefined || harness.clock.pendingTimerCount > armedBefore).toBe(true), - ); - - expect(outcome, "still waiting for the boot after the emulator exited").toMatchObject({ - message: expect.stringContaining("unknown option: -crash-report-mode"), - }); - }); - - it("sends the emulator's output to /emulator.log, mode 0600, truncated at each start", async () => { - const harness = await provisionedHarness(); - const logPath = `${avdDirectory}/simlock_one/simlock.avd/emulator.log`; - await harness.filesystem.writeFileAtomic(logPath, "left by an earlier start\n", { - mode: 0o644, - }); - - await harness.driver.makeReady(harness.device); - - const launch = harness.runner.calls.findLast( - (call) => call.command === binaries.emulator && call.args.includes("-avd"), - ); - expect(launch?.options.stdio).toEqual({ file: logPath }); - await expect(harness.filesystem.readFile(logPath)).resolves.toBe(""); - expect((await harness.filesystem.lstat(logPath)).mode & 0o777).toBe(0o600); - }); - - it("an Android boot waits android.bootTimeoutMs, then kills the emulator and fails with BootTimeoutError", async () => { - for (const { key, waitsMs } of [ - { key: undefined, waitsMs: 300_000 }, - { key: 600_000, waitsMs: 600_000 }, - ]) { - const harness = await provisionedHarness({ - bootCompleted: "0\n", - ...(key === undefined ? {} : { bootTimeoutMs: key }), - }); - const kills: string[] = []; - const originalSpawn = harness.runner.spawn.bind(harness.runner); - vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { - const handle = originalSpawn(command, args, options); - const kill = handle.kill.bind(handle); - return { - pid: handle.pid, - stderr: handle.stderr, - stdout: handle.stdout, - kill(signal) { - kills.push(signal ?? "SIGTERM"); - kill(signal); - }, - unref: () => handle.unref(), - wait: () => handle.wait(), - }; - }); - - // why: the script answers one poll; a boot that outlasts minutes keeps being told it is not done. - const scripted = harness.runner.run.bind(harness.runner); - vi.spyOn(harness.runner, "run").mockImplementation((command, args, options) => - args.at(-1) === "sys.boot_completed" - ? Promise.resolve({ code: 0, stderr: "", stdout: "0\n" }) - : scripted(command, args, options), - ); - - const ready = harness.driver.makeReady(harness.device); - const state = promiseState(ready); - const failed = expect(ready).rejects.toMatchObject({ name: "BootTimeoutError" }); - // why: each poll arms the next delay only after the previous one fired, so the clock moves in steps. - for (let elapsed = 0; elapsed < waitsMs - 1; elapsed += 1_000) { - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(Math.min(1_000, waitsMs - 1 - elapsed)); - for (let tick = 0; tick < 100; tick += 1) await Promise.resolve(); - } - expect(state.state, `still booting just before ${String(waitsMs)} ms`).toBe("pending"); - expect(kills).toEqual([]); - - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(1); - await failed; - expect(kills).toContain("SIGKILL"); - } - }); - - it("retries adb while the emulator is starting", async () => { - const harness = await provisionedHarness({ initialAdbFailures: 1 }); - const polledBefore = bootPolls(harness.runner); - const ready = harness.driver.makeReady(harness.device); - - await vi.waitFor(() => expect(bootPolls(harness.runner)).toBeGreaterThan(polledBefore)); - harness.clock.advance(2_000); - - await expect(ready).resolves.toMatchObject({ address: "emulator-5586" }); - }); - - it("is not ready until `pm path android` exits 0 and prints a package", async () => { - const harness = await provisionedHarness({ - unsettledPmAnswers: [ - { code: 1, stderr: "cmd: Can't find service: package", stdout: "" }, - { code: 0, stderr: "", stdout: "Error: could not access the Package Manager\n" }, - { code: 1, stderr: "", stdout: "package:/system/framework/framework-res.apk\n" }, - ], - }); - const pmCallsBefore = harness.runner.calls.filter((call) => call.args.includes("pm")).length; - const ready = harness.driver.makeReady(harness.device); - const settled = vi.fn(); - void ready.then(settled, settled); - const pmCalls = () => - harness.runner.calls.filter((call) => call.args.includes("pm")).length - pmCallsBefore; - - for (const asked of [1, 2, 3]) { - await vi.waitFor(() => expect(pmCalls()).toBe(asked)); - expect(settled, `ready after ${asked} unanswered pm`).not.toHaveBeenCalled(); - harness.clock.advance(2_000); - } - - await expect(ready).resolves.toMatchObject({ address: "emulator-5586" }); - }); - - describe("android.emulator launch options", () => { - const defaults: AndroidEmulatorLaunchOptions = { - audio: true, - bootAnimation: true, - gpu: "auto", - headless: false, - }; - it("launches with the same arguments as before these options existed under the default config", async () => { - for (const emulator of [undefined, defaults]) { - const harness = await provisionedHarness(emulator === undefined ? {} : { emulator }); - - await harness.driver.makeReady(harness.device); - - expect(emulatorLaunchArgs(harness.runner)).toEqual([ - [...seedLaunch, "-no-snapshot-load"], - [...seedLaunch, "-no-snapshot-load"], - [...seedLaunch, "-snapshot", "simlock_boot"], - ]); - } - }); - - it("adds -no-window for headless, -no-audio for audio: false, and -no-boot-anim for bootAnimation: false", async () => { - const flags = ["-no-window", "-no-audio", "-no-boot-anim"]; - const harness = await provisionedHarness({ - emulator: { ...defaults, audio: false, bootAnimation: false, headless: true }, - emulatorFlags: flags, - }); - - await harness.driver.makeReady(harness.device); - - const launches = emulatorLaunchArgs(harness.runner); - expect(launches).toHaveLength(3); - for (const args of launches) { - expect(args.slice(0, 10 + flags.length)).toEqual([...seedLaunch, ...flags]); - } - }); - - it("passes a gpu mode as -gpu , and passes nothing for auto", async () => { - const harness = await provisionedHarness({ - emulator: { ...defaults, gpu: "swiftshader_indirect" }, - emulatorFlags: ["-gpu", "swiftshader_indirect"], - }); - await harness.driver.makeReady(harness.device); - expect(emulatorLaunchArgs(harness.runner)[0]).toEqual([ - ...seedLaunch, - "-gpu", - "swiftshader_indirect", - "-no-snapshot-load", - ]); - - const auto = await provisionedHarness({ emulator: { ...defaults, gpu: "auto" } }); - await auto.driver.makeReady(auto.device); - for (const args of emulatorLaunchArgs(auto.runner)) { - expect(args).not.toContain("-gpu"); - } - }); - }); - - it("deletes only the copy's folder, keeps the seed it came from, and stops nothing for a copy that never booted", async () => { - const harness = await provisionedHarness(); - const callsBefore = harness.runner.calls.length; - - await expect(harness.driver.destroy(harness.device)).resolves.toBeUndefined(); - - expect(harness.runner.calls.slice(callsBefore)).toEqual([]); - expect(await harness.filesystem.exists(harness.copyHome)).toBe(false); - expect(await harness.filesystem.exists(`${harness.seedHome}/simlock.avd`)).toBe(true); - }); - - it("reports conservative cold-boot estimates through the Driver contract", async () => { - const driver: Driver = await createDriver( - await androidFilesystem(), - new EmulatorExitingRunner([]), - ); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - - expect(driver.estimate({ operation: "provision" }, spec)).toBe(150_000); - expect(driver.estimate({ operation: "boot" }, spec)).toBe(70_000); - }); - - it("prices reclaim by the strategy the clean level selects", async () => { - const driver: Driver = await createDriver( - await androidFilesystem(), - new EmulatorExitingRunner([]), - ); - const spec = { - physical: false, - model: "Pixel 5", - osVersion: "34", - platform: "android", - } as const; - - // why: Both measured on an M3 Pro / Pixel 8 / API 35. The gap is real: a `wipe` reclaim defers - // the wipe to the next `makeReady`; a reclaim that also boots the device back to warm - // spends a cold wipe boot in `reclaiming` rather than a shutdown. - expect(driver.estimate({ clean: "standard", operation: "reclaim" }, spec)).toBe(6_000); - expect(driver.estimate({ clean: "full", operation: "reclaim" }, spec)).toBe(32_000); - }); - - describe("avdmanager list device reads", () => { - it("seeds a library provision from the `avdmanager list device` read the catalog made", async () => { - const harness = await libraryHarness({ listing: pixelDevices }); - - await harness.driver.listCatalog(); - await harness.provision("Pixel 9"); - - expect(harness.seedDeviceId()).toBe("pixel_5"); - expect( - harness.runner.calls.filter((call) => call.args.join(" ") === "list device"), - ).toHaveLength(1); - }); - - it.each([ - { - env: { ANDROID_USER_HOME: "/user-home" }, - path: "/user-home/devices.xml", - situation: "ANDROID_USER_HOME", - }, - { env: {}, path: `${home}/.android/devices.xml`, situation: "its home directory" }, - ])( - "hands the catalog its scoped environment, its SDK root, its diagnostics callback and the devices.xml under $situation", - async ({ env, path }) => { - const addOnDevicesXml = `${sdk}/add-ons/addon-acme/devices.xml`; - const filesystem = await androidFilesystem(); - await filesystem.mkdirp(path.slice(0, path.lastIndexOf("/"))); - await filesystem.writeFileAtomic(path, "not xml at all {{{"); - await filesystem.mkdirp(`${sdk}/add-ons/addon-acme`); - await filesystem.writeFileAtomic(addOnDevicesXml, ""); - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ]); - const diagnostics: AndroidDriverDiagnostic[] = []; - const driver = await createDriver(filesystem, runner, { - extraEnv: env, - onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), - }); - - await driver.listCatalog(); - await filesystem.writeFileAtomic(addOnDevicesXml, ""); - await driver.listCatalog(); - - const listRuns = runner.calls.filter((call) => call.args.join(" ") === "list device"); - expect(listRuns.map((call) => call.options.env)).toEqual([ - { ...scopedEnv, ...env }, - { ...scopedEnv, ...env }, - ]); - expect(diagnostics).toContainEqual( - expect.objectContaining({ kind: "device-profile-source-unreadable", path }), - ); - }, - ); - }); - - it("answers virtualLeaseEnvironment with simlock's adb port, and leaseEnvironment for a virtual device equals it", async () => { - const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); - - expect(driver.virtualLeaseEnvironment()).toEqual({ - ANDROID_ADB_SERVER_PORT: String(adbServerPort), - }); - expect(driver.leaseEnvironment(VIRTUAL_DEVICE)).toEqual(driver.virtualLeaseEnvironment()); - }); - - it("hands a lease holder of a physical device its serial and the default adb server", async () => { - const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); - - expect(driver.leaseEnvironment(PHYSICAL_DEVICE)).toEqual({ - ANDROID_ADB_SERVER_PORT: "5037", - ANDROID_SERIAL: "R58M123ABC", - }); - }); - - it("answers uninstall for the reclaim strategy of a physical device, and refuses one for an emulator, which is never reclaimed", async () => { - const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); - - expect(() => driver.reclaimStrategy(VIRTUAL_DEVICE, { clean: "standard" })).toThrow( - "virtual devices are not reclaimed", - ); - expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "standard" })).toBe("uninstall"); - expect(driver.reclaimStrategy(PHYSICAL_DEVICE, { clean: "full" })).toBe("uninstall"); - }); - - it("rewrites the mark on makeReady's early-return branch without duplicating the config.ini line", async () => { - const harness = await provisionedHarness({ - afterwards: [ - // why: Second makeReady call: `state.handle` is still set from the first call, so this takes - // the early-return branch -- it must still wait for readiness and rewrite the mark. - processResult( - binaries.adb, - ["-s", "emulator-5586", "shell", "getprop", "sys.boot_completed"], - "1\n", - ), - processResult( - binaries.adb, - ["-s", "emulator-5586", "shell", "getprop", "init.svc.bootanim"], - "", - ), - copyPathExpectation("emulator-5586"), - processResult( - binaries.adb, - ["-s", "emulator-5586", "shell", "pm", "path", "android"], - "package:/system/framework/framework-res.apk\n", - ), - markWriteExpectation("emulator-5586", "device-4"), - ], - }); - - await harness.driver.makeReady(harness.device); - await harness.driver.makeReady(harness.device); - - const config = await harness.filesystem.readFile(`${harness.copyHome}/simlock.avd/config.ini`); - expect(config.split(/\r?\n/).filter((line) => line.startsWith("simlock.mark="))).toEqual([ - "simlock.mark=device-4", - ]); - expect(config).toContain("hw.ramSize=2048"); - }); - - it("rethrows a non-missing-file config.ini read error from the mark write instead of clobbering it as empty", async () => { - const configPath = `${avdDirectory}/simlock_one/simlock.avd/config.ini`; - const permissionError = Object.assign(new Error("EACCES: permission denied"), { - code: "EACCES", - }); - const failureFilesystem = new ReadFailureFilesystem(configPath, permissionError); - const harness = await provisionedHarness({ filesystem: failureFilesystem }); - - await expect(harness.driver.makeReady(harness.device)).rejects.toBe(permissionError); - - failureFilesystem.armed = false; - await expect(harness.filesystem.readFile(configPath)).resolves.toBe("hw.ramSize=2048\n"); - }); - - describe("device-profile-hidden advisories", () => { - const hiddenMessage = (xmlName: string, libraryName: string) => - `"${xmlName}" in devices.xml is not used: Simlock's own ${libraryName} profile answers to that name`; - - // why: the fake adb server answers no version, so the physical handler adds its own advisory. - async function hiddenProfileAdvisories(driver: AndroidDriver) { - return (await driver.advisories()).filter( - (advisory) => advisory.code === "device-profile-hidden", - ); - } - - async function advisoriesFor(devicesXml: string | undefined, listing = pixelDevices) { - const filesystem = await androidFilesystem(); - if (devicesXml !== undefined) await writeDevicesXml(filesystem, devicesXml); - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], listing), - ]); - const driver = await createDriver(filesystem, runner); - return { advisories: await hiddenProfileAdvisories(driver), runner }; - } - - it("reports `device-profile-hidden` for a `devices.xml` profile named like a library profile, with the message naming both", async () => { - const { advisories } = await advisoriesFor(customDeviceXml("Pixel 9", 4096)); - - expect(advisories).toEqual([ - { code: "device-profile-hidden", message: hiddenMessage("Pixel 9", "Pixel 9") }, - ]); - }); - - it("reports one matching a library profile's other name, in any letter case, naming the library profile's display name", async () => { - const { advisories } = await advisoriesFor( - customDeviceXml("pixel_9", 4096) + customDeviceXml("PIXEL 9", 4096), - ); - - expect(advisories).toEqual([ - { code: "device-profile-hidden", message: hiddenMessage("pixel_9", "Pixel 9") }, - { code: "device-profile-hidden", message: hiddenMessage("PIXEL 9", "Pixel 9") }, - ]); - }); - - it("reports two hidden profiles as two advisories, in `devices.xml` order", async () => { - const { advisories } = await advisoriesFor( - customDeviceXml("Pixel 9", 4096) + customDeviceXml("Pixel 8", 4096), - ); - - expect(advisories.map((advisory) => advisory.message)).toEqual([ - hiddenMessage("Pixel 9", "Pixel 9"), - hiddenMessage("Pixel 8", "Pixel 8"), - ]); - }); - - it("reports nothing for a `devices.xml` profile no library profile answers to", async () => { - const { advisories } = await advisoriesFor(customDeviceXml("My Tablet", 4096)); - - expect(advisories).toEqual([]); - }); - - it("reports nothing for a library name the SDK also lists", async () => { - const sdkListing = `Available devices:\nid: 0 or "pixel_9"\n Name: Pixel 9\n OEM : Google\n`; - - const { advisories } = await advisoriesFor(undefined, sdkListing); - - expect(advisories).toEqual([]); - }); - - it("reports one advisory per profile when `avdmanager list device` also lists it with `OEM : User`, and runs no `avdmanager`", async () => { - const userListing = `Available devices:\nid: 0 or "Pixel 9"\n Name: Pixel 9\n OEM : User\n`; - - const { advisories, runner } = await advisoriesFor( - customDeviceXml("Pixel 9", 4096), - userListing, - ); - - expect(advisories).toEqual([ - { code: "device-profile-hidden", message: hiddenMessage("Pixel 9", "Pixel 9") }, - ]); - expect(runner.calls.map((call) => call.command)).not.toContain(binaries.avdmanager); - }); - - it("reports nothing when `devices.xml` is missing or unreadable", async () => { - const missing = await advisoriesFor(undefined); - const filesystem = await androidFilesystem(); - await filesystem.mkdirp(`${home}/.android`); - await filesystem.writeFileAtomic(`${home}/.android/devices.xml`, "not xml at all {{{"); - const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); - - expect(missing.advisories).toEqual([]); - expect(await hiddenProfileAdvisories(driver)).toEqual([]); - }); - - it("keeps the physical handler's advisories, before the new ones", async () => { - const base = await androidFilesystem(); - await writeDevicesXml(base, customDeviceXml("Pixel 9", 4096)); - let broken = false; - const filesystem = new Proxy(base, { - get(target, property) { - if (property === "stat" && broken) { - return async () => { - throw new Error("EACCES"); - }; - } - const value = Reflect.get(target, property) as unknown; - return typeof value === "function" ? (value as () => unknown).bind(target) : value; - }, - }); - const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); - broken = true; - - const advisories = await driver.advisories(); - - expect(advisories.map((advisory) => advisory.code)).toEqual([ - "adb-protocol-mismatch", - "device-profile-hidden", - ]); - }); - - it("reports no hidden profiles for a driver built with injected `deviceProfileSources`", async () => { - const filesystem = await androidFilesystem(); - await writeDevicesXml(filesystem, customDeviceXml("Pixel 9", 4096)); - const driver = await createDriver(filesystem, new EmulatorExitingRunner([]), { - deviceProfileSources: [], - }); - - expect(await hiddenProfileAdvisories(driver)).toEqual([]); - }); - }); - - describe("device-profile sources", () => { - it("resolves a devices.xml-only model and applies its properties to config.ini", async () => { - const harness = await libraryHarness({ devicesXml: customDeviceXml("Custom A", 4096) }); - - const spec = await harness.resolve("Custom A"); - expect(spec).toEqual({ - physical: false, - model: "Custom A", - osVersion: "35", - platform: "android", - }); - - const device = await harness.driver.provision(spec); - - const config = await harness.filesystem.readFile(harness.configPath); - expect(config).toContain("hw.device.name=Custom A"); - expect(config).toContain("hw.ramSize=4096"); - // why: The avdmanager `-d` seed uses a built-in device only to skip the interactive prompt -- - // it must never leak into the resolved spec or the applied hardware properties. - expect(config).not.toContain("pixel_5"); - expect(device.driverData).toMatchObject({ avdName: "simlock" }); - }); - }); - - describe("library device profiles", () => { - it.each([ - { spelling: "Pixel 9", situation: "avdmanager does not list pixel_9", listing: pixelDevices }, - { spelling: "Pixel 9", situation: "avdmanager lists pixel_9 too", listing: twoPixelDevices }, - { - spelling: "Pixel 9", - situation: 'devices.xml defines a "Pixel 9"', - listing: pixelDevices, - devicesXml: customDeviceXml("Pixel 9", 4096), - ramSize: "2G", - }, - { spelling: "pixel_9", situation: "spelled as its avdmanager id", listing: pixelDevices }, - { spelling: "PIXEL 9", situation: "spelled in upper case", listing: pixelDevices }, - ])( - 'resolves "$spelling" to the library Pixel 9 when $situation', - async ({ spelling, listing, devicesXml, ramSize }) => { - const harness = await libraryHarness({ listing, ...(devicesXml ? { devicesXml } : {}) }); - - const spec = await harness.resolve(spelling); - await harness.driver.provision(spec); - - expect(spec).toEqual({ - model: "Pixel 9", - osVersion: "35", - physical: false, - platform: "android", - }); - const config = await harness.config(); - expect(config.get("hw.lcd.height")).toBe("2424"); - expect(config.get("hw.lcd.density")).toBe("420"); - if (ramSize !== undefined) expect(config.get("hw.ramSize")).toBe(ramSize); - }, - ); - - it("resolves a model the library does not have through the SDK, then devices.xml, as today", async () => { - const sdkHarness = await libraryHarness({ - devicesXml: customDeviceXml("Custom A", 4096), - listing: pixelDevices, - }); - const sdkSpec = await sdkHarness.resolve("Pixel 5"); - await sdkHarness.driver.provision(sdkSpec); - - expect(sdkSpec.model).toBe("Pixel 5"); - expect(sdkHarness.seedDeviceId()).toBe("pixel_5"); - expect((await sdkHarness.config()).has("hw.lcd.height")).toBe(false); - - const userHarness = await libraryHarness({ - devicesXml: customDeviceXml("Custom A", 4096), - listing: pixelDevices, - }); - const userSpec = await userHarness.resolve("Custom A"); - await userHarness.driver.provision(userSpec); - - expect(userSpec.model).toBe("Custom A"); - expect((await userHarness.config()).get("hw.ramSize")).toBe("4096"); - }); - - it("captures the config hash after the library's keys are written", async () => { - const hashOf = async (profile: LibraryDeviceProfile, seed: string) => { - const harness = await libraryHarness({ profiles: [profile], seed }); - const device = await harness.provision("Test Phone"); - return (device.driverData as { configHash: string }).configHash; - }; - const ram = (value: string) => - testLibraryProfile({ hardware: { "hw.lcd.height": "100", "hw.ramSize": value } }); - - const small = await hashOf(ram("2G"), seedConfig([])); - const large = await hashOf(ram("4G"), seedConfig([])); - const largeOverOtherSeed = await hashOf(ram("4G"), seedConfig(["hw.ramSize=512M"])); - - expect(small).not.toBe(large); - expect(largeOverOtherSeed).toBe(large); - }); - - it("refuses NUL in a `simlock.mark` value, through the one shared check", async () => { - const markedConfig = (harness: Awaited>) => - harness.filesystem.readFile(`${harness.copyHome}/simlock.avd/config.ini`); - const accepted = await provisionedHarness({ - firstMarkToken: "tok-en", - ids: ["one", "two", "tok-en"], - }); - await accepted.driver.makeReady(accepted.device); - expect(await markedConfig(accepted)).toContain("simlock.mark=tok-en"); - - const refused = await provisionedHarness({ - firstMarkToken: "tok\u0000en", - ids: ["one", "two", "tok\u0000en"], - }); - - await expect(refused.driver.makeReady(refused.device)).rejects.toThrow(/Refusing to merge/); - expect(await markedConfig(refused)).not.toContain("simlock.mark="); - }); - }); - - describe("Android SDK license handling", () => { - const licenseNotAcceptedOutput = - "Warning: License for package Android SDK Platform 35 not accepted.\n\n" + - "1 package(s) were skipped due to license issues. Please accept the license(s) and try " + - "again.\nTo resolve, run: sdkmanager --licenses\n"; - - it("accepts licenses through piped confirmation and retries the install once when the flag is on", async () => { - const filesystem = await androidFilesystem(); - const runner = new InstallReflectingProcessRunner( - [ - { - match: { - args: ["--install", "system-images;android-35;google_apis;arm64-v8a"], - command: binaries.sdkmanager, - }, - result: { code: 1, stderr: "", stdout: licenseNotAcceptedOutput }, - }, - processResult(binaries.sdkmanager, ["--licenses"], "All licenses accepted.\n"), - processResult(binaries.sdkmanager, [ - "--install", - "system-images;android-35;google_apis;arm64-v8a", - ]), - ], - filesystem, - ); - const driver = await createDriver(filesystem, runner, { acceptAndroidLicenses: true }); - - await expect(installFor(driver, "35")).resolves.toMatchObject({ - outcome: "installed", - version: "35", - }); - - const licensesCall = runner.calls.find( - (call) => call.command === binaries.sdkmanager && call.args[0] === "--licenses", - ); - expect(licensesCall?.options.input).toBe("y\n".repeat(100)); - expect(runner.calls.filter((call) => call.args[0] === "--install")).toHaveLength(2); - }); - }); - - describe("component install", () => { - it("refuses an unaccepted licence with AndroidLicenseNotAcceptedError, and runs no sdkmanager --licenses, when the config does not accept them", async () => { - const filesystem = await androidFilesystem(); - const runner = new EmulatorExitingRunner([ - { - match: { - args: ["--install", "system-images;android-35;google_apis;arm64-v8a"], - command: binaries.sdkmanager, - }, - result: { - code: 1, - stderr: "", - stdout: "Warning: License for package Android SDK Platform 35 not accepted.\n", - }, - }, - ]); - const driver = await createDriver(filesystem, runner); - - await expect(installFor(driver, "35")).rejects.toBeInstanceOf(AndroidLicenseNotAcceptedError); - expect(runner.calls.some((call) => call.args[0] === "--licenses")).toBe(false); - }); - - it("reports the revisions of the tool packages under the SDK root it was given", async () => { - const filesystem = await androidFilesystem(); - await filesystem.writeFileAtomic( - `${sdk}/emulator/source.properties`, - "Pkg.Revision=35.4.9\n", - ); - const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); - - await expect(driver.toolVersions()).resolves.toEqual([ - { name: "emulator", version: "35.4.9" }, - ]); - }); - - it("states 2 GiB on the SDK root as the footprint of one install", async () => { - const filesystem = await androidFilesystem(); - const driver = await createDriver(filesystem, new EmulatorExitingRunner([])); - - expect(driver.componentFootprint).toEqual({ bytes: 2 * 1024 ** 3, path: sdk }); - }); - }); -}); - -describe("AndroidDriver screen capture", () => { - const serverPath = "/simlock/vendor/scrcpy-server-v5.0.1"; - const running = { - address: "emulator-5586", - deviceId: "simlock_one", - driverData: { avdName: "simlock_one", configHash: "h", port: 5586, serial: "emulator-5586" }, - physical: false, - } as const; - const settings = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; - - it("is offered, with the device input, only when both the server path and a TCP stream connector are given", async () => { - const both = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { - scrcpyServerPath: serverPath, - tcpStreamConnector: new FakeTcpStreamConnector(), - }); - expect(both.screenCapture).toBeDefined(); - expect(both.input).toBeDefined(); - const noPath = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { - tcpStreamConnector: new FakeTcpStreamConnector(), - }); - expect(noPath.screenCapture).toBeUndefined(); - expect(noPath.input).toBeUndefined(); - const noConnector = await createDriver( - await androidFilesystem(), - new EmulatorExitingRunner([]), - { - scrcpyServerPath: serverPath, - }, - ); - expect(noConnector.screenCapture).toBeUndefined(); - expect(noConnector.input).toBeUndefined(); - }); - - it("pushes the server with this driver's adb, its scoped environment and the device's serial", async () => { - const filesystem = await androidFilesystem(); - await filesystem.mkdirp("/simlock/vendor"); - await filesystem.writeFileAtomic(serverPath, "jar"); - const runner = new EmulatorExitingRunner([ - { - match: { - args: ["-s", "emulator-5586", "push", serverPath, /simlock-scrcpy-/], - command: binaries.adb, - }, - result: { code: 1, stderr: "error: closed", stdout: "" }, - }, - { - match: { - args: ["-s", "emulator-5586", "shell", "rm", "-f", /simlock-scrcpy-/], - command: binaries.adb, - }, - }, - ]); - const driver = await createDriver(filesystem, runner, { - scrcpyServerPath: serverPath, - tcpStreamConnector: new FakeTcpStreamConnector(), - }); - await expect( - driver.screenCapture?.open(running, settings, new AbortController().signal), - ).rejects.toThrow(/error: closed/); - expect(runner.calls[0]?.options.env).toEqual(scopedEnv); - expect(runner.calls[1]?.options.env).toEqual(scopedEnv); - }); - - it("refuses an emulator that is not running, before running anything", async () => { - const filesystem = await androidFilesystem(); - await filesystem.mkdirp("/simlock/vendor"); - await filesystem.writeFileAtomic(serverPath, "jar"); - const runner = new EmulatorExitingRunner([]); - const driver = await createDriver(filesystem, runner, { - scrcpyServerPath: serverPath, - tcpStreamConnector: new FakeTcpStreamConnector(), - }); - const stopped = { ...running, driverData: { ...running.driverData, port: 0, serial: "" } }; - await expect( - driver.screenCapture?.open(stopped, settings, new AbortController().signal), - ).rejects.toThrow(/not running/); - expect(runner.calls).toEqual([]); - }); - - it("routes each device's capture to its own adb server: a phone's commands run adb -P 5037 and its socket connects to 5037, an emulator's use Simlock's port", async () => { - const filesystem = await androidFilesystem(); - await filesystem.mkdirp("/simlock/vendor"); - await filesystem.writeFileAtomic(serverPath, "jar"); - const phone = { - address: "R58M123ABC", - deviceId: "R58M123ABC", - driverData: {}, - enrolledApps: [], - physical: true, - } as const; - const clock = new FakeClock(1_000); - // why: the shared server answers the version check with Simlock's own protocol, so the phone's - // target is open and its commands are expected to go out. - const probe = new FakeTcpProbe([5038, 5037]); - probe.replyWith("OKAY00040029"); - const runner = new CaptureRoutingRunner(); - const tcp = new FakeTcpStreamConnector(); - const driver = await createDriver(filesystem, runner, { - clock, - scrcpyServerPath: serverPath, - tcpProbe: probe, - tcpStreamConnector: tcp, - }); - - // why: each open is stopped while it waits on its socket's first reply — which is exactly the - // moment this test owns: every command has gone out through the device's own server, the socket - // has been aimed at the device, and a stop runs nothing further. - const phoneStop = new AbortController(); - const phoneOpening = driver.screenCapture?.open(phone, settings, phoneStop.signal); - if (phoneOpening === undefined) throw new Error("the driver offers no screen capture"); - // why: the rejection is asserted below; marking it handled here keeps a rejection that arrives - // while this test is still reading the socket from being reported as unhandled. - phoneOpening.catch(() => undefined); - await waitForSocket(clock, tcp, 1); - expect(tcp.connections[0]?.port).toBe(5037); - expect(tcp.connections[0]?.peer.writtenText()).toContain("host:transport:R58M123ABC"); - phoneStop.abort(); - await expect(driveOpen(clock, phoneOpening)).rejects.toThrow(/closed the connection/); - - const emulatorStop = new AbortController(); - const emulatorOpening = driver.screenCapture?.open(running, settings, emulatorStop.signal); - if (emulatorOpening === undefined) throw new Error("the driver offers no screen capture"); - emulatorOpening.catch(() => undefined); - await waitForSocket(clock, tcp, 2); - expect(tcp.connections[1]?.port).toBe(adbServerPort); - expect(tcp.connections[1]?.peer.writtenText()).toContain("host:transport:emulator-5586"); - emulatorStop.abort(); - await expect(driveOpen(clock, emulatorOpening)).rejects.toThrow(/closed the connection/); - - // why: every command the phone's session ran aims at the shared server — `adb -P 5037` with - // that server's env, Simlock's port in it and the three keys that could aim adb elsewhere - // removed — and every command the emulator's session ran carries this driver's scoped env, - // which points at Simlock's own port. Which commands a phone's session runs (its screen read - // included) is the capture's and the handler's own claim, proved in their own files. - const commands = runner.calls.filter((call) => call.args[0] !== "version"); - const phoneCommands = commands.filter((call) => call.args[0] === "-P"); - const emulatorCommands = commands.filter((call) => call.args[0] === "-s"); - expect(phoneCommands.length).toBeGreaterThanOrEqual(3); - for (const call of phoneCommands) { - expect(call.args.slice(0, 3)).toEqual(["-P", "5037", "-s"]); - expect(call.args[3]).toBe("R58M123ABC"); - expect(call.options.env).toEqual({ - ANDROID_ADB_SERVER_PORT: "5037", - ANDROID_HOME: sdk, - }); - } - expect( - phoneCommands.map((call) => - call.args[4] === "push" - ? "push" - : call.args[5] === "rm" - ? "rm" - : call.args[5]?.startsWith("CLASSPATH=") - ? "server" - : String(call.args[4]), - ), - ).toEqual(expect.arrayContaining(["push", "server", "rm"])); - expect(emulatorCommands.map((call) => call.args.slice(0, 3))).toEqual([ - ["-s", "emulator-5586", "push"], - ["-s", "emulator-5586", "shell"], - ["-s", "emulator-5586", "shell"], - ]); - for (const call of emulatorCommands) { - expect(call.options.env).toEqual(scopedEnv); - } - }); -}); - -// why: flushes work until the n-th raw connection has been made, advancing the fake clock so the -// capture's own bounds cannot strand it; an open that died first fails on a named assertion. -async function waitForSocket( - clock: FakeClock, - tcp: FakeTcpStreamConnector, - count: number, -): Promise { - for (let spent = 0; spent < 60_000 && tcp.connections.length < count; spent += 50) { - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - clock.advance(50); - } - expect(tcp.connections.length, "the capture reached its socket").toBe(count); -} - -// why: awaits an open with the fake clock turning, so its own 15 s bound can fire while the test waits. -async function driveOpen(clock: FakeClock, opening: Promise): Promise { - let done = false; - void opening.then( - () => (done = true), - () => (done = true), - ); - for (let spent = 0; spent < 60_000 && !done; spent += 50) { - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - clock.advance(50); - } - return await opening; -} + androidFilesystem, + avdDirectory, + binaries, + EmulatorExitingRunner, + instanceId, + simlockHome, + InstallReflectingProcessRunner, + memoryAndroidSourceDatabase, + pixelDevices, + processResult, + userAvd, + adbServerPort, + adbServerPid, + adbRecordPath, + scopedEnv, + createDriver, + recordRunningAdbServer, + ok, + WorkingDirectoryFilesystem, +} from "./testing.js"; +import { sdk, home, VIRTUAL_DEVICE } from "./test-index.js"; /** - * why: answers every adb call a capture makes by what it is rather than by its place in a queue, so - * the routing this test reads is proven the same way whether or not the capture reads the phone's - * screen state first -- the server's shell hangs until the capture's own close kills it, as a live - * scrcpy server does. + * why: What a promise settled with, once it has: a promise still open fails a named assertion rather + * than the test's own timeout, so a deadline that never ends anything reads as that. */ -class CaptureRoutingRunner implements ProcessRunner { - readonly calls: { readonly args: readonly string[]; readonly options: ProcessRunOptions }[] = []; - #nextPid = 1; - - run( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): Promise { - return this.spawn(command, args, options).wait(); - } - - spawn(_command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { - this.calls.push({ args, options }); - const rest = args[0] === "-P" ? args.slice(2) : [...args]; - const line = rest.join(" "); - if (line === "version") { - return endedHandle(this.#nextPid++, "Android Debug Bridge version 1.0.41\nVersion 35.0.2\n"); - } - if (line.includes("dumpsys power")) { - return endedHandle(this.#nextPid++, " mWakefulness=Awake\n"); - } - if (/shell CLASSPATH=\S*app_process/.test(line)) { - return hangingHandle(this.#nextPid++); - } - return endedHandle(this.#nextPid++, ""); - } - - spawnStreaming(..._args: Parameters): never { - throw new Error("a screen capture spawns no streaming process"); - } -} - -function endedHandle(pid: number, stdout: string): ProcessHandle { - return { - kill: () => undefined, - pid, - stderr: emptyLines(), - stdout: emptyLines(), - unref: () => undefined, - wait: async () => ({ code: 0, stderr: "", stdout }), +async function settledValue(promise: Promise): Promise { + const state: { done: boolean; value: unknown } = { done: false, value: undefined }; + const settle = (value: unknown): void => { + state.done = true; + state.value = value; }; + void promise.then(settle, settle); + await vi.waitFor(() => expect(state.done).toBe(true)); + return state.value; } -function hangingHandle(pid: number): ProcessHandle { - let settle: (() => void) | undefined; - const ended = new Promise((resolve) => (settle = resolve)); - return { - kill: () => settle?.(), - pid, - stderr: emptyLines(), - stdout: emptyLines(), - unref: () => undefined, - wait: async () => { - await ended; - return { code: null, stderr: "", stdout: "" }; - }, +/** why: every case below asks for the `adb` tool, so this one place names it and leaves the device and context optional. */ +async function passthroughOf( + driver: object, + args: readonly string[], + context?: { readonly hasTerminal: boolean; readonly platform?: string }, + device?: unknown, +): Promise<{ readonly args: readonly string[]; readonly command: string; readonly env: object }> { + const routed = driver as { + passthrough( + tool: string, + device: unknown, + args: readonly string[], + context?: { readonly hasTerminal: boolean; readonly platform?: string }, + ): Promise<{ + readonly args: readonly string[]; + readonly command: string; + readonly env: object; + }>; }; + return routed.passthrough("adb", device, args, context); } describe("AndroidDriver.create", () => { @@ -2032,129 +566,3 @@ describe("AndroidDriver listComponents()", () => { }); }); }); - -/** - * why: What a promise settled with, once it has: a promise still open fails a named assertion rather - * than the test's own timeout, so a deadline that never ends anything reads as that. - */ -async function settledValue(promise: Promise): Promise { - const state: { done: boolean; value: unknown } = { done: false, value: undefined }; - const settle = (value: unknown): void => { - state.done = true; - state.value = value; - }; - void promise.then(settle, settle); - await vi.waitFor(() => expect(state.done).toBe(true)); - return state.value; -} - -const seedLaunch = ["-avd", "simlock", ...seedEmulatorLine]; - -/** - * why: A driver whose adb server is already running and recorded, so `create` adopts it: that is - * the one start-up path that spawns nothing, which keeps the scripted process expectations - * in every test about the behaviour the test is actually for. The start, reap, and refusal - * paths are covered against the supervisor itself in `adb-server.test.ts`, and end to end - * in this file's own `AndroidDriver.create` block. - */ -describe("AndroidDriver defaultModels", () => { - it("lists the built-in Android preference list per class, newest first, with phone starting at Pixel 9 and tablet holding Medium Tablet", async () => { - const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([])); - - expect(driver.defaultModels).toEqual({ - auto: ["Automotive (1080p landscape)", "Automotive (1024p landscape)"], - desktop: ["Medium Desktop", "Large Desktop"], - phone: ["Pixel 9", "Pixel 8", "Pixel 7", "Pixel 6", "Medium Phone"], - tablet: ["Medium Tablet"], - tv: ["Television (1080p)", "Television (4K)"], - watch: ["Wear OS Large Round", "Wear OS Small Round"], - }); - }); -}); - -/** why: every case below asks for the `adb` tool, so this one place names it and leaves the device and context optional. */ -async function passthroughOf( - driver: object, - args: readonly string[], - context?: { readonly hasTerminal: boolean; readonly platform?: string }, - device?: unknown, -): Promise<{ readonly args: readonly string[]; readonly command: string; readonly env: object }> { - const routed = driver as { - passthrough( - tool: string, - device: unknown, - args: readonly string[], - context?: { readonly hasTerminal: boolean; readonly platform?: string }, - ): Promise<{ - readonly args: readonly string[]; - readonly command: string; - readonly env: object; - }>; - }; - return routed.passthrough("adb", device, args, context); -} - -/** - * why: Fails every `readFile` of `failingPath` with a non-ENOENT error while `armed`, so a test can - * assert a caller rethrows it instead of treating it as an absent file -- then disarm to read - * the path back and confirm nothing overwrote it in the meantime. - */ -class ReadFailureFilesystem extends MemoryFilesystem { - armed = true; - - constructor( - private readonly failingPath: string, - private readonly error: Error, - freeDiskBytes?: number, - ) { - super(freeDiskBytes); - } - - override async readFile(path: string): Promise { - if (this.armed && path === this.failingPath) { - throw this.error; - } - return super.readFile(path); - } -} - -function installFor( - driver: AndroidDriver, - component: string, - options: { onProgress?: (progress: unknown) => void; signal?: AbortSignal } = {}, -) { - return driver.installComponent(component, { - onProgress: options.onProgress ?? (() => undefined), - signal: options.signal ?? new AbortController().signal, - }); -} - -/** why: how many times the emulator has been asked whether it finished booting. */ -function bootPolls(runner: ScriptedProcessRunner): number { - return runner.calls.filter((call) => call.args.at(-1) === "sys.boot_completed").length; -} - -describe("AndroidDriver custom profile fit from devices.xml", () => { - it("resolveSpec with --os 30 and provision refuse a custom profile whose api-level is 31-", async () => { - const images: readonly FitImage[] = [ - ["30", "google_apis", "arm64-v8a"], - ["31", "google_apis", "arm64-v8a"], - ]; - const { createdPackage, driver, refusal, resolve } = await fitHarness(images, { - devicesXml: customFitXml({ apiLevel: "31-", name: "My Phone" }), - listing: userListing(["My Phone"]), - }); - await expect(resolve("My Phone", "31")).resolves.toMatchObject({ osVersion: "31" }); - - expect(await refusal("My Phone", "30")).toMatchObject({ name: "RuntimeMissingError" }); - await expect( - driver.provision({ - model: "My Phone", - osVersion: "30", - physical: false, - platform: "android", - }), - ).rejects.toMatchObject({ name: "RuntimeMissingError" }); - expect(createdPackage()).toBeUndefined(); - }); -}); diff --git a/src/drivers/android/test-index.ts b/src/drivers/android/test-index.ts new file mode 100644 index 000000000..6e403ff87 --- /dev/null +++ b/src/drivers/android/test-index.ts @@ -0,0 +1,25 @@ +import { + ANDROID_SDK_ROOT, + type BuildProvisioner, + libraryHarness as libraryHarnessWith, + createDriver, + ANDROID_HOME_DIRECTORY, +} from "./testing.js"; + +export const sdk = ANDROID_SDK_ROOT; + +export const home = ANDROID_HOME_DIRECTORY; + +export const VIRTUAL_DEVICE = { + address: "", + deviceId: "device", + driverData: {}, + physical: false, +} as const; + +/** why: the harnesses the seed tests share with `seeds.test.ts` build whatever makes the seed; here it is the whole driver. */ +export const buildDriver: BuildProvisioner = (filesystem, processRunner, options) => + createDriver(filesystem, processRunner, options); + +export const libraryHarness = (options?: Parameters[1]) => + libraryHarnessWith(buildDriver, options); From ef2b09bc09541fe00f3d96f996d4985b16ddeb59 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:25:20 +0100 Subject: [PATCH 34/83] test(drivers): split index-lifecycle.test.ts by topic --- .../android/index-copies-recovery.test.ts | 406 ++++ src/drivers/android/index-device-tcp.test.ts | 469 +++++ src/drivers/android/index-image-fit.test.ts | 583 ++++++ src/drivers/android/index-lifecycle.test.ts | 1673 ++--------------- src/drivers/android/test-index-lifecycle.ts | 14 + 5 files changed, 1626 insertions(+), 1519 deletions(-) create mode 100644 src/drivers/android/index-copies-recovery.test.ts create mode 100644 src/drivers/android/index-device-tcp.test.ts create mode 100644 src/drivers/android/index-image-fit.test.ts create mode 100644 src/drivers/android/test-index-lifecycle.ts diff --git a/src/drivers/android/index-copies-recovery.test.ts b/src/drivers/android/index-copies-recovery.test.ts new file mode 100644 index 000000000..5c9b81dac --- /dev/null +++ b/src/drivers/android/index-copies-recovery.test.ts @@ -0,0 +1,406 @@ +import { describe, expect, it, vi } from "vitest"; +import { FakeClock, MemoryFilesystem, type ScriptedProcessExpectation } from "../../ports/index.js"; +import { type AndroidDriverDiagnostic, type AndroidEmulatorLaunchOptions } from "./index.js"; +import { + androidFilesystem, + avdDirectory, + binaries, + bootingAt, + makingASeedFrom, + markWriteExpectation, + EmulatorExitingRunner, + makingTheSeed, + seedEmulatorLine, + writeSeedFiles, + memoryAndroidSourceDatabase, + pixelDevices, + processResult, + emulatorLaunchArgs, + createDriver, + copyPathExpectation, +} from "./testing.js"; +import { avdPathExpectation } from "./test-index-lifecycle.js"; +import { sdk } from "./test-index.js"; + +describe("AndroidDriver image fit", () => { + describe("seeds and copies", () => { + const sources = `${avdDirectory}/sources`; + const copyHome = `${avdDirectory}/simlock_one`; + const seedHome = `${sources}/seed_two`; + async function seedHarness( + expectations: readonly ScriptedProcessExpectation[], + options: { + readonly clock?: FakeClock; + readonly config?: string; + readonly filesystem?: MemoryFilesystem; + readonly ids?: readonly string[]; + readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; + } = {}, + ) { + const filesystem = await androidFilesystem({}, options.filesystem); + await writeSeedFiles(filesystem, seedHome, options.config); + const database = await memoryAndroidSourceDatabase(); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...expectations, + ]); + const driver = await createDriver(filesystem, runner, { + ...(options.clock === undefined ? {} : { clock: options.clock }), + ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }), + database, + ids: options.ids ?? ["one", "two", "three"], + }); + const spec = await driver.resolveSpec({ + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + return { database, driver, filesystem, runner, spec }; + } + + /** why: the daemon after a restart: a new driver over the same device folders and the same source table. */ + async function reopened( + harness: Awaited>, + expectations: readonly ScriptedProcessExpectation[], + options: { + readonly clock?: FakeClock; + readonly emulator?: AndroidEmulatorLaunchOptions; + } = {}, + ) { + const runner = new EmulatorExitingRunner([...expectations]); + const driver = await createDriver(harness.filesystem, runner, { + ...options, + database: harness.database, + ids: ["four", "five", "six"], + }); + return { driver, runner }; + } + + it("quotes a provision at 1 s with a fresh seed and 150 s without one, and a prepare boot at 4 s for a copy and 70 s otherwise", async () => { + const harness = await seedHarness([ + ...makingTheSeed(), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + const spec = { ...harness.spec, physical: false } as const; + const provision = { operation: "provision" } as const; + expect(harness.driver.estimate(provision, spec)).toBe(150_000); + + const copy = await harness.driver.provision(harness.spec); + + expect(harness.driver.estimate(provision, spec)).toBe(1_000); + expect(harness.driver.estimate(provision, { ...spec, model: "Pixel 9" })).toBe(150_000); + expect(harness.driver.estimate(provision, { ...spec, imageTag: "android-tv" })).toBe(150_000); + const prepare = { device: copy, operation: "boot", purpose: "prepare" } as const; + const oldLayout = { + address: "", + deviceId: "simlock_old", + driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, + }; + expect( + await Promise.all([ + harness.driver.estimate(prepare, spec), + harness.driver.estimate({ ...prepare, purpose: "recover" }, spec), + harness.driver.estimate({ ...prepare, device: oldLayout }, spec), + ]), + ).toEqual([4_000, 70_000, 70_000]); + await expect( + harness.driver.estimate({ ...prepare, device: { ...copy, deviceId: "other" } }, spec), + "a device id that does not match its AVD rejects the returned promise: a throw would skip the caller's fallback", + ).rejects.toThrow("does not match AVD"); + + const staled = await reopened(harness, [ + processResult(binaries.emulator, ["-version"], "Android emulator version 36.2.0"), + ]); + await staled.driver.listSources(); + expect(staled.driver.estimate(provision, spec)).toBe(150_000); + }); + + it("destroys the seeds made from a system image when it is removed, and a seed it cannot destroy stays listed as stale", async () => { + const packageName = "system-images;android-34;google_apis;arm64-v8a"; + const imagePath = `${sdk}/system-images/android-34/google_apis/arm64-v8a`; + const uninstall = processResult(binaries.sdkmanager, ["--uninstall", packageName]); + const noEmulators = processResult(binaries.adb, ["devices"], "List of devices attached\n"); + const remove = async ( + harness: Awaited>, + failingFolder?: string, + alsoMake?: { readonly home: string; readonly imageTag: string }, + ) => { + await harness.driver.provision(harness.spec); + if (alsoMake !== undefined) { + await harness.filesystem.mkdirp( + `${sdk}/system-images/android-34/${alsoMake.imageTag}/arm64-v8a`, + ); + await writeSeedFiles(harness.filesystem, alsoMake.home); + await harness.driver.provision( + await harness.driver.resolveSpec({ + imageTag: alsoMake.imageTag, + model: "Pixel 5", + osVersion: "34", + platform: "android", + }), + ); + } + if (failingFolder !== undefined) harness.filesystem.defineFailure(failingFolder, "EACCES"); + const target = (await harness.driver.listComponents()).find( + (component) => component.variant === "google_apis/arm64-v8a", + ); + // why: the scripted runner only answers; real sdkmanager also deletes the image's folder. + const spawn = harness.runner.spawn.bind(harness.runner); + vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { + if (args[0] === "--uninstall") void harness.filesystem.rm(imagePath); + return spawn(command, args, options); + }); + await harness.driver.removeComponent(target?.receipt ?? {}, { + signal: new AbortController().signal, + }); + }; + + const destroyed = await seedHarness([...makingTheSeed(), uninstall, noEmulators]); + await remove(destroyed); + expect(await destroyed.filesystem.exists(seedHome)).toBe(false); + await expect(destroyed.driver.listSources()).resolves.toEqual([]); + + const otherHome = `${sources}/seed_device-4`; + const other = await seedHarness([ + ...makingTheSeed(), + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...makingASeedFrom(otherHome), + uninstall, + noEmulators, + ]); + await remove(other, undefined, { home: otherHome, imageTag: "google_apis_playstore" }); + expect(await other.filesystem.exists(seedHome)).toBe(false); + expect(await other.filesystem.exists(otherHome)).toBe(true); + + const stuck = await seedHarness([ + ...makingTheSeed(), + uninstall, + noEmulators, + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + await remove(stuck, seedHome); + await expect(stuck.driver.listSources()).resolves.toMatchObject([ + { id: "seed_two", staleReason: `system image ${packageName} is not installed` }, + ]); + }); + + it("boots a recovered copy cold from its disk on its recorded port: no snapshot is loaded and nothing is wiped, a second makeReady does not boot again and destroy stops it", async () => { + const harness = await seedHarness([ + ...bootingAt(copyHome, ["-no-snapshot-load"], true), + markWriteExpectation("emulator-5586", "one"), + ...bootingAt(copyHome, [], true).slice(1), + markWriteExpectation("emulator-5586", "two"), + copyPathExpectation("emulator-5586"), + processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), + ]); + await writeSeedFiles(harness.filesystem, copyHome); + + const recovered = { + address: "emulator-5586", + deviceId: "simlock_one", + driverData: { + avdHome: copyHome, + avdName: "simlock", + configHash: "hash", + port: 5586, + serial: "emulator-5586", + }, + }; + await harness.driver.makeReady(recovered, { purpose: "recover" }); + + const launch = harness.runner.calls.find((call) => call.args.includes("-avd")); + expect(launch?.args).toContain("-no-snapshot-load"); + expect(launch?.args).not.toContain("-snapshot"); + expect(launch?.args).not.toContain("-wipe-data"); + expect(launch?.options.env).toMatchObject({ ANDROID_AVD_HOME: copyHome }); + + await harness.driver.makeReady(recovered); + await harness.driver.destroy(recovered); + expect(emulatorLaunchArgs(harness.runner)).toHaveLength(1); + expect( + harness.runner.calls.some((call) => call.args.slice(-2).join(" ") === "emu kill"), + "destroy stops the emulator the recover boot started", + ).toBe(true); + }); + + it("boots a device made before copies once with a wipe: a second makeReady does not boot again and destroy stops it and deletes it through avdmanager", async () => { + const serial = "emulator-5586"; + const polls = (name: string) => [ + processResult( + binaries.adb, + ["-s", serial, "shell", "getprop", "sys.boot_completed"], + "1\n", + ), + processResult(binaries.adb, ["-s", serial, "shell", "getprop", "init.svc.bootanim"], ""), + avdPathExpectation(serial, name), + processResult( + binaries.adb, + ["-s", serial, "shell", "pm", "path", "android"], + "package:/system/framework/framework-res.apk\n", + ), + ]; + const harness = await seedHarness([ + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + { + hangs: true, + match: { + args: ["-avd", "simlock_old", ...seedEmulatorLine, "-wipe-data", "-no-snapshot-load"], + command: binaries.emulator, + }, + }, + ...polls("simlock_old"), + markWriteExpectation(serial, "one"), + ...polls("simlock_old"), + markWriteExpectation(serial, "two"), + avdPathExpectation(serial, "simlock_old"), + processResult(binaries.adb, ["-s", serial, "emu", "kill"]), + processResult(binaries.avdmanager, ["delete", "avd", "-n", "simlock_old"]), + ]); + await harness.filesystem.mkdirp(`${avdDirectory}/simlock_old.avd`); + await harness.filesystem.writeFileAtomic( + `${avdDirectory}/simlock_old.avd/config.ini`, + "hw.ramSize=2048\n", + ); + const legacy = { + address: "", + deviceId: "simlock_old", + driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, + }; + + await harness.driver.makeReady(legacy); + expect(emulatorLaunchArgs(harness.runner)[0]).toEqual( + expect.arrayContaining(["-wipe-data", "-no-snapshot-load"]), + ); + await harness.driver.makeReady(legacy); + await harness.driver.destroy(legacy); + + expect(emulatorLaunchArgs(harness.runner), "the wipe boot happens once").toHaveLength(1); + expect( + harness.runner.calls.some((call) => call.args.slice(-2).join(" ") === "emu kill"), + "destroy stops the emulator the wipe boot started", + ).toBe(true); + expect( + harness.runner.calls.some( + (call) => + call.command === binaries.avdmanager && + call.args.join(" ") === "delete avd -n simlock_old", + ), + "destroy deletes the device through avdmanager", + ).toBe(true); + }); + + it("destroys a plain Android device after its ownership check: the emulator is killed and its folder deleted, an unreadable check or a failing delete throws and keeps it, and a folder outside the root is never deleted", async () => { + const serial = "emulator-5586"; + const attached = `List of devices attached\n${serial}\tdevice\n`; + const own = { + address: serial, + deviceId: "simlock_one", + driverData: { + avdHome: copyHome, + avdName: "simlock", + configHash: "hash", + port: 5586, + serial, + }, + }; + const harness = async (expectations: readonly ScriptedProcessExpectation[]) => { + const filesystem = await androidFilesystem(); + for (const home of [copyHome, `${avdDirectory}/simlock_two`]) { + await filesystem.mkdirp(`${home}/simlock.avd`); + await filesystem.writeFileAtomic(`${home}/simlock.avd/config.ini`, "hw.ramSize=2048\n"); + } + const runner = new EmulatorExitingRunner([...expectations]); + return { driver: await createDriver(filesystem, runner), filesystem, runner }; + }; + const answersWith = (folder: string) => + processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${folder}/simlock.avd\r\nOK\r\n`, + ); + + const passes = await harness([ + processResult(binaries.adb, ["devices"], attached), + answersWith(copyHome), + processResult(binaries.adb, ["-s", serial, "emu", "kill"]), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ]); + await passes.driver.destroy(own); + expect(await passes.filesystem.exists(copyHome)).toBe(false); + expect(await passes.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); + + const neighbour = await harness([ + processResult( + binaries.adb, + ["devices"], + "List of devices attached\nemulator-5588\tdevice\n", + ), + processResult( + binaries.adb, + ["-s", "emulator-5588", "emu", "avd", "path"], + `${avdDirectory}/simlock_two/simlock.avd\r\nOK\r\n`, + ), + ]); + await neighbour.driver.destroy(own); + expect(await neighbour.filesystem.exists(copyHome)).toBe(false); + expect(neighbour.runner.calls.flatMap((call) => call.args)).not.toContain("kill"); + + const unreadable = await harness([ + processResult(binaries.adb, ["devices"], attached), + { + match: { args: ["-s", serial, "emu", "avd", "path"], command: binaries.adb }, + result: { code: 1, stderr: "error", stdout: "" }, + }, + ]); + await expect(unreadable.driver.destroy(own)).rejects.toThrow( + `Refusing to delete simlock_one: could not tell which AVD the emulator on ${serial} runs`, + ); + expect(await unreadable.filesystem.exists(copyHome)).toBe(true); + expect(unreadable.runner.calls.flatMap((call) => call.args)).not.toContain("kill"); + + const failingDelete = await harness([ + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ]); + failingDelete.filesystem.defineFailure(copyHome, "EACCES"); + await expect(failingDelete.driver.destroy(own)).rejects.toMatchObject({ code: "EACCES" }); + expect(await failingDelete.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); + + const outside = await harness([]); + await outside.filesystem.mkdirp("/elsewhere/simlock_two"); + await expect( + outside.driver.destroy({ + ...own, + deviceId: "simlock_two", + driverData: { ...own.driverData, avdHome: "/elsewhere/simlock_two" }, + }), + ).rejects.toThrow("Android device simlock_two names a folder outside the root"); + expect(await outside.filesystem.exists("/elsewhere/simlock_two")).toBe(true); + expect(await outside.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); + + const malformed = await harness([]); + await expect( + malformed.driver.destroy({ + ...own, + driverData: { ...own.driverData, avdHome: 5 }, + }), + ).rejects.toThrow("Android device simlock_one has invalid driver data"); + expect(await malformed.filesystem.exists(copyHome)).toBe(true); + + const old = await harness([ + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + processResult(binaries.avdmanager, ["delete", "avd", "-n", "simlock_old"]), + ]); + await old.driver.destroy({ + ...own, + deviceId: "simlock_old", + driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, + }); + expect(old.runner.calls[1]?.options.env).toMatchObject({ ANDROID_AVD_HOME: avdDirectory }); + + await expect(old.driver.reclaim(own, { clean: "standard" })).rejects.toMatchObject({ + message: "virtual devices are not reclaimed", + name: "DriverCrashError", + }); + }); + }); +}); diff --git a/src/drivers/android/index-device-tcp.test.ts b/src/drivers/android/index-device-tcp.test.ts new file mode 100644 index 000000000..5da00f5aa --- /dev/null +++ b/src/drivers/android/index-device-tcp.test.ts @@ -0,0 +1,469 @@ +import { describe, expect, it, vi } from "vitest"; +import { DeviceTcpError } from "../../core/index.js"; +import { promiseState } from "../../test-support/promise-state.js"; +import { + FakeClock, + FakeTcpListenerFactory, + FakeTcpProbe, + FakeTcpStreamConnector, + type ScriptedProcessExpectation, + type TcpStream, +} from "../../ports/index.js"; +import { + androidFilesystem, + binaries, + EmulatorExitingRunner, + processResult, + scopedEnv, + PHYSICAL_DEVICE, + provisionedHarness, + createDriver, + copyPathExpectation, +} from "./testing.js"; +import { avdPathExpectation } from "./test-index-lifecycle.js"; + +// why: stands in for adb's server: it answers the host protocol with the bytes a test scripts, and says nothing when there are none. +class RepliesTcpStreamConnector extends FakeTcpStreamConnector { + constructor(private readonly replies: string) { + super(); + } + + override async connect(port: number): Promise { + const stream = await super.connect(port); + if (this.replies !== "") this.connections.at(-1)?.peer.send(this.replies); + return stream; + } +} + +// why: stands in for a loopback port that accepts nothing and refuses nothing, so a connect to it never answers. +class HangingTcpStreamConnector extends FakeTcpStreamConnector { + override connect(): Promise { + return new Promise(() => undefined); + } +} + +describe("AndroidDriver deviceTcp", () => { + const emulator = { + address: "emulator-5586", + deviceId: "simlock_one", + driverData: { avdName: "simlock_one", configHash: "h", port: 5586, serial: "emulator-5586" }, + physical: false, + } as const; + const reverseArgs = (port: number) => [ + "reverse", + "--no-rebind", + "tcp:5173", + `tcp:${String(port)}`, + ]; + const ok = { code: 0, stderr: "", stdout: "" }; + const adb = (...args: string[]) => ({ args, command: binaries.adb }); + const signal = () => new AbortController().signal; + + async function driverWith( + expectations: ScriptedProcessExpectation[], + listeners = new FakeTcpListenerFactory(), + tcpProbe?: FakeTcpProbe, + ) { + const runner = new EmulatorExitingRunner(expectations); + const driver = await createDriver(await androidFilesystem(), runner, { + tcpListener: listeners, + ...(tcpProbe === undefined ? {} : { tcpProbe }), + }); + return { driver, listeners, runner }; + } + + // fallow-ignore-next-line complexity -- one Tests line: the driver's reverse, its close and each refusal are the cases of one scenario. + it("sets adb reverse --no-rebind to a loopback listener in 47000-47999 and removes it on close, as an emulator or a phone, keeps the port of a rule that could not be removed until the device's destroy (a phone's reclaim), and refuses with TUNNEL_PORT_IN_USE when adb cannot rebind and TUNNEL_FAILED for any other adb failure or when every tunnel port is busy, and opens a device port for a forward tunnel through the target's adb server (an emulator's configured one, a phone's after its check) with no adb forward rule, and rejects with the check's message when a phone's check is closed, with adb's reason when adb answers FAIL, and with \"timeout\" when adb does not answer in 5 s", async () => { + const { driver, listeners, runner } = await driverWith([ + { match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_002)), result: ok }, + { + match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), + result: ok, + }, + ]); + listeners.busy.add(47_000).add(47_001); + const reverse = await driver.deviceTcp?.reverse(emulator, 5173, signal()); + expect(reverse?.hostPort).toBe(47_002); + expect(runner.calls[0]?.options.env).toEqual(scopedEnv); + const arrived: unknown[] = []; + reverse?.onConnection((stream) => arrived.push(stream)); + listeners.listeners[0]?.connect(); + expect(arrived).toHaveLength(1); + await reverse?.close(); + expect(runner.calls[1]?.args).toEqual([ + "-P", + "5038", + "-s", + "emulator-5586", + "reverse", + "--remove", + "tcp:5173", + ]); + expect(listeners.listeners[0]?.closed).toBe(true); + + const probe = new FakeTcpProbe([5038, 5037]); + probe.replyWith("OKAY00040029"); + const phoneRun = await driverWith( + [ + { + match: adb("version"), + result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, + }, + { + match: adb("-P", "5037", "-s", PHYSICAL_DEVICE.address, ...reverseArgs(47_000)), + result: ok, + }, + ], + new FakeTcpListenerFactory(), + probe, + ); + const phone = await phoneRun.driver.deviceTcp?.reverse(PHYSICAL_DEVICE, 5173, signal()); + expect(phone?.hostPort).toBe(47_000); + expect(phoneRun.runner.calls[1]?.options.env?.ANDROID_ADB_SERVER_PORT).toBe("5037"); + + const stuck = await driverWith([ + { match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), result: ok }, + { + match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), + result: { code: 1, stderr: "device offline", stdout: "" }, + }, + { + match: adb( + "-P", + "5038", + "-s", + "emulator-5586", + "reverse", + "--no-rebind", + "tcp:5174", + "tcp:47001", + ), + result: ok, + }, + ]); + const held = await stuck.driver.deviceTcp?.reverse(emulator, 5173, signal()); + const delivered: unknown[] = []; + held?.onConnection((stream) => delivered.push(stream)); + await held?.close().catch(() => undefined); + expect(stuck.listeners.listeners[0]?.closed).toBe(false); + const late = stuck.listeners.listeners[0]?.connect(); + expect({ closed: late?.closed, delivered: delivered.length }).toEqual({ + closed: true, + delivered: 0, + }); + const second = await stuck.driver.deviceTcp?.reverse(emulator, 5174, signal()); + expect(second?.hostPort).toBe(47_001); + await stuck.driver.deviceTcp?.release(emulator); + expect(stuck.listeners.listeners.map((listener) => listener.closed)).toEqual([true, true]); + + const cannotRebind = await driverWith([ + { + match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), + result: { code: 1, stderr: "adb.exe: error: cannot rebind existing socket", stdout: "" }, + }, + ]); + await expect(cannotRebind.driver.deviceTcp?.reverse(emulator, 5173, signal())).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_PORT_IN_USE", + message: "reverse 5173: the device port is already reversed; remove that rule first", + }), + ); + expect(cannotRebind.listeners.listeners[0]?.closed).toBe(true); + + const failed = await driverWith([ + { + match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), + result: { code: 1, stderr: "x".repeat(300), stdout: "" }, + }, + ]); + const error = await failed.driver.deviceTcp + ?.reverse(emulator, 5173, signal()) + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(DeviceTcpError); + expect(error).toEqual( + expect.objectContaining({ + code: "TUNNEL_FAILED", + message: `reverse 5173: ${"x".repeat(200)}`, + }), + ); + expect(failed.listeners.listeners[0]?.closed).toBe(true); + + const busy = new FakeTcpListenerFactory(); + for (let port = 47_000; port < 48_000; port += 1) busy.busy.add(port); + const exhausted = await driverWith([], busy); + await expect(exhausted.driver.deviceTcp?.reverse(emulator, 5173, signal())).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_FAILED", + message: "no free tunnel port on the worker", + }), + ); + expect(exhausted.runner.calls).toEqual([]); + + const connectWith = async ( + device: typeof emulator | typeof PHYSICAL_DEVICE, + adbReplies: string, + options: { readonly probe?: FakeTcpProbe; readonly port?: number } = {}, + ) => { + const connector = new RepliesTcpStreamConnector(adbReplies); + const clock = new FakeClock(); + const runner = new EmulatorExitingRunner( + options.probe === undefined + ? [] + : [ + { + match: adb("version"), + result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, + }, + ], + ); + const driver = await createDriver(await androidFilesystem(), runner, { + clock, + ...(options.port === undefined ? {} : { driverConfig: { adbServerPort: options.port } }), + tcpListener: new FakeTcpListenerFactory(), + ...(options.probe === undefined ? {} : { tcpProbe: options.probe }), + tcpStreamConnector: connector, + }); + const connecting = driver.deviceTcp?.connect(device, 8081, signal()); + return { clock, connecting, connector, runner }; + }; + + const opened = await connectWith(emulator, "OKAYOKAY", { port: 5050 }); + await expect(opened.connecting).resolves.toBeDefined(); + expect(opened.connector.connections.map(({ port }) => port)).toEqual([5050]); + expect(opened.connector.connections[0]?.peer.writtenText()).toBe( + "001chost:transport:emulator-55860008tcp:8081", + ); + expect(opened.runner.calls).toEqual([]); + + const closedProbe = new FakeTcpProbe([5038, 5037]); + closedProbe.replyWith("OKAY00040028"); + const gated = await connectWith(PHYSICAL_DEVICE, "OKAYOKAY", { probe: closedProbe }); + await expect(gated.connecting).rejects.toThrow(/speaks protocol 40/); + expect(gated.connector.connections).toEqual([]); + + const refused = await connectWith(emulator, "OKAYFAIL0006closed"); + await expect(refused.connecting).rejects.toEqual( + expect.objectContaining({ code: "TUNNEL_FAILED", message: "closed" }), + ); + + const silent = await connectWith(emulator, ""); + // why: the connect is pending only once the socket is open and waiting for adb's answer; a timer alone is set before that. + await vi.waitFor(() => expect(silent.connector.connections).toHaveLength(1)); + await new Promise((resolve) => setImmediate(resolve)); + expect(silent.clock.pendingTimerCount).toBeGreaterThan(0); + silent.clock.advance(4_999); + await new Promise((resolve) => setImmediate(resolve)); + expect(silent.connector.connections[0]?.peer.closed).toBe(false); + silent.clock.advance(1); + await expect(silent.connecting).rejects.toEqual( + expect.objectContaining({ code: "TUNNEL_FAILED", message: "timeout" }), + ); + }); + + it("connects an emulator port to the worker port above the emulator's console port, refuses a phone, and rejects with the connector's reason when nothing listens and with \"timeout\" when nothing answers in 5 s", async () => { + const connector = new FakeTcpStreamConnector(); + const clock = new FakeClock(); + const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { + clock, + tcpListener: new FakeTcpListenerFactory(), + tcpStreamConnector: connector, + }); + const tcp = driver.deviceTcp; + const second = { ...emulator, address: "emulator-5682" }; + + expect(tcp?.supportsEmulatorPort(emulator)).toBe(true); + expect(tcp?.supportsEmulatorPort(PHYSICAL_DEVICE)).toBe(false); + await tcp?.connectEmulatorPort(emulator, "adb", signal()); + await tcp?.connectEmulatorPort(second, "adb", signal()); + expect(connector.connections.map(({ host, port }) => `${host}:${String(port)}`)).toEqual([ + "127.0.0.1:5587", + "127.0.0.1:5683", + ]); + + await expect(tcp?.connectEmulatorPort(PHYSICAL_DEVICE, "adb", signal())).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_UNSUPPORTED", + message: + "emulator adb: the lease is on an Android phone; emulator ports need an Android emulator", + }), + ); + await expect( + tcp?.connectEmulatorPort({ ...emulator, address: "" }, "adb", signal()), + ).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_FAILED", + message: "the emulator has no adb port yet", + }), + ); + + const bare = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { + clock, + tcpListener: new FakeTcpListenerFactory(), + }); + await expect(bare.deviceTcp?.connectEmulatorPort(emulator, "adb", signal())).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_FAILED", + message: "the emulator has no adb port yet", + }), + ); + + connector.refusedPorts.add(5587); + await expect(tcp?.connectEmulatorPort(emulator, "adb", signal())).rejects.toEqual( + expect.objectContaining({ + code: "TUNNEL_FAILED", + message: "connect ECONNREFUSED 127.0.0.1:5587", + }), + ); + + const silent = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { + clock, + tcpListener: new FakeTcpListenerFactory(), + tcpStreamConnector: new HangingTcpStreamConnector(), + }); + const waiting = Promise.resolve( + silent.deviceTcp?.connectEmulatorPort(emulator, "adb", signal()), + ); + const state = promiseState(waiting); + clock.advance(4_999); + await new Promise((resolve) => setImmediate(resolve)); + expect(state.state).toBe("pending"); + clock.advance(1); + await expect(waiting).rejects.toEqual( + expect.objectContaining({ code: "TUNNEL_FAILED", message: "timeout" }), + ); + }); + + it("sweeps at start only the reverse rules into 47000-47999 on an idle device proved its own, only reads a leased device's rules and keeps their ports from the next reverse until its destroy, touches nothing on an emulator that answers as another's, and on an idle phone removes nothing and keeps the ports its rules point to from the next reverse", async () => { + const own = await driverWith([ + avdPathExpectation("emulator-5586"), + { + match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--list"), + result: { + code: 0, + stderr: "", + stdout: + "host-19 tcp:5173 tcp:47003\nhost-19 tcp:8081 tcp:8082\nhost-19 tcp:3000 tcp:47999\n", + }, + }, + { + match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), + result: ok, + }, + { + match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:3000"), + result: ok, + }, + ]); + await own.driver.deviceTcp?.sweep([{ device: emulator, leased: false }]); + expect(own.runner.calls.filter((call) => call.args.includes("--remove"))).toHaveLength(2); + + const probe = new FakeTcpProbe([5038, 5037]); + probe.replyWith("OKAY00040029"); + const phones = await driverWith( + [ + { + match: adb("version"), + result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, + }, + { + match: adb("-P", "5037", "-s", PHYSICAL_DEVICE.address, "reverse", "--list"), + result: { + code: 0, + stderr: "", + stdout: "UsbFfs tcp:5173 tcp:47000\nUsbFfs tcp:3000 tcp:3001\n", + }, + }, + { + match: adb( + "-P", + "5037", + "-s", + PHYSICAL_DEVICE.address, + "reverse", + "--no-rebind", + "tcp:5174", + "tcp:47001", + ), + result: ok, + }, + ], + new FakeTcpListenerFactory(), + probe, + ); + await phones.driver.deviceTcp?.sweep([{ device: PHYSICAL_DEVICE, leased: false }]); + const onPhone = await phones.driver.deviceTcp?.reverse(PHYSICAL_DEVICE, 5174, signal()); + expect(onPhone?.hostPort).toBe(47_001); + expect(phones.runner.calls.some((call) => call.args.includes("--remove"))).toBe(false); + + const foreign = await driverWith([avdPathExpectation("emulator-5586", "simlock_two")]); + await foreign.driver.deviceTcp?.sweep([{ device: emulator, leased: false }]); + expect(foreign.runner.calls.some((call) => call.args.includes("reverse"))).toBe(false); + + const onServer = ["-P", "5038", "-s", "emulator-5586"]; + const leased = await driverWith([ + avdPathExpectation("emulator-5586"), + { + match: adb(...onServer, "reverse", "--list"), + result: { + code: 0, + stderr: "", + stdout: "host-19 tcp:5173 tcp:47000\nhost-19 tcp:3000 tcp:3001\n", + }, + }, + { + match: adb(...onServer, "reverse", "--no-rebind", "tcp:5174", "tcp:47001"), + result: ok, + }, + { + match: adb(...onServer, "reverse", "--no-rebind", "tcp:5175", "tcp:47000"), + result: ok, + }, + ]); + await leased.driver.deviceTcp?.sweep([{ device: emulator, leased: true }]); + expect(leased.runner.calls.filter((call) => call.args.includes("--remove"))).toEqual([]); + const next = await leased.driver.deviceTcp?.reverse(emulator, 5174, signal()); + expect(next?.hostPort).toBe(47_001); + await leased.driver.deviceTcp?.release(emulator); + const afterReclaim = await leased.driver.deviceTcp?.reverse(emulator, 5175, signal()); + expect(afterReclaim?.hostPort).toBe(47_000); + }); +}); + +describe("AndroidDriver tunnels of a destroyed device", () => { + it("closes the loopback listeners held for a virtual device's tunnels when the device is destroyed", async () => { + const listeners = new FakeTcpListenerFactory(); + const harness = await provisionedHarness({ + afterwards: [ + processResult(binaries.adb, [ + "-P", + "5038", + "-s", + "emulator-5586", + "reverse", + "--no-rebind", + "tcp:5173", + "tcp:47000", + ]), + copyPathExpectation("emulator-5586"), + processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), + ], + tcpListener: listeners, + }); + const ready = await harness.driver.makeReady(harness.device); + await harness.driver.deviceTcp?.reverse( + { ...ready, physical: false }, + 5173, + new AbortController().signal, + ); + expect(listeners.listeners.map((listener) => listener.closed)).toEqual([false]); + const launch = harness.runner.calls.findLastIndex( + (call) => call.command === binaries.emulator && call.args.includes("-avd"), + ); + + const destroying = harness.driver.destroy(ready); + harness.runner.handles[launch]?.kill("SIGTERM"); + await destroying; + + expect(listeners.listeners.map((listener) => listener.closed)).toEqual([true]); + }); +}); diff --git a/src/drivers/android/index-image-fit.test.ts b/src/drivers/android/index-image-fit.test.ts new file mode 100644 index 000000000..4ee535dce --- /dev/null +++ b/src/drivers/android/index-image-fit.test.ts @@ -0,0 +1,583 @@ +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { + FakeClock, + MemoryFilesystem, + NodeFilesystem, + NodeProcessRunner, + NodeProcessSupervisor, + NodeTcpProbe, + type ScriptedProcessExpectation, + SystemClock, +} from "../../ports/index.js"; +import { + AndroidDriver, + type AndroidDriverDiagnostic, + type AndroidEmulatorLaunchOptions, +} from "./index.js"; +import { + androidFilesystem, + avdDirectory, + binaries, + bootingAt, + markWriteExpectation, + EmulatorExitingRunner, + makingTheSeed, + seedStages, + writeSeedFiles, + memoryAndroidSourceDatabase, + pixelDevices, + processResult, + createDriver, +} from "./testing.js"; +import { sdk, home } from "./test-index.js"; + +/** why: copies work until a test arms the failure, so a device can be provisioned from a seed first. */ +class ArmedCopyFailureFilesystem extends MemoryFilesystem { + armed = false; + + constructor(private readonly message: string) { + super(); + } + + override copyTree( + from: string, + to: string, + options: { readonly timeoutMs: number }, + ): Promise { + return this.armed ? Promise.reject(new Error(this.message)) : super.copyTree(from, to, options); + } +} + +const live = process.env.SIMLOCK_LIVE_ANDROID === "1" ? it : it.skip; + +live( + "live smoke: provision, boot from the boot snapshot, restart, and destroy", + async () => { + const driver = await AndroidDriver.create({ + clock: new SystemClock(), + database: await memoryAndroidSourceDatabase(), + driverConfig: {}, + env: process.env, + filesystem: new NodeFilesystem(), + homeDirectory: process.env.HOME ?? home, + hostAbi: "arm64-v8a", + instanceId: `live-${process.pid}`, + processRunner: new NodeProcessRunner(), + processSupervisor: new NodeProcessSupervisor({ + platform: process.platform, + processRunner: new NodeProcessRunner(), + }), + simlockHome: join(tmpdir(), `simlock-live-android-${process.pid}`), + tcpProbe: new NodeTcpProbe(), + }); + const spec = await driver.resolveSpec({ + model: process.env.SIMLOCK_LIVE_ANDROID_MODEL ?? "Pixel 5", + platform: "android", + ...(process.env.SIMLOCK_LIVE_ANDROID_API === undefined + ? {} + : { osVersion: process.env.SIMLOCK_LIVE_ANDROID_API }), + }); + const device = await driver.provision(spec); + + try { + const ready = await driver.makeReady(device); + await driver.shutdown(ready); + const restarted = await driver.makeReady(ready); + + expect(restarted.address).toMatch(/^emulator-\d+$/); + } finally { + await driver.destroy(device); + } + }, + 480_000, +); + +describe("AndroidDriver image fit", () => { + describe("seeds and copies", () => { + const sources = `${avdDirectory}/sources`; + const copyHome = `${avdDirectory}/simlock_one`; + const seedHome = `${sources}/seed_two`; + async function seedHarness( + expectations: readonly ScriptedProcessExpectation[], + options: { + readonly clock?: FakeClock; + readonly config?: string; + readonly filesystem?: MemoryFilesystem; + readonly ids?: readonly string[]; + readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; + } = {}, + ) { + const filesystem = await androidFilesystem({}, options.filesystem); + await writeSeedFiles(filesystem, seedHome, options.config); + const database = await memoryAndroidSourceDatabase(); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...expectations, + ]); + const driver = await createDriver(filesystem, runner, { + ...(options.clock === undefined ? {} : { clock: options.clock }), + ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }), + database, + ids: options.ids ?? ["one", "two", "three"], + }); + const spec = await driver.resolveSpec({ + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + return { database, driver, filesystem, runner, spec }; + } + + /** why: the daemon after a restart: a new driver over the same device folders and the same source table. */ + async function reopened( + harness: Awaited>, + expectations: readonly ScriptedProcessExpectation[], + options: { + readonly clock?: FakeClock; + readonly emulator?: AndroidEmulatorLaunchOptions; + } = {}, + ) { + const runner = new EmulatorExitingRunner([...expectations]); + const driver = await createDriver(harness.filesystem, runner, { + ...options, + database: harness.database, + ids: ["four", "five", "six"], + }); + return { driver, runner }; + } + + it("hands the seeds part its clock, its source table, its device root and its id generator", async () => { + const clock = new FakeClock(5_000); + const harness = await seedHarness(makingTheSeed(), { clock }); + + await harness.driver.provision(harness.spec); + + expect( + harness.database + .prepare("SELECT source_id, folder, created_at FROM android_source_data") + .all(), + ).toEqual([{ created_at: 5_000, folder: seedHome, source_id: "seed_two" }]); + }); + + it("hands the avd config the emulator launch settings, so a seed made under other ones is stale", async () => { + const harness = await seedHarness(makingTheSeed()); + await harness.driver.provision(harness.spec); + + const restarted = await reopened( + harness, + [processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9")], + { emulator: { audio: true, bootAnimation: true, gpu: "auto", headless: true } }, + ); + + await expect(restarted.driver.listSources()).resolves.toMatchObject([ + { id: "seed_two", staleReason: "android.emulator launch settings changed" }, + ]); + }); + + it("leases a plain Android device from a copy of the kind's seed, made first when there is none, booted from the boot snapshot", async () => { + const harness = await seedHarness([ + ...makingTheSeed(), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true), + markWriteExpectation("emulator-5586", "three"), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + + const device = await harness.driver.provision(harness.spec); + const ready = await harness.driver.makeReady(device); + + const create = harness.runner.calls.find((call) => call.args[0] === "create"); + expect(create?.options.env).toMatchObject({ ANDROID_AVD_HOME: seedHome }); + expect(device).toMatchObject({ + deviceId: "simlock_one", + driverData: { avdHome: copyHome, avdName: "simlock" }, + }); + await expect(harness.filesystem.readFile(`${copyHome}/simlock.avd/config.ini`)).resolves.toBe( + "hw.ramSize=2048\nsimlock.mark=three\n", + ); + await expect(harness.filesystem.readFile(`${copyHome}/simlock.ini`)).resolves.toBe( + `avd.ini.encoding=UTF-8\npath=${copyHome}/simlock.avd\ntarget=android-34\n`, + ); + await expect(harness.driver.listSources()).resolves.toMatchObject([ + { + id: "seed_two", + imageTag: "google_apis", + kind: "seed", + model: "Pixel 5", + osVersion: "34", + }, + ]); + const launches = harness.runner.calls.filter((call) => call.args.includes("-avd")); + expect(launches.map((call) => call.options.env?.["ANDROID_AVD_HOME"])).toEqual([ + seedHome, + seedHome, + copyHome, + ]); + expect(launches.flatMap((call) => call.args)).not.toContain("-wipe-data"); + const shellRuns = harness.runner.calls.filter( + (call) => call.args.includes("sync") || call.args.includes("-p"), + ); + expect( + shellRuns.map((call) => [call.args.at(-1), call.options.timeoutMs]), + "the seed's sync and power-off are each bounded, so a wedged adbd cannot hold the make", + ).toEqual([ + ["sync", 60_000], + ["-p", 60_000], + ]); + expect(ready.address).toBe("emulator-5586"); + }); + + it("reports a copy whose boot snapshot does not load by its device id, only when it took over 12 s, and never wipes it", async () => { + for (const { readyAfterMs, reported } of [ + { readyAfterMs: 12_000, reported: false }, + { readyAfterMs: 12_001, reported: true }, + ]) { + const clock = new FakeClock(); + const diagnostics: AndroidDriverDiagnostic[] = []; + const harness = await seedHarness( + [ + ...makingTheSeed(), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true, 1), + markWriteExpectation("emulator-5586", "three"), + ], + { clock, onDiagnostic: (diagnostic) => diagnostics.push(diagnostic) }, + ); + const device = await harness.driver.provision(harness.spec); + + const ready = harness.driver.makeReady(device); + await vi.waitFor(() => + expect( + harness.runner.calls.filter((call) => call.args.includes("sys.boot_completed")), + ).toHaveLength(3), + ); + clock.advance(readyAfterMs); + await ready; + + expect(diagnostics).toEqual( + reported ? [{ deviceId: "simlock_one", kind: "snapshot-cold-boot", readyAfterMs }] : [], + ); + expect(harness.runner.calls.flatMap((call) => call.args)).not.toContain("-wipe-data"); + } + }); + + describe("saveSource", () => { + const savedHome = `${sources}/baseline_four`; + const serial = "emulator-5586"; + const config = + "hw.device.name=pixel_5\nimage.sysdir.1=system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize=2048\n"; + const ids = ["one", "two", "three", "four"]; + const adbShell = (command: string) => + processResult(binaries.adb, ["-s", serial, "shell", command]); + const sourceBoot = () => { + const stage = seedStages(savedHome); + return [ + stage.devices, + ...stage.secondBoot, + stage.save, + stage.list, + stage.probe, + stage.kill, + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]; + }; + const leasing = (hangs: boolean) => [ + ...makingTheSeed(), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], hangs), + markWriteExpectation(serial, "three"), + ]; + const probe = processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${copyHome}/simlock.avd\r\nOK\r\n`, + ); + const savingSteps = [ + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + probe, + adbShell("sync"), + processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), + ]; + + it("provisions a device named after a baseline source as a copy of that source's folder with no seed made and only the emulator's version read, and refuses a source whose system image is gone", async () => { + const version = () => + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"); + const harness = await seedHarness( + [...leasing(true), ...savingSteps, ...sourceBoot(), version(), version()], + { + config, + ids: [...ids, "five", "six"], + }, + ); + const ready = await harness.driver.makeReady(await harness.driver.provision(harness.spec)); + await harness.driver.saveSource({ ...ready, physical: false as const }); + await harness.filesystem.writeFileAtomic(`${savedHome}/simlock.avd/marker.txt`, "saved"); + const callsBefore = harness.runner.calls.length; + + const copy = await harness.driver.provision(harness.spec, { source: "baseline_four" }); + + expect(copy.deviceId).toBe("simlock_five"); + await expect( + harness.filesystem.readFile(`${avdDirectory}/simlock_five/simlock.avd/marker.txt`), + ).resolves.toBe("saved"); + expect(harness.runner.calls).toHaveLength(callsBefore + 1); + + await harness.filesystem.rm(`${sdk}/system-images/android-34/google_apis/arm64-v8a`); + await expect( + harness.driver.provision(harness.spec, { source: "baseline_four" }), + ).rejects.toThrow( + "source baseline_four is stale: system image system-images;android-34;google_apis;arm64-v8a is not installed", + ); + expect(await harness.filesystem.exists(`${avdDirectory}/simlock_six`)).toBe(false); + }); + + it("keeps a powered-off copy of the device as a ready baseline source with one boot snapshot, powering the emulator off first, or killing it when it will not leave in 60 s, or polling adb with no process handle", async () => { + const kept = async (harness: Awaited>) => { + expect(await harness.filesystem.exists(`${savedHome}/simlock.avd/config.ini`)).toBe(true); + await expect(harness.filesystem.readFile(`${savedHome}/simlock.ini`)).resolves.toContain( + `path=${savedHome}/simlock.avd`, + ); + expect(await harness.filesystem.exists(copyHome)).toBe(true); + const listed = await harness.driver.listSources(); + expect(listed.find((source) => source.id === "baseline_four")).toMatchObject({ + imageTag: "google_apis", + kind: "baseline", + model: "pixel_5", + osVersion: "34", + }); + expect( + harness.database + .prepare("SELECT state FROM android_source_data WHERE source_id = 'baseline_four'") + .all(), + ).toEqual([{ state: "ready" }]); + }; + const sent = (harness: Awaited>): string[] => + harness.runner.calls + .filter((call) => call.command === binaries.adb && call.args[0] === "-s") + .map((call) => call.args.slice(2).join(" ")); + const saveLaunch = (harness: Awaited>) => + harness.runner.calls + .filter((call) => call.args.includes("-avd")) + .map((call) => [call.options.env?.["ANDROID_AVD_HOME"], call.args.at(-1)]); + + const powered = await seedHarness([...leasing(true), ...savingSteps, ...sourceBoot()], { + config, + ids, + }); + const poweredReady = await powered.driver.makeReady( + await powered.driver.provision(powered.spec), + ); + const poweredSaved = await powered.driver.saveSource({ + ...poweredReady, + physical: false as const, + }); + expect(poweredSaved).toEqual({ + imageTag: "google_apis", + sizeBytes: 121, + sourceId: "baseline_four", + }); + await kept(powered); + expect(sent(powered)).toEqual( + expect.arrayContaining([ + "shell sync", + "shell reboot -p", + "emu avd snapshot save simlock_boot", + ]), + ); + expect(sent(powered).indexOf("shell reboot -p")).toBeLessThan( + sent(powered).lastIndexOf("emu avd snapshot save simlock_boot"), + ); + expect(saveLaunch(powered).at(-1)).toEqual([savedHome, "-no-snapshot-load"]); + + const clock = new FakeClock(); + const stuck = await seedHarness( + [ + ...leasing(true), + ...savingSteps, + processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${copyHome}/simlock.avd\r\nOK\r\n`, + ), + processResult(binaries.adb, ["-s", serial, "emu", "kill"]), + ...sourceBoot(), + ], + { clock, config, ids }, + ); + const stuckReady = await stuck.driver.makeReady(await stuck.driver.provision(stuck.spec)); + // why: the stuck emulator ignores `reboot -p` and `emu kill`; only the deadlines and a SIGKILL end it. + stuck.runner.ignoresEmuKill = true; + const stuckSaving = stuck.driver.saveSource({ ...stuckReady, physical: false as const }); + await vi.waitFor(() => + expect(stuck.runner.calls.filter((call) => call.args.includes("reboot"))).toHaveLength(2), + ); + clock.advance(60_000); + await vi.waitFor(() => + expect(stuck.runner.calls.filter((call) => call.args.includes("kill"))).toHaveLength(2), + ); + stuck.runner.ignoresEmuKill = false; + let stuckDone = false; + void stuckSaving.then( + () => (stuckDone = true), + () => (stuckDone = true), + ); + await vi.waitFor(() => { + clock.advance(60_000); + expect(stuckDone).toBe(true); + }); + await expect(stuckSaving).resolves.toMatchObject({ sourceId: "baseline_four" }); + await kept(stuck); + expect(sent(stuck).slice(sent(stuck).indexOf("shell reboot -p"))).toEqual( + expect.arrayContaining(["emu avd path", "emu kill"]), + ); + + const restarted = await seedHarness(leasing(true), { config, ids }); + const restartedReady = await restarted.driver.makeReady( + await restarted.driver.provision(restarted.spec), + ); + const clockAfter = new FakeClock(); + const adopted = await reopened( + restarted, + [ + ...savingSteps, + processResult( + binaries.adb, + ["devices"], + `List of devices attached\n${serial}\tdevice\n`, + ), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ...sourceBoot(), + ], + { clock: clockAfter }, + ); + const adoptedSaving = adopted.driver.saveSource({ + ...restartedReady, + physical: false as const, + }); + await vi.waitFor(() => + expect(adopted.runner.calls.filter((call) => call.args[0] === "devices")).toHaveLength(1), + ); + clockAfter.advance(1_000); + await expect(adoptedSaving).resolves.toMatchObject({ sourceId: "baseline_four" }); + await kept({ ...restarted, driver: adopted.driver }); + }); + + it("saves a device whose config.ini has spaces around `=`, as avdmanager writes it, with the model and image it names", async () => { + const spaced = + "hw.device.name = pixel_6\nimage.sysdir.1 = system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize = 2048\n"; + const harness = await seedHarness([...leasing(true), ...savingSteps, ...sourceBoot()], { + config: spaced, + ids, + }); + const ready = await harness.driver.makeReady(await harness.driver.provision(harness.spec)); + + await expect( + harness.driver.saveSource({ ...ready, physical: false as const }), + ).resolves.toMatchObject({ imageTag: "google_apis", sourceId: "baseline_four" }); + + expect( + (await harness.driver.listSources()).find((source) => source.id === "baseline_four"), + ).toMatchObject({ + imageTag: "google_apis", + kind: "baseline", + model: "pixel_6", + osVersion: "34", + }); + }); + }); + + describe("saveSource failures", () => { + const savedHome = `${sources}/baseline_four`; + const serial = "emulator-5586"; + const config = + "hw.device.name=pixel_5\nimage.sysdir.1=system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize=2048\n"; + const version = processResult( + binaries.emulator, + ["-version"], + "Android emulator version 36.1.9", + ); + const probe = (path = `${copyHome}/simlock.avd`) => + processResult(binaries.adb, ["-s", serial, "emu", "avd", "path"], `${path}\r\nOK\r\n`); + const poweredOff = [ + version, + probe(), + processResult(binaries.adb, ["-s", serial, "shell", "sync"]), + processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), + ]; + const stage = seedStages(savedHome); + + it("deletes the source's folder and row and throws when the ownership check fails, the copy fails or the boot snapshot is not saved, leaving the device as it was", async () => { + const cases = [ + { + expectations: [version, probe(`${avdDirectory}/other.avd`)], + message: "emulator-5586 is not its emulator", + sent: false, + }, + { + copyFails: true, + expectations: poweredOff, + message: "copying Android device simlock_one to source baseline_four failed: no space", + }, + { + expectations: [ + ...poweredOff, + stage.devices, + ...stage.secondBoot, + { ...stage.save, result: { code: 1, stderr: "emulator gone", stdout: "" } }, + ], + message: "Android seed baseline_four: boot snapshot simlock_boot was not saved", + }, + ] as const; + + for (const failing of cases) { + const filesystem = new ArmedCopyFailureFilesystem("no space"); + const harness = await seedHarness( + [ + ...makingTheSeed(), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true), + markWriteExpectation(serial, "three"), + ...failing.expectations, + ], + { config, filesystem, ids: ["one", "two", "three", "four"] }, + ); + const ready = await harness.driver.makeReady( + await harness.driver.provision(harness.spec), + ); + filesystem.armed = "copyFails" in failing; + const before = harness.runner.calls.length; + + await expect( + harness.driver.saveSource({ ...ready, physical: false as const }), + ).rejects.toThrow(failing.message); + + const sentToDevice = harness.runner.calls + .slice(before) + .filter((call) => call.args.includes("sync") || call.args.includes("reboot")); + expect(sentToDevice.length > 0, failing.message).toBe(!("sent" in failing)); + expect(await harness.filesystem.exists(savedHome), failing.message).toBe(false); + expect(await harness.filesystem.exists(copyHome), failing.message).toBe(true); + expect( + harness.database + .prepare("SELECT source_id FROM android_source_data WHERE kind = 'baseline'") + .all(), + failing.message, + ).toEqual([]); + const emulators = harness.runner.calls.flatMap((call, index) => + call.args.includes("-avd") ? [index] : [], + ); + // why: the first two launches make the seed and the third is the device's own, which a failure before `reboot -p` leaves running; what follows is the source's boot. + for (const emulator of emulators.slice(3)) { + let exited = false; + void harness.runner.handles[emulator]?.wait().then(() => { + exited = true; + }); + await vi.waitFor(() => expect(exited, "the source's emulator was stopped").toBe(true)); + } + } + }); + }); + }); +}); diff --git a/src/drivers/android/index-lifecycle.test.ts b/src/drivers/android/index-lifecycle.test.ts index 407f15615..88e8ae3e3 100644 --- a/src/drivers/android/index-lifecycle.test.ts +++ b/src/drivers/android/index-lifecycle.test.ts @@ -1,64 +1,30 @@ -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { DeviceTcpError } from "../../core/index.js"; import { promiseState } from "../../test-support/promise-state.js"; import { FakeClock, - FakeTcpListenerFactory, FakeTcpProbe, - FakeTcpStreamConnector, MemoryFilesystem, - NodeFilesystem, - NodeProcessRunner, - NodeProcessSupervisor, - NodeTcpProbe, type ProcessResult, type ProcessRunner, - type ScriptedProcessExpectation, - SystemClock, - type TcpStream, } from "../../ports/index.js"; -import { - AndroidDriver, - type AndroidDriverDiagnostic, - type AndroidEmulatorLaunchOptions, -} from "./index.js"; import { answeringAvdPath, androidFilesystem, avdDirectory, binaries, - bootingAt, bootProbes, - makingASeedFrom, - markWriteExpectation, EmulatorExitingRunner, - makingTheSeed, - seedEmulatorLine, - seedStages, - writeSeedFiles, - memoryAndroidSourceDatabase, - pixelDevices, - processResult, adbServerPort, scopedEnv, - PHYSICAL_DEVICE, - emulatorLaunchArgs, provisionedHarness, createDriver, EmulatorHost, RecordingRunner, consolePortOf, ok, - copyPathExpectation, WorkingDirectoryFilesystem, - ANDROID_SDK_ROOT, - ANDROID_HOME_DIRECTORY, } from "./testing.js"; - -const sdk = ANDROID_SDK_ROOT; -const home = ANDROID_HOME_DIRECTORY; +import { home } from "./test-index.js"; // why: The emulator outlives the daemon, so it is spawned detached; its output goes to the log file in its AVD folder. const emulatorOptions = { @@ -66,451 +32,178 @@ const emulatorOptions = { stdio: { file: `${avdDirectory}/simlock_one/simlock.avd/emulator.log` }, } as const; -// why: stands in for adb's server: it answers the host protocol with the bytes a test scripts, and says nothing when there are none. -class RepliesTcpStreamConnector extends FakeTcpStreamConnector { - constructor(private readonly replies: string) { - super(); - } - - override async connect(port: number): Promise { - const stream = await super.connect(port); - if (this.replies !== "") this.connections.at(-1)?.peer.send(this.replies); - return stream; - } +function recordKillTimeout( + into: (number | undefined)[], + command: string, + args: readonly string[], + timeoutMs: number | undefined, +): void { + if (command === binaries.adb && args.slice(2).join(" ") === "emu kill") into.push(timeoutMs); } -// why: stands in for a loopback port that accepts nothing and refuses nothing, so a connect to it never answers. -class HangingTcpStreamConnector extends FakeTcpStreamConnector { - override connect(): Promise { - return new Promise(() => undefined); +/** why: tells a test what is running when a path is removed, so "stopped before deleted" is observed, not assumed. */ +class RemovalObservingFilesystem extends MemoryFilesystem { + onRemove: (path: string) => void = () => undefined; + + override async rm(path: string): Promise { + this.onRemove(path); + await super.rm(path); } } -describe("AndroidDriver deviceTcp", () => { - const emulator = { - address: "emulator-5586", - deviceId: "simlock_one", - driverData: { avdName: "simlock_one", configHash: "h", port: 5586, serial: "emulator-5586" }, - physical: false, - } as const; - const reverseArgs = (port: number) => [ - "reverse", - "--no-rebind", - "tcp:5173", - `tcp:${String(port)}`, - ]; - const ok = { code: 0, stderr: "", stdout: "" }; - const adb = (...args: string[]) => ({ args, command: binaries.adb }); - const signal = () => new AbortController().signal; - - async function driverWith( - expectations: ScriptedProcessExpectation[], - listeners = new FakeTcpListenerFactory(), - tcpProbe?: FakeTcpProbe, - ) { - const runner = new EmulatorExitingRunner(expectations); - const driver = await createDriver(await androidFilesystem(), runner, { - tcpListener: listeners, - ...(tcpProbe === undefined ? {} : { tcpProbe }), - }); - return { driver, listeners, runner }; +/** + * why: An `EmulatorHost` on which another emulator takes a device's console port just before the + * device's own emulator is spawned on it, and which answers the queries that tell emulators + * apart: `emu avd path`, `emu avd name` and `getprop ro.boot.qemu.avd_name`. + * + * The device's emulator cannot bind the taken port and exits -- but not before the other + * emulator has answered the readiness poll, as it does on a real machine, where the other + * emulator answers at once and the spawned one checks its ports only once it has started. + * So in this fake its exit is seen only once it is killed: a driver that waited for that + * exit would already have readied the device. + */ +class PortTakingMachine { + readonly #inner: ProcessRunner; + readonly #otherPaths = new Map(); + #takeOnNextBoot: ((port: number) => void) | undefined; + + constructor(private readonly host: EmulatorHost) { + this.#inner = host.runner(); } - // fallow-ignore-next-line complexity -- one Tests line: the driver's reverse, its close and each refusal are the cases of one scenario. - it("sets adb reverse --no-rebind to a loopback listener in 47000-47999 and removes it on close, as an emulator or a phone, keeps the port of a rule that could not be removed until the device's destroy (a phone's reclaim), and refuses with TUNNEL_PORT_IN_USE when adb cannot rebind and TUNNEL_FAILED for any other adb failure or when every tunnel port is busy, and opens a device port for a forward tunnel through the target's adb server (an emulator's configured one, a phone's after its check) with no adb forward rule, and rejects with the check's message when a phone's check is closed, with adb's reason when adb answers FAIL, and with \"timeout\" when adb does not answer in 5 s", async () => { - const { driver, listeners, runner } = await driverWith([ - { match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_002)), result: ok }, - { - match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), - result: ok, - }, - ]); - listeners.busy.add(47_000).add(47_001); - const reverse = await driver.deviceTcp?.reverse(emulator, 5173, signal()); - expect(reverse?.hostPort).toBe(47_002); - expect(runner.calls[0]?.options.env).toEqual(scopedEnv); - const arrived: unknown[] = []; - reverse?.onConnection((stream) => arrived.push(stream)); - listeners.listeners[0]?.connect(); - expect(arrived).toHaveLength(1); - await reverse?.close(); - expect(runner.calls[1]?.args).toEqual([ - "-P", - "5038", - "-s", - "emulator-5586", - "reverse", - "--remove", - "tcp:5173", - ]); - expect(listeners.listeners[0]?.closed).toBe(true); - - const probe = new FakeTcpProbe([5038, 5037]); - probe.replyWith("OKAY00040029"); - const phoneRun = await driverWith( - [ - { - match: adb("version"), - result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, - }, - { - match: adb("-P", "5037", "-s", PHYSICAL_DEVICE.address, ...reverseArgs(47_000)), - result: ok, - }, - ], - new FakeTcpListenerFactory(), - probe, - ); - const phone = await phoneRun.driver.deviceTcp?.reverse(PHYSICAL_DEVICE, 5173, signal()); - expect(phone?.hostPort).toBe(47_000); - expect(phoneRun.runner.calls[1]?.options.env?.ANDROID_ADB_SERVER_PORT).toBe("5037"); - - const stuck = await driverWith([ - { match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), result: ok }, - { - match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), - result: { code: 1, stderr: "device offline", stdout: "" }, - }, - { - match: adb( - "-P", - "5038", - "-s", - "emulator-5586", - "reverse", - "--no-rebind", - "tcp:5174", - "tcp:47001", - ), - result: ok, - }, - ]); - const held = await stuck.driver.deviceTcp?.reverse(emulator, 5173, signal()); - const delivered: unknown[] = []; - held?.onConnection((stream) => delivered.push(stream)); - await held?.close().catch(() => undefined); - expect(stuck.listeners.listeners[0]?.closed).toBe(false); - const late = stuck.listeners.listeners[0]?.connect(); - expect({ closed: late?.closed, delivered: delivered.length }).toEqual({ - closed: true, - delivered: 0, - }); - const second = await stuck.driver.deviceTcp?.reverse(emulator, 5174, signal()); - expect(second?.hostPort).toBe(47_001); - await stuck.driver.deviceTcp?.release(emulator); - expect(stuck.listeners.listeners.map((listener) => listener.closed)).toEqual([true, true]); - - const cannotRebind = await driverWith([ - { - match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), - result: { code: 1, stderr: "adb.exe: error: cannot rebind existing socket", stdout: "" }, - }, - ]); - await expect(cannotRebind.driver.deviceTcp?.reverse(emulator, 5173, signal())).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_PORT_IN_USE", - message: "reverse 5173: the device port is already reversed; remove that rule first", - }), - ); - expect(cannotRebind.listeners.listeners[0]?.closed).toBe(true); + avdPathAnswering(serial: string | undefined): string | undefined { + const path = this.host.pathAnswering(serial); + if (path === undefined) return undefined; + return this.#otherPaths.get(consolePortOf(serial)) ?? path; + } - const failed = await driverWith([ - { - match: adb("-P", "5038", "-s", "emulator-5586", ...reverseArgs(47_000)), - result: { code: 1, stderr: "x".repeat(300), stdout: "" }, - }, - ]); - const error = await failed.driver.deviceTcp - ?.reverse(emulator, 5173, signal()) - .catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(DeviceTcpError); - expect(error).toEqual( - expect.objectContaining({ - code: "TUNNEL_FAILED", - message: `reverse 5173: ${"x".repeat(200)}`, - }), - ); - expect(failed.listeners.listeners[0]?.closed).toBe(true); - - const busy = new FakeTcpListenerFactory(); - for (let port = 47_000; port < 48_000; port += 1) busy.busy.add(port); - const exhausted = await driverWith([], busy); - await expect(exhausted.driver.deviceTcp?.reverse(emulator, 5173, signal())).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_FAILED", - message: "no free tunnel port on the worker", - }), - ); - expect(exhausted.runner.calls).toEqual([]); - - const connectWith = async ( - device: typeof emulator | typeof PHYSICAL_DEVICE, - adbReplies: string, - options: { readonly probe?: FakeTcpProbe; readonly port?: number } = {}, - ) => { - const connector = new RepliesTcpStreamConnector(adbReplies); - const clock = new FakeClock(); - const runner = new EmulatorExitingRunner( - options.probe === undefined - ? [] - : [ - { - match: adb("version"), - result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, - }, - ], + takePortOnNextBoot(avdPath: string): { + observed(): { running: boolean; sentByDriver: string[] }; + } { + let running = false; + let port = Number.NaN; + const sent: string[] = []; + this.#takeOnNextBoot = (taken) => { + port = taken; + const avdName = /([^/]+)\.avd$/.exec(avdPath)?.[1] ?? ""; + const handle = this.#inner.spawn( + binaries.emulator, + ["-avd", avdName, "-port", String(taken)], + emulatorOptions, ); - const driver = await createDriver(await androidFilesystem(), runner, { - clock, - ...(options.port === undefined ? {} : { driverConfig: { adbServerPort: options.port } }), - tcpListener: new FakeTcpListenerFactory(), - ...(options.probe === undefined ? {} : { tcpProbe: options.probe }), - tcpStreamConnector: connector, + running = true; + void handle.wait().then(() => { + running = false; }); - const connecting = driver.deviceTcp?.connect(device, 8081, signal()); - return { clock, connecting, connector, runner }; + this.#otherPaths.set(taken, avdPath); }; + this.#sentTo = (serial, command) => { + if (consolePortOf(serial) === port && running && !QUERIES.test(command)) sent.push(command); + }; + return { observed: () => ({ running, sentByDriver: sent }) }; + } - const opened = await connectWith(emulator, "OKAYOKAY", { port: 5050 }); - await expect(opened.connecting).resolves.toBeDefined(); - expect(opened.connector.connections.map(({ port }) => port)).toEqual([5050]); - expect(opened.connector.connections[0]?.peer.writtenText()).toBe( - "001chost:transport:emulator-55860008tcp:8081", - ); - expect(opened.runner.calls).toEqual([]); - - const closedProbe = new FakeTcpProbe([5038, 5037]); - closedProbe.replyWith("OKAY00040028"); - const gated = await connectWith(PHYSICAL_DEVICE, "OKAYOKAY", { probe: closedProbe }); - await expect(gated.connecting).rejects.toThrow(/speaks protocol 40/); - expect(gated.connector.connections).toEqual([]); - - const refused = await connectWith(emulator, "OKAYFAIL0006closed"); - await expect(refused.connecting).rejects.toEqual( - expect.objectContaining({ code: "TUNNEL_FAILED", message: "closed" }), - ); - - const silent = await connectWith(emulator, ""); - // why: the connect is pending only once the socket is open and waiting for adb's answer; a timer alone is set before that. - await vi.waitFor(() => expect(silent.connector.connections).toHaveLength(1)); - await new Promise((resolve) => setImmediate(resolve)); - expect(silent.clock.pendingTimerCount).toBeGreaterThan(0); - silent.clock.advance(4_999); - await new Promise((resolve) => setImmediate(resolve)); - expect(silent.connector.connections[0]?.peer.closed).toBe(false); - silent.clock.advance(1); - await expect(silent.connecting).rejects.toEqual( - expect.objectContaining({ code: "TUNNEL_FAILED", message: "timeout" }), - ); - }); - - it("connects an emulator port to the worker port above the emulator's console port, refuses a phone, and rejects with the connector's reason when nothing listens and with \"timeout\" when nothing answers in 5 s", async () => { - const connector = new FakeTcpStreamConnector(); - const clock = new FakeClock(); - const driver = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { - clock, - tcpListener: new FakeTcpListenerFactory(), - tcpStreamConnector: connector, - }); - const tcp = driver.deviceTcp; - const second = { ...emulator, address: "emulator-5682" }; - - expect(tcp?.supportsEmulatorPort(emulator)).toBe(true); - expect(tcp?.supportsEmulatorPort(PHYSICAL_DEVICE)).toBe(false); - await tcp?.connectEmulatorPort(emulator, "adb", signal()); - await tcp?.connectEmulatorPort(second, "adb", signal()); - expect(connector.connections.map(({ host, port }) => `${host}:${String(port)}`)).toEqual([ - "127.0.0.1:5587", - "127.0.0.1:5683", - ]); - - await expect(tcp?.connectEmulatorPort(PHYSICAL_DEVICE, "adb", signal())).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_UNSUPPORTED", - message: - "emulator adb: the lease is on an Android phone; emulator ports need an Android emulator", - }), - ); - await expect( - tcp?.connectEmulatorPort({ ...emulator, address: "" }, "adb", signal()), - ).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_FAILED", - message: "the emulator has no adb port yet", - }), - ); - - const bare = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { - clock, - tcpListener: new FakeTcpListenerFactory(), - }); - await expect(bare.deviceTcp?.connectEmulatorPort(emulator, "adb", signal())).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_FAILED", - message: "the emulator has no adb port yet", - }), - ); - - connector.refusedPorts.add(5587); - await expect(tcp?.connectEmulatorPort(emulator, "adb", signal())).rejects.toEqual( - expect.objectContaining({ - code: "TUNNEL_FAILED", - message: "connect ECONNREFUSED 127.0.0.1:5587", - }), - ); - - const silent = await createDriver(await androidFilesystem(), new EmulatorExitingRunner([]), { - clock, - tcpListener: new FakeTcpListenerFactory(), - tcpStreamConnector: new HangingTcpStreamConnector(), - }); - const waiting = Promise.resolve( - silent.deviceTcp?.connectEmulatorPort(emulator, "adb", signal()), - ); - const state = promiseState(waiting); - clock.advance(4_999); - await new Promise((resolve) => setImmediate(resolve)); - expect(state.state).toBe("pending"); - clock.advance(1); - await expect(waiting).rejects.toEqual( - expect.objectContaining({ code: "TUNNEL_FAILED", message: "timeout" }), - ); - }); + #sentTo: (serial: string, command: string) => void = () => undefined; - it("sweeps at start only the reverse rules into 47000-47999 on an idle device proved its own, only reads a leased device's rules and keeps their ports from the next reverse until its destroy, touches nothing on an emulator that answers as another's, and on an idle phone removes nothing and keeps the ports its rules point to from the next reverse", async () => { - const own = await driverWith([ - avdPathExpectation("emulator-5586"), - { - match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--list"), - result: { - code: 0, - stderr: "", - stdout: - "host-19 tcp:5173 tcp:47003\nhost-19 tcp:8081 tcp:8082\nhost-19 tcp:3000 tcp:47999\n", - }, - }, - { - match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:5173"), - result: ok, - }, - { - match: adb("-P", "5038", "-s", "emulator-5586", "reverse", "--remove", "tcp:3000"), - result: ok, + runner(): ProcessRunner { + return { + run: async (command, args, options) => { + if (command === binaries.adb && args[0] === "-s") { + const serial = args[1] ?? ""; + const rest = args.slice(2).join(" "); + this.#sentTo(serial, rest); + const answer = this.#identity(serial, rest); + if (answer !== undefined) return answer; + } + return this.#inner.run(command, args, options); }, - ]); - await own.driver.deviceTcp?.sweep([{ device: emulator, leased: false }]); - expect(own.runner.calls.filter((call) => call.args.includes("--remove"))).toHaveLength(2); - - const probe = new FakeTcpProbe([5038, 5037]); - probe.replyWith("OKAY00040029"); - const phones = await driverWith( - [ - { - match: adb("version"), - result: { code: 0, stderr: "", stdout: "Android Debug Bridge version 1.0.41\n" }, - }, - { - match: adb("-P", "5037", "-s", PHYSICAL_DEVICE.address, "reverse", "--list"), - result: { - code: 0, - stderr: "", - stdout: "UsbFfs tcp:5173 tcp:47000\nUsbFfs tcp:3000 tcp:3001\n", + spawn: (command, args, options) => { + const take = this.#takeOnNextBoot; + if (take === undefined || command !== binaries.emulator) { + return this.#inner.spawn(command, args, options); + } + this.#takeOnNextBoot = undefined; + take(Number(args[args.indexOf("-port") + 1])); + const handle = this.#inner.spawn(command, args, options); + let killed!: () => void; + const exitSeen = new Promise((resolve) => { + killed = resolve; + }); + return { + ...handle, + kill: (signal) => { + handle.kill(signal); + killed(); }, - }, - { - match: adb( - "-P", - "5037", - "-s", - PHYSICAL_DEVICE.address, - "reverse", - "--no-rebind", - "tcp:5174", - "tcp:47001", - ), - result: ok, - }, - ], - new FakeTcpListenerFactory(), - probe, - ); - await phones.driver.deviceTcp?.sweep([{ device: PHYSICAL_DEVICE, leased: false }]); - const onPhone = await phones.driver.deviceTcp?.reverse(PHYSICAL_DEVICE, 5174, signal()); - expect(onPhone?.hostPort).toBe(47_001); - expect(phones.runner.calls.some((call) => call.args.includes("--remove"))).toBe(false); - - const foreign = await driverWith([avdPathExpectation("emulator-5586", "simlock_two")]); - await foreign.driver.deviceTcp?.sweep([{ device: emulator, leased: false }]); - expect(foreign.runner.calls.some((call) => call.args.includes("reverse"))).toBe(false); - - const onServer = ["-P", "5038", "-s", "emulator-5586"]; - const leased = await driverWith([ - avdPathExpectation("emulator-5586"), - { - match: adb(...onServer, "reverse", "--list"), - result: { - code: 0, - stderr: "", - stdout: "host-19 tcp:5173 tcp:47000\nhost-19 tcp:3000 tcp:3001\n", - }, - }, - { - match: adb(...onServer, "reverse", "--no-rebind", "tcp:5174", "tcp:47001"), - result: ok, - }, - { - match: adb(...onServer, "reverse", "--no-rebind", "tcp:5175", "tcp:47000"), - result: ok, + wait: async () => { + await exitSeen; + return handle.wait(); + }, + }; }, - ]); - await leased.driver.deviceTcp?.sweep([{ device: emulator, leased: true }]); - expect(leased.runner.calls.filter((call) => call.args.includes("--remove"))).toEqual([]); - const next = await leased.driver.deviceTcp?.reverse(emulator, 5174, signal()); - expect(next?.hostPort).toBe(47_001); - await leased.driver.deviceTcp?.release(emulator); - const afterReclaim = await leased.driver.deviceTcp?.reverse(emulator, 5175, signal()); - expect(afterReclaim?.hostPort).toBe(47_000); - }); -}); + spawnStreaming: (command, args, options) => + this.#inner.spawnStreaming(command, args, options), + }; + } -describe("AndroidDriver tunnels of a destroyed device", () => { - it("closes the loopback listeners held for a virtual device's tunnels when the device is destroyed", async () => { - const listeners = new FakeTcpListenerFactory(); - const harness = await provisionedHarness({ - afterwards: [ - processResult(binaries.adb, [ - "-P", - "5038", - "-s", - "emulator-5586", - "reverse", - "--no-rebind", - "tcp:5173", - "tcp:47000", - ]), - copyPathExpectation("emulator-5586"), - processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), - ], - tcpListener: listeners, - }); - const ready = await harness.driver.makeReady(harness.device); - await harness.driver.deviceTcp?.reverse( - { ...ready, physical: false }, - 5173, - new AbortController().signal, - ); - expect(listeners.listeners.map((listener) => listener.closed)).toEqual([false]); - const launch = harness.runner.calls.findLastIndex( - (call) => call.command === binaries.emulator && call.args.includes("-avd"), - ); + #identity(serial: string, command: string): ProcessResult | undefined { + const path = this.avdPathAnswering(serial); + if (path === undefined) return undefined; + const name = /([^/]+)\.avd$/.exec(path)?.[1] ?? ""; + if (command === "emu avd path") return ok(`${path}\nOK\n`); + if (command === "emu avd name") return ok(`${name}\nOK\n`); + if (command === "shell getprop ro.boot.qemu.avd_name") return ok(`${name}\n`); + return undefined; + } +} + +/** why: `emu avd path` answers truthfully until the test sets an answer, so a seed is made first and only the copy's boot is lied to. */ +function overridingAvdPath(inner: ProcessRunner): { + readonly answer: (answer: ProcessResult) => void; + readonly runner: ProcessRunner; +} { + let answer: ProcessResult | undefined; + return { + answer: (next) => { + answer = next; + }, + runner: { + run: async (command, args, options) => + answer !== undefined && + command === binaries.adb && + args[0] === "-s" && + args.slice(2).join(" ") === "emu avd path" + ? answer + : inner.run(command, args, options), + spawn: (command, args, options) => inner.spawn(command, args, options), + spawnStreaming: (command, args, options) => inner.spawnStreaming(command, args, options), + }, + }; +} - const destroying = harness.driver.destroy(ready); - harness.runner.handles[launch]?.kill("SIGTERM"); - await destroying; +class LinkedHomeFilesystem extends MemoryFilesystem { + override async realpath(path: string): Promise { + const resolved = await super.realpath(path.replace(/^\/private\/home\//, "/home/")); + return resolved.replace(/^\/home\//, "/private/home/"); + } +} - expect(listeners.listeners.map((listener) => listener.closed)).toEqual([true]); - }); -}); +const QUERIES = /^(shell getprop \S+|emu avd (name|path|id|status|discoverypath))$/; + +/** + * why: The message of the driver's `DriverCrashError`; anything else -- a boot timeout, the fake's + * own complaint about an unscripted call -- is rethrown, so it fails the test as itself. + */ +function driverCrashMessage(error: unknown): { readonly refused: string } { + if (error instanceof Error && error.name === "DriverCrashError") { + return { refused: error.message }; + } + throw error; +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} describe("AndroidDriver readiness on a taken console port", () => { it("refuses to ready a device whose serial an emulator of an AVD outside its root answers, and changes nothing on that emulator, even when the device is then shut down (#256)", async () => { @@ -731,15 +424,6 @@ describe("AndroidDriver emulator registration", () => { }); }); -function recordKillTimeout( - into: (number | undefined)[], - command: string, - args: readonly string[], - timeoutMs: number | undefined, -): void { - if (command === binaries.adb && args.slice(2).join(" ") === "emu kill") into.push(timeoutMs); -} - describe("AndroidDriver console ports across a restart", () => { it("stops a discovered running emulator before deleting its AVD after a restart", async () => { const filesystem = new RemovalObservingFilesystem(); @@ -1049,1052 +733,3 @@ describe("AndroidDriver console port reservations", () => { expect(host.avdAnswering(running.address)).toBe("simlock_two"); }); }); - -const live = process.env.SIMLOCK_LIVE_ANDROID === "1" ? it : it.skip; - -live( - "live smoke: provision, boot from the boot snapshot, restart, and destroy", - async () => { - const driver = await AndroidDriver.create({ - clock: new SystemClock(), - database: await memoryAndroidSourceDatabase(), - driverConfig: {}, - env: process.env, - filesystem: new NodeFilesystem(), - homeDirectory: process.env.HOME ?? home, - hostAbi: "arm64-v8a", - instanceId: `live-${process.pid}`, - processRunner: new NodeProcessRunner(), - processSupervisor: new NodeProcessSupervisor({ - platform: process.platform, - processRunner: new NodeProcessRunner(), - }), - simlockHome: join(tmpdir(), `simlock-live-android-${process.pid}`), - tcpProbe: new NodeTcpProbe(), - }); - const spec = await driver.resolveSpec({ - model: process.env.SIMLOCK_LIVE_ANDROID_MODEL ?? "Pixel 5", - platform: "android", - ...(process.env.SIMLOCK_LIVE_ANDROID_API === undefined - ? {} - : { osVersion: process.env.SIMLOCK_LIVE_ANDROID_API }), - }); - const device = await driver.provision(spec); - - try { - const ready = await driver.makeReady(device); - await driver.shutdown(ready); - const restarted = await driver.makeReady(ready); - - expect(restarted.address).toMatch(/^emulator-\d+$/); - } finally { - await driver.destroy(device); - } - }, - 480_000, -); - -describe("AndroidDriver image fit", () => { - describe("seeds and copies", () => { - const sources = `${avdDirectory}/sources`; - const copyHome = `${avdDirectory}/simlock_one`; - const seedHome = `${sources}/seed_two`; - async function seedHarness( - expectations: readonly ScriptedProcessExpectation[], - options: { - readonly clock?: FakeClock; - readonly config?: string; - readonly filesystem?: MemoryFilesystem; - readonly ids?: readonly string[]; - readonly onDiagnostic?: (diagnostic: AndroidDriverDiagnostic) => void; - } = {}, - ) { - const filesystem = await androidFilesystem({}, options.filesystem); - await writeSeedFiles(filesystem, seedHome, options.config); - const database = await memoryAndroidSourceDatabase(); - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ...expectations, - ]); - const driver = await createDriver(filesystem, runner, { - ...(options.clock === undefined ? {} : { clock: options.clock }), - ...(options.onDiagnostic === undefined ? {} : { onDiagnostic: options.onDiagnostic }), - database, - ids: options.ids ?? ["one", "two", "three"], - }); - const spec = await driver.resolveSpec({ - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - return { database, driver, filesystem, runner, spec }; - } - - /** why: the daemon after a restart: a new driver over the same device folders and the same source table. */ - async function reopened( - harness: Awaited>, - expectations: readonly ScriptedProcessExpectation[], - options: { - readonly clock?: FakeClock; - readonly emulator?: AndroidEmulatorLaunchOptions; - } = {}, - ) { - const runner = new EmulatorExitingRunner([...expectations]); - const driver = await createDriver(harness.filesystem, runner, { - ...options, - database: harness.database, - ids: ["four", "five", "six"], - }); - return { driver, runner }; - } - - it("hands the seeds part its clock, its source table, its device root and its id generator", async () => { - const clock = new FakeClock(5_000); - const harness = await seedHarness(makingTheSeed(), { clock }); - - await harness.driver.provision(harness.spec); - - expect( - harness.database - .prepare("SELECT source_id, folder, created_at FROM android_source_data") - .all(), - ).toEqual([{ created_at: 5_000, folder: seedHome, source_id: "seed_two" }]); - }); - - it("hands the avd config the emulator launch settings, so a seed made under other ones is stale", async () => { - const harness = await seedHarness(makingTheSeed()); - await harness.driver.provision(harness.spec); - - const restarted = await reopened( - harness, - [processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9")], - { emulator: { audio: true, bootAnimation: true, gpu: "auto", headless: true } }, - ); - - await expect(restarted.driver.listSources()).resolves.toMatchObject([ - { id: "seed_two", staleReason: "android.emulator launch settings changed" }, - ]); - }); - - it("leases a plain Android device from a copy of the kind's seed, made first when there is none, booted from the boot snapshot", async () => { - const harness = await seedHarness([ - ...makingTheSeed(), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true), - markWriteExpectation("emulator-5586", "three"), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - - const device = await harness.driver.provision(harness.spec); - const ready = await harness.driver.makeReady(device); - - const create = harness.runner.calls.find((call) => call.args[0] === "create"); - expect(create?.options.env).toMatchObject({ ANDROID_AVD_HOME: seedHome }); - expect(device).toMatchObject({ - deviceId: "simlock_one", - driverData: { avdHome: copyHome, avdName: "simlock" }, - }); - await expect(harness.filesystem.readFile(`${copyHome}/simlock.avd/config.ini`)).resolves.toBe( - "hw.ramSize=2048\nsimlock.mark=three\n", - ); - await expect(harness.filesystem.readFile(`${copyHome}/simlock.ini`)).resolves.toBe( - `avd.ini.encoding=UTF-8\npath=${copyHome}/simlock.avd\ntarget=android-34\n`, - ); - await expect(harness.driver.listSources()).resolves.toMatchObject([ - { - id: "seed_two", - imageTag: "google_apis", - kind: "seed", - model: "Pixel 5", - osVersion: "34", - }, - ]); - const launches = harness.runner.calls.filter((call) => call.args.includes("-avd")); - expect(launches.map((call) => call.options.env?.["ANDROID_AVD_HOME"])).toEqual([ - seedHome, - seedHome, - copyHome, - ]); - expect(launches.flatMap((call) => call.args)).not.toContain("-wipe-data"); - const shellRuns = harness.runner.calls.filter( - (call) => call.args.includes("sync") || call.args.includes("-p"), - ); - expect( - shellRuns.map((call) => [call.args.at(-1), call.options.timeoutMs]), - "the seed's sync and power-off are each bounded, so a wedged adbd cannot hold the make", - ).toEqual([ - ["sync", 60_000], - ["-p", 60_000], - ]); - expect(ready.address).toBe("emulator-5586"); - }); - - it("reports a copy whose boot snapshot does not load by its device id, only when it took over 12 s, and never wipes it", async () => { - for (const { readyAfterMs, reported } of [ - { readyAfterMs: 12_000, reported: false }, - { readyAfterMs: 12_001, reported: true }, - ]) { - const clock = new FakeClock(); - const diagnostics: AndroidDriverDiagnostic[] = []; - const harness = await seedHarness( - [ - ...makingTheSeed(), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true, 1), - markWriteExpectation("emulator-5586", "three"), - ], - { clock, onDiagnostic: (diagnostic) => diagnostics.push(diagnostic) }, - ); - const device = await harness.driver.provision(harness.spec); - - const ready = harness.driver.makeReady(device); - await vi.waitFor(() => - expect( - harness.runner.calls.filter((call) => call.args.includes("sys.boot_completed")), - ).toHaveLength(3), - ); - clock.advance(readyAfterMs); - await ready; - - expect(diagnostics).toEqual( - reported ? [{ deviceId: "simlock_one", kind: "snapshot-cold-boot", readyAfterMs }] : [], - ); - expect(harness.runner.calls.flatMap((call) => call.args)).not.toContain("-wipe-data"); - } - }); - - describe("saveSource", () => { - const savedHome = `${sources}/baseline_four`; - const serial = "emulator-5586"; - const config = - "hw.device.name=pixel_5\nimage.sysdir.1=system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize=2048\n"; - const ids = ["one", "two", "three", "four"]; - const adbShell = (command: string) => - processResult(binaries.adb, ["-s", serial, "shell", command]); - const sourceBoot = () => { - const stage = seedStages(savedHome); - return [ - stage.devices, - ...stage.secondBoot, - stage.save, - stage.list, - stage.probe, - stage.kill, - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]; - }; - const leasing = (hangs: boolean) => [ - ...makingTheSeed(), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], hangs), - markWriteExpectation(serial, "three"), - ]; - const probe = processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${copyHome}/simlock.avd\r\nOK\r\n`, - ); - const savingSteps = [ - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - probe, - adbShell("sync"), - processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), - ]; - - it("provisions a device named after a baseline source as a copy of that source's folder with no seed made and only the emulator's version read, and refuses a source whose system image is gone", async () => { - const version = () => - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"); - const harness = await seedHarness( - [...leasing(true), ...savingSteps, ...sourceBoot(), version(), version()], - { - config, - ids: [...ids, "five", "six"], - }, - ); - const ready = await harness.driver.makeReady(await harness.driver.provision(harness.spec)); - await harness.driver.saveSource({ ...ready, physical: false as const }); - await harness.filesystem.writeFileAtomic(`${savedHome}/simlock.avd/marker.txt`, "saved"); - const callsBefore = harness.runner.calls.length; - - const copy = await harness.driver.provision(harness.spec, { source: "baseline_four" }); - - expect(copy.deviceId).toBe("simlock_five"); - await expect( - harness.filesystem.readFile(`${avdDirectory}/simlock_five/simlock.avd/marker.txt`), - ).resolves.toBe("saved"); - expect(harness.runner.calls).toHaveLength(callsBefore + 1); - - await harness.filesystem.rm(`${sdk}/system-images/android-34/google_apis/arm64-v8a`); - await expect( - harness.driver.provision(harness.spec, { source: "baseline_four" }), - ).rejects.toThrow( - "source baseline_four is stale: system image system-images;android-34;google_apis;arm64-v8a is not installed", - ); - expect(await harness.filesystem.exists(`${avdDirectory}/simlock_six`)).toBe(false); - }); - - it("keeps a powered-off copy of the device as a ready baseline source with one boot snapshot, powering the emulator off first, or killing it when it will not leave in 60 s, or polling adb with no process handle", async () => { - const kept = async (harness: Awaited>) => { - expect(await harness.filesystem.exists(`${savedHome}/simlock.avd/config.ini`)).toBe(true); - await expect(harness.filesystem.readFile(`${savedHome}/simlock.ini`)).resolves.toContain( - `path=${savedHome}/simlock.avd`, - ); - expect(await harness.filesystem.exists(copyHome)).toBe(true); - const listed = await harness.driver.listSources(); - expect(listed.find((source) => source.id === "baseline_four")).toMatchObject({ - imageTag: "google_apis", - kind: "baseline", - model: "pixel_5", - osVersion: "34", - }); - expect( - harness.database - .prepare("SELECT state FROM android_source_data WHERE source_id = 'baseline_four'") - .all(), - ).toEqual([{ state: "ready" }]); - }; - const sent = (harness: Awaited>): string[] => - harness.runner.calls - .filter((call) => call.command === binaries.adb && call.args[0] === "-s") - .map((call) => call.args.slice(2).join(" ")); - const saveLaunch = (harness: Awaited>) => - harness.runner.calls - .filter((call) => call.args.includes("-avd")) - .map((call) => [call.options.env?.["ANDROID_AVD_HOME"], call.args.at(-1)]); - - const powered = await seedHarness([...leasing(true), ...savingSteps, ...sourceBoot()], { - config, - ids, - }); - const poweredReady = await powered.driver.makeReady( - await powered.driver.provision(powered.spec), - ); - const poweredSaved = await powered.driver.saveSource({ - ...poweredReady, - physical: false as const, - }); - expect(poweredSaved).toEqual({ - imageTag: "google_apis", - sizeBytes: 121, - sourceId: "baseline_four", - }); - await kept(powered); - expect(sent(powered)).toEqual( - expect.arrayContaining([ - "shell sync", - "shell reboot -p", - "emu avd snapshot save simlock_boot", - ]), - ); - expect(sent(powered).indexOf("shell reboot -p")).toBeLessThan( - sent(powered).lastIndexOf("emu avd snapshot save simlock_boot"), - ); - expect(saveLaunch(powered).at(-1)).toEqual([savedHome, "-no-snapshot-load"]); - - const clock = new FakeClock(); - const stuck = await seedHarness( - [ - ...leasing(true), - ...savingSteps, - processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${copyHome}/simlock.avd\r\nOK\r\n`, - ), - processResult(binaries.adb, ["-s", serial, "emu", "kill"]), - ...sourceBoot(), - ], - { clock, config, ids }, - ); - const stuckReady = await stuck.driver.makeReady(await stuck.driver.provision(stuck.spec)); - // why: the stuck emulator ignores `reboot -p` and `emu kill`; only the deadlines and a SIGKILL end it. - stuck.runner.ignoresEmuKill = true; - const stuckSaving = stuck.driver.saveSource({ ...stuckReady, physical: false as const }); - await vi.waitFor(() => - expect(stuck.runner.calls.filter((call) => call.args.includes("reboot"))).toHaveLength(2), - ); - clock.advance(60_000); - await vi.waitFor(() => - expect(stuck.runner.calls.filter((call) => call.args.includes("kill"))).toHaveLength(2), - ); - stuck.runner.ignoresEmuKill = false; - let stuckDone = false; - void stuckSaving.then( - () => (stuckDone = true), - () => (stuckDone = true), - ); - await vi.waitFor(() => { - clock.advance(60_000); - expect(stuckDone).toBe(true); - }); - await expect(stuckSaving).resolves.toMatchObject({ sourceId: "baseline_four" }); - await kept(stuck); - expect(sent(stuck).slice(sent(stuck).indexOf("shell reboot -p"))).toEqual( - expect.arrayContaining(["emu avd path", "emu kill"]), - ); - - const restarted = await seedHarness(leasing(true), { config, ids }); - const restartedReady = await restarted.driver.makeReady( - await restarted.driver.provision(restarted.spec), - ); - const clockAfter = new FakeClock(); - const adopted = await reopened( - restarted, - [ - ...savingSteps, - processResult( - binaries.adb, - ["devices"], - `List of devices attached\n${serial}\tdevice\n`, - ), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ...sourceBoot(), - ], - { clock: clockAfter }, - ); - const adoptedSaving = adopted.driver.saveSource({ - ...restartedReady, - physical: false as const, - }); - await vi.waitFor(() => - expect(adopted.runner.calls.filter((call) => call.args[0] === "devices")).toHaveLength(1), - ); - clockAfter.advance(1_000); - await expect(adoptedSaving).resolves.toMatchObject({ sourceId: "baseline_four" }); - await kept({ ...restarted, driver: adopted.driver }); - }); - - it("saves a device whose config.ini has spaces around `=`, as avdmanager writes it, with the model and image it names", async () => { - const spaced = - "hw.device.name = pixel_6\nimage.sysdir.1 = system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize = 2048\n"; - const harness = await seedHarness([...leasing(true), ...savingSteps, ...sourceBoot()], { - config: spaced, - ids, - }); - const ready = await harness.driver.makeReady(await harness.driver.provision(harness.spec)); - - await expect( - harness.driver.saveSource({ ...ready, physical: false as const }), - ).resolves.toMatchObject({ imageTag: "google_apis", sourceId: "baseline_four" }); - - expect( - (await harness.driver.listSources()).find((source) => source.id === "baseline_four"), - ).toMatchObject({ - imageTag: "google_apis", - kind: "baseline", - model: "pixel_6", - osVersion: "34", - }); - }); - }); - - describe("saveSource failures", () => { - const savedHome = `${sources}/baseline_four`; - const serial = "emulator-5586"; - const config = - "hw.device.name=pixel_5\nimage.sysdir.1=system-images/android-34/google_apis/arm64-v8a/\nhw.ramSize=2048\n"; - const version = processResult( - binaries.emulator, - ["-version"], - "Android emulator version 36.1.9", - ); - const probe = (path = `${copyHome}/simlock.avd`) => - processResult(binaries.adb, ["-s", serial, "emu", "avd", "path"], `${path}\r\nOK\r\n`); - const poweredOff = [ - version, - probe(), - processResult(binaries.adb, ["-s", serial, "shell", "sync"]), - processResult(binaries.adb, ["-s", serial, "shell", "reboot", "-p"]), - ]; - const stage = seedStages(savedHome); - - it("deletes the source's folder and row and throws when the ownership check fails, the copy fails or the boot snapshot is not saved, leaving the device as it was", async () => { - const cases = [ - { - expectations: [version, probe(`${avdDirectory}/other.avd`)], - message: "emulator-5586 is not its emulator", - sent: false, - }, - { - copyFails: true, - expectations: poweredOff, - message: "copying Android device simlock_one to source baseline_four failed: no space", - }, - { - expectations: [ - ...poweredOff, - stage.devices, - ...stage.secondBoot, - { ...stage.save, result: { code: 1, stderr: "emulator gone", stdout: "" } }, - ], - message: "Android seed baseline_four: boot snapshot simlock_boot was not saved", - }, - ] as const; - - for (const failing of cases) { - const filesystem = new ArmedCopyFailureFilesystem("no space"); - const harness = await seedHarness( - [ - ...makingTheSeed(), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ...bootingAt(copyHome, ["-snapshot", "simlock_boot"], true), - markWriteExpectation(serial, "three"), - ...failing.expectations, - ], - { config, filesystem, ids: ["one", "two", "three", "four"] }, - ); - const ready = await harness.driver.makeReady( - await harness.driver.provision(harness.spec), - ); - filesystem.armed = "copyFails" in failing; - const before = harness.runner.calls.length; - - await expect( - harness.driver.saveSource({ ...ready, physical: false as const }), - ).rejects.toThrow(failing.message); - - const sentToDevice = harness.runner.calls - .slice(before) - .filter((call) => call.args.includes("sync") || call.args.includes("reboot")); - expect(sentToDevice.length > 0, failing.message).toBe(!("sent" in failing)); - expect(await harness.filesystem.exists(savedHome), failing.message).toBe(false); - expect(await harness.filesystem.exists(copyHome), failing.message).toBe(true); - expect( - harness.database - .prepare("SELECT source_id FROM android_source_data WHERE kind = 'baseline'") - .all(), - failing.message, - ).toEqual([]); - const emulators = harness.runner.calls.flatMap((call, index) => - call.args.includes("-avd") ? [index] : [], - ); - // why: the first two launches make the seed and the third is the device's own, which a failure before `reboot -p` leaves running; what follows is the source's boot. - for (const emulator of emulators.slice(3)) { - let exited = false; - void harness.runner.handles[emulator]?.wait().then(() => { - exited = true; - }); - await vi.waitFor(() => expect(exited, "the source's emulator was stopped").toBe(true)); - } - } - }); - }); - - it("quotes a provision at 1 s with a fresh seed and 150 s without one, and a prepare boot at 4 s for a copy and 70 s otherwise", async () => { - const harness = await seedHarness([ - ...makingTheSeed(), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - const spec = { ...harness.spec, physical: false } as const; - const provision = { operation: "provision" } as const; - expect(harness.driver.estimate(provision, spec)).toBe(150_000); - - const copy = await harness.driver.provision(harness.spec); - - expect(harness.driver.estimate(provision, spec)).toBe(1_000); - expect(harness.driver.estimate(provision, { ...spec, model: "Pixel 9" })).toBe(150_000); - expect(harness.driver.estimate(provision, { ...spec, imageTag: "android-tv" })).toBe(150_000); - const prepare = { device: copy, operation: "boot", purpose: "prepare" } as const; - const oldLayout = { - address: "", - deviceId: "simlock_old", - driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, - }; - expect( - await Promise.all([ - harness.driver.estimate(prepare, spec), - harness.driver.estimate({ ...prepare, purpose: "recover" }, spec), - harness.driver.estimate({ ...prepare, device: oldLayout }, spec), - ]), - ).toEqual([4_000, 70_000, 70_000]); - await expect( - harness.driver.estimate({ ...prepare, device: { ...copy, deviceId: "other" } }, spec), - "a device id that does not match its AVD rejects the returned promise: a throw would skip the caller's fallback", - ).rejects.toThrow("does not match AVD"); - - const staled = await reopened(harness, [ - processResult(binaries.emulator, ["-version"], "Android emulator version 36.2.0"), - ]); - await staled.driver.listSources(); - expect(staled.driver.estimate(provision, spec)).toBe(150_000); - }); - - it("destroys the seeds made from a system image when it is removed, and a seed it cannot destroy stays listed as stale", async () => { - const packageName = "system-images;android-34;google_apis;arm64-v8a"; - const imagePath = `${sdk}/system-images/android-34/google_apis/arm64-v8a`; - const uninstall = processResult(binaries.sdkmanager, ["--uninstall", packageName]); - const noEmulators = processResult(binaries.adb, ["devices"], "List of devices attached\n"); - const remove = async ( - harness: Awaited>, - failingFolder?: string, - alsoMake?: { readonly home: string; readonly imageTag: string }, - ) => { - await harness.driver.provision(harness.spec); - if (alsoMake !== undefined) { - await harness.filesystem.mkdirp( - `${sdk}/system-images/android-34/${alsoMake.imageTag}/arm64-v8a`, - ); - await writeSeedFiles(harness.filesystem, alsoMake.home); - await harness.driver.provision( - await harness.driver.resolveSpec({ - imageTag: alsoMake.imageTag, - model: "Pixel 5", - osVersion: "34", - platform: "android", - }), - ); - } - if (failingFolder !== undefined) harness.filesystem.defineFailure(failingFolder, "EACCES"); - const target = (await harness.driver.listComponents()).find( - (component) => component.variant === "google_apis/arm64-v8a", - ); - // why: the scripted runner only answers; real sdkmanager also deletes the image's folder. - const spawn = harness.runner.spawn.bind(harness.runner); - vi.spyOn(harness.runner, "spawn").mockImplementation((command, args, options) => { - if (args[0] === "--uninstall") void harness.filesystem.rm(imagePath); - return spawn(command, args, options); - }); - await harness.driver.removeComponent(target?.receipt ?? {}, { - signal: new AbortController().signal, - }); - }; - - const destroyed = await seedHarness([...makingTheSeed(), uninstall, noEmulators]); - await remove(destroyed); - expect(await destroyed.filesystem.exists(seedHome)).toBe(false); - await expect(destroyed.driver.listSources()).resolves.toEqual([]); - - const otherHome = `${sources}/seed_device-4`; - const other = await seedHarness([ - ...makingTheSeed(), - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ...makingASeedFrom(otherHome), - uninstall, - noEmulators, - ]); - await remove(other, undefined, { home: otherHome, imageTag: "google_apis_playstore" }); - expect(await other.filesystem.exists(seedHome)).toBe(false); - expect(await other.filesystem.exists(otherHome)).toBe(true); - - const stuck = await seedHarness([ - ...makingTheSeed(), - uninstall, - noEmulators, - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - await remove(stuck, seedHome); - await expect(stuck.driver.listSources()).resolves.toMatchObject([ - { id: "seed_two", staleReason: `system image ${packageName} is not installed` }, - ]); - }); - - it("boots a recovered copy cold from its disk on its recorded port: no snapshot is loaded and nothing is wiped, a second makeReady does not boot again and destroy stops it", async () => { - const harness = await seedHarness([ - ...bootingAt(copyHome, ["-no-snapshot-load"], true), - markWriteExpectation("emulator-5586", "one"), - ...bootingAt(copyHome, [], true).slice(1), - markWriteExpectation("emulator-5586", "two"), - copyPathExpectation("emulator-5586"), - processResult(binaries.adb, ["-s", "emulator-5586", "emu", "kill"]), - ]); - await writeSeedFiles(harness.filesystem, copyHome); - - const recovered = { - address: "emulator-5586", - deviceId: "simlock_one", - driverData: { - avdHome: copyHome, - avdName: "simlock", - configHash: "hash", - port: 5586, - serial: "emulator-5586", - }, - }; - await harness.driver.makeReady(recovered, { purpose: "recover" }); - - const launch = harness.runner.calls.find((call) => call.args.includes("-avd")); - expect(launch?.args).toContain("-no-snapshot-load"); - expect(launch?.args).not.toContain("-snapshot"); - expect(launch?.args).not.toContain("-wipe-data"); - expect(launch?.options.env).toMatchObject({ ANDROID_AVD_HOME: copyHome }); - - await harness.driver.makeReady(recovered); - await harness.driver.destroy(recovered); - expect(emulatorLaunchArgs(harness.runner)).toHaveLength(1); - expect( - harness.runner.calls.some((call) => call.args.slice(-2).join(" ") === "emu kill"), - "destroy stops the emulator the recover boot started", - ).toBe(true); - }); - - it("boots a device made before copies once with a wipe: a second makeReady does not boot again and destroy stops it and deletes it through avdmanager", async () => { - const serial = "emulator-5586"; - const polls = (name: string) => [ - processResult( - binaries.adb, - ["-s", serial, "shell", "getprop", "sys.boot_completed"], - "1\n", - ), - processResult(binaries.adb, ["-s", serial, "shell", "getprop", "init.svc.bootanim"], ""), - avdPathExpectation(serial, name), - processResult( - binaries.adb, - ["-s", serial, "shell", "pm", "path", "android"], - "package:/system/framework/framework-res.apk\n", - ), - ]; - const harness = await seedHarness([ - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - { - hangs: true, - match: { - args: ["-avd", "simlock_old", ...seedEmulatorLine, "-wipe-data", "-no-snapshot-load"], - command: binaries.emulator, - }, - }, - ...polls("simlock_old"), - markWriteExpectation(serial, "one"), - ...polls("simlock_old"), - markWriteExpectation(serial, "two"), - avdPathExpectation(serial, "simlock_old"), - processResult(binaries.adb, ["-s", serial, "emu", "kill"]), - processResult(binaries.avdmanager, ["delete", "avd", "-n", "simlock_old"]), - ]); - await harness.filesystem.mkdirp(`${avdDirectory}/simlock_old.avd`); - await harness.filesystem.writeFileAtomic( - `${avdDirectory}/simlock_old.avd/config.ini`, - "hw.ramSize=2048\n", - ); - const legacy = { - address: "", - deviceId: "simlock_old", - driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, - }; - - await harness.driver.makeReady(legacy); - expect(emulatorLaunchArgs(harness.runner)[0]).toEqual( - expect.arrayContaining(["-wipe-data", "-no-snapshot-load"]), - ); - await harness.driver.makeReady(legacy); - await harness.driver.destroy(legacy); - - expect(emulatorLaunchArgs(harness.runner), "the wipe boot happens once").toHaveLength(1); - expect( - harness.runner.calls.some((call) => call.args.slice(-2).join(" ") === "emu kill"), - "destroy stops the emulator the wipe boot started", - ).toBe(true); - expect( - harness.runner.calls.some( - (call) => - call.command === binaries.avdmanager && - call.args.join(" ") === "delete avd -n simlock_old", - ), - "destroy deletes the device through avdmanager", - ).toBe(true); - }); - - it("destroys a plain Android device after its ownership check: the emulator is killed and its folder deleted, an unreadable check or a failing delete throws and keeps it, and a folder outside the root is never deleted", async () => { - const serial = "emulator-5586"; - const attached = `List of devices attached\n${serial}\tdevice\n`; - const own = { - address: serial, - deviceId: "simlock_one", - driverData: { - avdHome: copyHome, - avdName: "simlock", - configHash: "hash", - port: 5586, - serial, - }, - }; - const harness = async (expectations: readonly ScriptedProcessExpectation[]) => { - const filesystem = await androidFilesystem(); - for (const home of [copyHome, `${avdDirectory}/simlock_two`]) { - await filesystem.mkdirp(`${home}/simlock.avd`); - await filesystem.writeFileAtomic(`${home}/simlock.avd/config.ini`, "hw.ramSize=2048\n"); - } - const runner = new EmulatorExitingRunner([...expectations]); - return { driver: await createDriver(filesystem, runner), filesystem, runner }; - }; - const answersWith = (folder: string) => - processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${folder}/simlock.avd\r\nOK\r\n`, - ); - - const passes = await harness([ - processResult(binaries.adb, ["devices"], attached), - answersWith(copyHome), - processResult(binaries.adb, ["-s", serial, "emu", "kill"]), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ]); - await passes.driver.destroy(own); - expect(await passes.filesystem.exists(copyHome)).toBe(false); - expect(await passes.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); - - const neighbour = await harness([ - processResult( - binaries.adb, - ["devices"], - "List of devices attached\nemulator-5588\tdevice\n", - ), - processResult( - binaries.adb, - ["-s", "emulator-5588", "emu", "avd", "path"], - `${avdDirectory}/simlock_two/simlock.avd\r\nOK\r\n`, - ), - ]); - await neighbour.driver.destroy(own); - expect(await neighbour.filesystem.exists(copyHome)).toBe(false); - expect(neighbour.runner.calls.flatMap((call) => call.args)).not.toContain("kill"); - - const unreadable = await harness([ - processResult(binaries.adb, ["devices"], attached), - { - match: { args: ["-s", serial, "emu", "avd", "path"], command: binaries.adb }, - result: { code: 1, stderr: "error", stdout: "" }, - }, - ]); - await expect(unreadable.driver.destroy(own)).rejects.toThrow( - `Refusing to delete simlock_one: could not tell which AVD the emulator on ${serial} runs`, - ); - expect(await unreadable.filesystem.exists(copyHome)).toBe(true); - expect(unreadable.runner.calls.flatMap((call) => call.args)).not.toContain("kill"); - - const failingDelete = await harness([ - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ]); - failingDelete.filesystem.defineFailure(copyHome, "EACCES"); - await expect(failingDelete.driver.destroy(own)).rejects.toMatchObject({ code: "EACCES" }); - expect(await failingDelete.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); - - const outside = await harness([]); - await outside.filesystem.mkdirp("/elsewhere/simlock_two"); - await expect( - outside.driver.destroy({ - ...own, - deviceId: "simlock_two", - driverData: { ...own.driverData, avdHome: "/elsewhere/simlock_two" }, - }), - ).rejects.toThrow("Android device simlock_two names a folder outside the root"); - expect(await outside.filesystem.exists("/elsewhere/simlock_two")).toBe(true); - expect(await outside.filesystem.exists(`${avdDirectory}/simlock_two`)).toBe(true); - - const malformed = await harness([]); - await expect( - malformed.driver.destroy({ - ...own, - driverData: { ...own.driverData, avdHome: 5 }, - }), - ).rejects.toThrow("Android device simlock_one has invalid driver data"); - expect(await malformed.filesystem.exists(copyHome)).toBe(true); - - const old = await harness([ - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - processResult(binaries.avdmanager, ["delete", "avd", "-n", "simlock_old"]), - ]); - await old.driver.destroy({ - ...own, - deviceId: "simlock_old", - driverData: { avdName: "simlock_old", configHash: "hash", port: 0, serial: "" }, - }); - expect(old.runner.calls[1]?.options.env).toMatchObject({ ANDROID_AVD_HOME: avdDirectory }); - - await expect(old.driver.reclaim(own, { clean: "standard" })).rejects.toMatchObject({ - message: "virtual devices are not reclaimed", - name: "DriverCrashError", - }); - }); - }); -}); - -/** why: copies work until a test arms the failure, so a device can be provisioned from a seed first. */ -class ArmedCopyFailureFilesystem extends MemoryFilesystem { - armed = false; - - constructor(private readonly message: string) { - super(); - } - - override copyTree( - from: string, - to: string, - options: { readonly timeoutMs: number }, - ): Promise { - return this.armed ? Promise.reject(new Error(this.message)) : super.copyTree(from, to, options); - } -} - -/** why: tells a test what is running when a path is removed, so "stopped before deleted" is observed, not assumed. */ -class RemovalObservingFilesystem extends MemoryFilesystem { - onRemove: (path: string) => void = () => undefined; - - override async rm(path: string): Promise { - this.onRemove(path); - await super.rm(path); - } -} - -/** why: what a running AVD made before copies answers to `emu avd path`: `/.avd`. */ -function avdPathExpectation(serial: string, avdName = "simlock_one"): ScriptedProcessExpectation { - return processResult( - binaries.adb, - ["-s", serial, "emu", "avd", "path"], - `${avdDirectory}/${avdName}.avd\r\nOK\r\n`, - ); -} - -/** - * why: An `EmulatorHost` on which another emulator takes a device's console port just before the - * device's own emulator is spawned on it, and which answers the queries that tell emulators - * apart: `emu avd path`, `emu avd name` and `getprop ro.boot.qemu.avd_name`. - * - * The device's emulator cannot bind the taken port and exits -- but not before the other - * emulator has answered the readiness poll, as it does on a real machine, where the other - * emulator answers at once and the spawned one checks its ports only once it has started. - * So in this fake its exit is seen only once it is killed: a driver that waited for that - * exit would already have readied the device. - */ -class PortTakingMachine { - readonly #inner: ProcessRunner; - readonly #otherPaths = new Map(); - #takeOnNextBoot: ((port: number) => void) | undefined; - - constructor(private readonly host: EmulatorHost) { - this.#inner = host.runner(); - } - - avdPathAnswering(serial: string | undefined): string | undefined { - const path = this.host.pathAnswering(serial); - if (path === undefined) return undefined; - return this.#otherPaths.get(consolePortOf(serial)) ?? path; - } - - takePortOnNextBoot(avdPath: string): { - observed(): { running: boolean; sentByDriver: string[] }; - } { - let running = false; - let port = Number.NaN; - const sent: string[] = []; - this.#takeOnNextBoot = (taken) => { - port = taken; - const avdName = /([^/]+)\.avd$/.exec(avdPath)?.[1] ?? ""; - const handle = this.#inner.spawn( - binaries.emulator, - ["-avd", avdName, "-port", String(taken)], - emulatorOptions, - ); - running = true; - void handle.wait().then(() => { - running = false; - }); - this.#otherPaths.set(taken, avdPath); - }; - this.#sentTo = (serial, command) => { - if (consolePortOf(serial) === port && running && !QUERIES.test(command)) sent.push(command); - }; - return { observed: () => ({ running, sentByDriver: sent }) }; - } - - #sentTo: (serial: string, command: string) => void = () => undefined; - - runner(): ProcessRunner { - return { - run: async (command, args, options) => { - if (command === binaries.adb && args[0] === "-s") { - const serial = args[1] ?? ""; - const rest = args.slice(2).join(" "); - this.#sentTo(serial, rest); - const answer = this.#identity(serial, rest); - if (answer !== undefined) return answer; - } - return this.#inner.run(command, args, options); - }, - spawn: (command, args, options) => { - const take = this.#takeOnNextBoot; - if (take === undefined || command !== binaries.emulator) { - return this.#inner.spawn(command, args, options); - } - this.#takeOnNextBoot = undefined; - take(Number(args[args.indexOf("-port") + 1])); - const handle = this.#inner.spawn(command, args, options); - let killed!: () => void; - const exitSeen = new Promise((resolve) => { - killed = resolve; - }); - return { - ...handle, - kill: (signal) => { - handle.kill(signal); - killed(); - }, - wait: async () => { - await exitSeen; - return handle.wait(); - }, - }; - }, - spawnStreaming: (command, args, options) => - this.#inner.spawnStreaming(command, args, options), - }; - } - - #identity(serial: string, command: string): ProcessResult | undefined { - const path = this.avdPathAnswering(serial); - if (path === undefined) return undefined; - const name = /([^/]+)\.avd$/.exec(path)?.[1] ?? ""; - if (command === "emu avd path") return ok(`${path}\nOK\n`); - if (command === "emu avd name") return ok(`${name}\nOK\n`); - if (command === "shell getprop ro.boot.qemu.avd_name") return ok(`${name}\n`); - return undefined; - } -} - -/** why: `emu avd path` answers truthfully until the test sets an answer, so a seed is made first and only the copy's boot is lied to. */ -function overridingAvdPath(inner: ProcessRunner): { - readonly answer: (answer: ProcessResult) => void; - readonly runner: ProcessRunner; -} { - let answer: ProcessResult | undefined; - return { - answer: (next) => { - answer = next; - }, - runner: { - run: async (command, args, options) => - answer !== undefined && - command === binaries.adb && - args[0] === "-s" && - args.slice(2).join(" ") === "emu avd path" - ? answer - : inner.run(command, args, options), - spawn: (command, args, options) => inner.spawn(command, args, options), - spawnStreaming: (command, args, options) => inner.spawnStreaming(command, args, options), - }, - }; -} - -class LinkedHomeFilesystem extends MemoryFilesystem { - override async realpath(path: string): Promise { - const resolved = await super.realpath(path.replace(/^\/private\/home\//, "/home/")); - return resolved.replace(/^\/home\//, "/private/home/"); - } -} - -const QUERIES = /^(shell getprop \S+|emu avd (name|path|id|status|discoverypath))$/; - -/** - * why: The message of the driver's `DriverCrashError`; anything else -- a boot timeout, the fake's - * own complaint about an unscripted call -- is rethrown, so it fails the test as itself. - */ -function driverCrashMessage(error: unknown): { readonly refused: string } { - if (error instanceof Error && error.name === "DriverCrashError") { - return { refused: error.message }; - } - throw error; -} - -function escapeRegExp(text: string): string { - return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} diff --git a/src/drivers/android/test-index-lifecycle.ts b/src/drivers/android/test-index-lifecycle.ts new file mode 100644 index 000000000..8192f7b2b --- /dev/null +++ b/src/drivers/android/test-index-lifecycle.ts @@ -0,0 +1,14 @@ +import { type ScriptedProcessExpectation } from "../../ports/index.js"; +import { avdDirectory, binaries, processResult } from "./testing.js"; + +/** why: what a running AVD made before copies answers to `emu avd path`: `/.avd`. */ +export function avdPathExpectation( + serial: string, + avdName = "simlock_one", +): ScriptedProcessExpectation { + return processResult( + binaries.adb, + ["-s", serial, "emu", "avd", "path"], + `${avdDirectory}/${avdName}.avd\r\nOK\r\n`, + ); +} From 0c7e3eaf4e596be433e2546d40e9c8b839088b9f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:22 +0100 Subject: [PATCH 35/83] test(drivers): split physical-handler.test.ts by topic --- .../physical-handler-inspect-list.test.ts | 337 ++++++ .../physical-handler-passthrough.test.ts | 297 ++++++ .../android/physical-handler-windows.test.ts | 80 ++ src/drivers/android/physical-handler.test.ts | 999 +----------------- src/drivers/android/test-physical-handler.ts | 321 ++++++ 5 files changed, 1070 insertions(+), 964 deletions(-) create mode 100644 src/drivers/android/physical-handler-inspect-list.test.ts create mode 100644 src/drivers/android/physical-handler-passthrough.test.ts create mode 100644 src/drivers/android/physical-handler-windows.test.ts create mode 100644 src/drivers/android/test-physical-handler.ts diff --git a/src/drivers/android/physical-handler-inspect-list.test.ts b/src/drivers/android/physical-handler-inspect-list.test.ts new file mode 100644 index 000000000..903ebbbcb --- /dev/null +++ b/src/drivers/android/physical-handler-inspect-list.test.ts @@ -0,0 +1,337 @@ +import { describe, expect, it } from "vitest"; +import { FakeTcpProbe, ProcessSpawnError } from "../../ports/index.js"; +import { AndroidPhysicalHandler } from "./physical-handler.js"; +import { + sdk, + adb, + PHONE, + VERSION_REPLY_41, + VERSION_REPLY_40, + USB_LINE, + FAILED, + TIMED_OUT, + PhoneAdb, + world, + androidFilesystem, + devicesListing, + isDevicesListing, + isGetprop, + isUserRead, + isPackageList, +} from "./test-physical-handler.js"; + +// why: the physical handler is a part of a decomposed module, so its tests reach it directly. +describe("Android physical handler: inspectPhysical", () => { + it("returns the serial, ro.product.model, the API level, class phone and the packages pm list packages -3 --user lists for the id am get-current-user prints", async () => { + const { driver, runner } = await world(); + runner.userOutput = "10\n"; + runner.installed.splice(0, runner.installed.length, "com.example.kept", "com.example.stray"); + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toEqual({ + apps: ["com.example.kept", "com.example.stray"], + class: "phone", + driverDeviceId: PHONE, + enrollable: true, + model: "SM-G991B", + osVersion: "34", + }); + expect(runner.rests).toContainEqual([ + "-s", + PHONE, + "shell", + "pm", + "list", + "packages", + "-3", + "--user", + "10", + ]); + expect(runner.rests).toContainEqual(["-s", PHONE, "shell", "getprop", "ro.product.model"]); + }); + + it("refuses a serial in state unauthorized as untrusted, and runs no command naming it", async () => { + const { driver, runner } = await world(); + expect(await driver.inspectPhysical(PHONE)).toMatchObject({ enrollable: true }); + runner.calls.length = 0; + runner.devicesOutput = devicesListing(`${PHONE} unauthorized usb:1-1 transport_id:3`); + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toMatchObject({ enrollable: false, reason: "untrusted" }); + expect(runner.commands.filter((call) => call.args.includes("-s"))).toEqual([]); + }); + + it.each([ + [ + "a network serial", + "192.168.1.5:5555 device product:o1s model:SM_G991B transport_id:4", + "192.168.1.5:5555", + ], + [ + "an mDNS name", + "adb-R58M123ABC-xyz._adb-tls-connect._tcp device product:o1s model:SM_G991B transport_id:4", + "adb-R58M123ABC-xyz._adb-tls-connect._tcp", + ], + [ + "a listed emulator", + "emulator-5554 device product:sdk_gphone model:sdk transport_id:1", + "emulator-5554", + ], + [ + "a USB-shaped serial on a network line", + `${PHONE} device product:o1s model:SM_G991B transport_id:3`, + PHONE, + ], + ])( + "refuses %s as network-only with the USB-only message, and runs no command naming it", + async (_name, line, id) => { + const { driver, runner } = await world(); + runner.devicesOutput = devicesListing(line); + + const inspection = await driver.inspectPhysical(id); + + expect(inspection).toMatchObject({ enrollable: false, reason: "network-only" }); + expect((inspection as { message: string }).message).toContain( + "only USB-connected devices can be enrolled", + ); + expect(runner.commands.filter((call) => call.args.includes("-s"))).toEqual([]); + }, + ); + + it("inspects a host:port-shaped serial on a USB transport: the transport decides, not the shape", async () => { + const { driver, runner } = await world(); + runner.devicesOutput = devicesListing( + "192.168.1.5:5555 device usb:1-1 product:o1s model:SM_G991B transport_id:5", + ); + + const inspection = await driver.inspectPhysical("192.168.1.5:5555"); + + expect(inspection).toMatchObject({ driverDeviceId: "192.168.1.5:5555", enrollable: true }); + }); + + it.each([ + ["not listed", devicesListing()], + ["listed offline", devicesListing(`${PHONE} offline usb:1-1 transport_id:3`)], + ])("refuses a serial that is %s as not-connected", async (_name, listing) => { + const { driver, runner } = await world(); + expect(await driver.inspectPhysical(PHONE)).toMatchObject({ enrollable: true }); + runner.devicesOutput = listing; + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toMatchObject({ enrollable: false, reason: "not-connected" }); + }); + + it.each([ + [ + "adb devices -l fails", + (rest: readonly string[]) => (isDevicesListing(rest) ? FAILED : undefined), + /could not read the device list \(adb devices -l\)/, + ], + [ + "adb devices -l passes 30 s", + (rest: readonly string[]) => (isDevicesListing(rest) ? TIMED_OUT : undefined), + /could not read the device list \(adb devices -l\)/, + ], + [ + "getprop fails", + (rest: readonly string[]) => (isGetprop(rest) ? FAILED : undefined), + /could not read the model/, + ], + [ + "the user id read fails", + (rest: readonly string[]) => (isUserRead(rest) ? FAILED : undefined), + /could not read the current user id/, + ], + [ + "the package list fails", + (rest: readonly string[]) => (isPackageList(rest) ? FAILED : undefined), + /could not read the user apps/, + ], + ])( + "is refused unreadable naming the read, and does not reject, when %s", + async (_name, override, named) => { + const { driver, runner } = await world(); + runner.override(override); + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); + expect((inspection as { message: string }).message).toMatch(named); + }, + ); + + it("is refused unreadable, and does not reject, when the process runner cannot spawn adb", async () => { + const { driver, runner } = await world(); + runner.override((rest) => { + if (isDevicesListing(rest)) throw new ProcessSpawnError(adb, rest); + return undefined; + }); + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); + }); + + it("lists nothing present, and does not reject, when the process runner cannot spawn adb", async () => { + const { driver, runner } = await world(); + runner.override((rest) => { + if (isDevicesListing(rest)) throw new ProcessSpawnError(adb, rest); + return undefined; + }); + + const present = await driver.listPresentPhysical([PHONE]); + + expect(present).toEqual([{ driverDeviceId: PHONE, present: false, reason: "not-present" }]); + }); + + it.each([ + ["a shell injection", "0; rm -r /sdcard\n"], + ["nothing", ""], + ["seven digits", "1234567\n"], + ])( + "is refused unreadable and runs no pm list packages when am get-current-user prints %s", + async (_name, printed) => { + const { driver, runner } = await world(); + runner.userOutput = printed; + + const inspection = await driver.inspectPhysical(PHONE); + + expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); + expect(runner.rests.some(isPackageList)).toBe(false); + }, + ); +}); + +describe("Android physical handler: listPresentPhysical", () => { + it("answers a USB serial in state device present with its API level, an unauthorized one not present untrusted, and an offline, network-only or missing one not present not-present", async () => { + const { driver, runner } = await world(); + runner.devicesOutput = devicesListing( + USB_LINE, + "UNAUTH1 unauthorized usb:1-2 transport_id:4", + "OFFLINE1 offline usb:1-3 transport_id:5", + "192.168.1.5:5555 device product:o1s transport_id:6", + ); + + const present = await driver.listPresentPhysical([ + PHONE, + "UNAUTH1", + "OFFLINE1", + "192.168.1.5:5555", + "MISSING1", + ]); + + expect(present).toEqual([ + { driverDeviceId: PHONE, osVersion: "34", present: true }, + { driverDeviceId: "UNAUTH1", present: false, reason: "untrusted" }, + { driverDeviceId: "OFFLINE1", present: false, reason: "not-present" }, + { driverDeviceId: "192.168.1.5:5555", present: false, reason: "not-present" }, + { driverDeviceId: "MISSING1", present: false, reason: "not-present" }, + ]); + }); + + it("gives no entry, and runs no command, for a USB phone that is not among the given serials", async () => { + const { driver, runner } = await world(); + runner.devicesOutput = devicesListing(USB_LINE, "OTHER99 device usb:1-2 transport_id:4"); + + const present = await driver.listPresentPhysical([PHONE]); + + expect(present).toEqual([{ driverDeviceId: PHONE, osVersion: "34", present: true }]); + expect(runner.rests.flat()).not.toContain("OTHER99"); + }); + + it.each([ + ["fails", FAILED], + ["has no exit code", TIMED_OUT], + ])( + "answers every given serial not present not-present, and does not reject, when adb devices -l %s", + async (_name, result) => { + const { driver, runner } = await world(); + runner.override((rest) => (isDevicesListing(rest) ? result : undefined)); + + const present = await driver.listPresentPhysical([PHONE, "SERIAL2"]); + + expect(present).toEqual([ + { driverDeviceId: PHONE, present: false, reason: "not-present" }, + { driverDeviceId: "SERIAL2", present: false, reason: "not-present" }, + ]); + }, + ); + + it("answers a present serial whose getprop fails not present not-present", async () => { + const { driver, runner } = await world(); + runner.override((rest) => (isGetprop(rest) ? FAILED : undefined)); + + const present = await driver.listPresentPhysical([PHONE]); + + expect(present).toEqual([{ driverDeviceId: PHONE, present: false, reason: "not-present" }]); + }); +}); + +describe("Android physical handler: listAttachedPhysical", () => { + // why: the Windows rule needs `platform: "win32"`, which the driver surface cannot fake per host; + // the handler is built straight, as the Windows describe below does. + async function handlerFor( + listing: string, + options: { readonly platform?: NodeJS.Platform; readonly reply?: string } = {}, + ) { + const runner = new PhoneAdb([]); + runner.devicesOutput = listing; + const probe = new FakeTcpProbe([]); + probe.replyWith(options.reply ?? VERSION_REPLY_41); + const handler = new AndroidPhysicalHandler({ + adb, + baseEnv: { ANDROID_HOME: sdk }, + filesystem: await androidFilesystem(), + isBareShell: () => false, + platform: options.platform ?? "linux", + processRunner: runner, + tcpProbe: probe, + }); + return { handler, runner }; + } + + it("lists the phones on USB in any state, none on the network or an emulator, throws when the list cannot be read, and runs no adb command at a protocol mismatch", async () => { + const listings = [ + { + listing: devicesListing( + USB_LINE, + "UNAUTH1 unauthorized usb:1-2 transport_id:4", + "OFFLINE1 offline usb:1-3 transport_id:5", + "192.168.1.5:5555 device product:o1s transport_id:6", + "emulator-5554 device product:sdk transport_id:7", + ), + listed: [PHONE, "UNAUTH1", "OFFLINE1"], + name: "linux", + platform: "linux", + }, + { + listing: devicesListing( + `${PHONE} device product:a70qxtc transport_id:1`, + "192.168.1.5:5555 device product:o1s transport_id:6", + ), + listed: [PHONE], + name: "windows, with no usb: path and the header line", + platform: "win32", + }, + ] as const; + for (const { listing, listed, name, platform } of listings) { + const { handler, runner } = await handlerFor(listing, { platform }); + + await expect(handler.listAttachedPhysical(), name).resolves.toEqual([...listed].sort()); + expect(runner.rests, name).toEqual([["devices", "-l"]]); + } + + const failing = await handlerFor(devicesListing(USB_LINE)); + failing.runner.override((rest) => (isDevicesListing(rest) ? FAILED : undefined)); + await expect(failing.handler.listAttachedPhysical()).rejects.toThrow( + /could not read the device list \(adb devices -l\)/, + ); + + const mismatch = await handlerFor(devicesListing(USB_LINE), { reply: VERSION_REPLY_40 }); + await expect(mismatch.handler.listAttachedPhysical()).rejects.toThrow(/speaks protocol 40/); + expect(mismatch.runner.commands).toEqual([]); + }); +}); diff --git a/src/drivers/android/physical-handler-passthrough.test.ts b/src/drivers/android/physical-handler-passthrough.test.ts new file mode 100644 index 000000000..564b1b33b --- /dev/null +++ b/src/drivers/android/physical-handler-passthrough.test.ts @@ -0,0 +1,297 @@ +import { describe, expect, it } from "vitest"; +import { PassthroughRefusedError, ScreenUnavailableError } from "../../core/index.js"; +import { FakeTcpProbe, FakeTcpStreamConnector } from "../../ports/index.js"; +import { AndroidDriver } from "./index.js"; +import { AndroidPhysicalHandler } from "./physical-handler.js"; +import type { ScreenTarget } from "./screen/index.js"; +import { + sdk, + adb, + PHONE, + VERSION_REPLY_41, + VERSION_REPLY_40, + PHYSICAL, + PHYSICAL_SPEC, + PhoneAdb, + world, + androidFilesystem, + defaultServerSends, + NO_DEVICE, + passthroughOf, + rejectionOf, +} from "./test-physical-handler.js"; + +const EMULATOR_DEVICE = { + address: "", + deviceId: "device", + driverData: {}, + physical: false, +} as const; + +const VIRTUAL_SPEC = { + model: "Pixel 8", + osVersion: "34", + physical: false, + platform: "android", +} as never; + +const REFUSED_SUBCOMMANDS = [ + "kill-server", + "start-server", + "server", + "nodaemon", + "connect", + "disconnect", + "pair", + "reconnect", + "tcpip", + "usb", + "detach", + "reboot-bootloader", +]; + +const REFUSED_REBOOT_TARGETS = [ + "bootloader", + "sideload", + "sideload-auto-reboot", + "recovery", + "fastboot", +]; + +const REFUSED_LEADING_FLAGS = [ + "-s", + "-t", + "-d", + "-e", + "-P", + "-H", + "-L", + "--one-device", + "--reply-fd", +]; + +async function expectPassthroughAccepted(driver: AndroidDriver): Promise { + const accepted = await passthroughOf(driver, ["shell", "getprop"]); + expect(accepted.args).toEqual(["-P", "5037", "-s", PHONE, "shell", "getprop"]); +} + +// why: the physical handler is a part of a decomposed module, so its tests reach it directly. +describe("Android physical handler: passthrough", () => { + it("resolves passthrough of adb shell getprop on a physical device to adb -P 5037 -s shell getprop with ANDROID_ADB_SERVER_PORT=5037", async () => { + const { driver } = await world(); + + const command = await passthroughOf(driver, ["shell", "getprop"]); + + expect(command).toMatchObject({ + args: ["-P", "5037", "-s", PHONE, "shell", "getprop"], + command: adb, + env: { ANDROID_ADB_SERVER_PORT: "5037" }, + }); + }); + + it.each(REFUSED_LEADING_FLAGS)( + "refuses the caller argument %s before the subcommand", + async (flag) => { + const { driver } = await world(); + await expectPassthroughAccepted(driver); + + const failure = await rejectionOf(passthroughOf(driver, [flag, "x", "shell", "getprop"])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + }, + ); + + it.each(REFUSED_SUBCOMMANDS)("refuses %s as the subcommand", async (subcommand) => { + const { driver } = await world(); + await expectPassthroughAccepted(driver); + + const failure = await rejectionOf(passthroughOf(driver, [subcommand])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + }); + + it.each(REFUSED_REBOOT_TARGETS)("refuses reboot followed by %s", async (target) => { + const { driver } = await world(); + expect(await passthroughOf(driver, ["reboot"])).toBeDefined(); + + const failure = await rejectionOf(passthroughOf(driver, ["reboot", target])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + }); + + it("refuses a bare shell with no terminal", async () => { + const { driver } = await world(); + await expectPassthroughAccepted(driver); + + const failure = await rejectionOf(passthroughOf(driver, ["shell"])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + }); + + it.each([ + ["shell dumpsys usb", ["shell", "dumpsys", "usb"]], + ["shell echo connect", ["shell", "echo", "connect"]], + ["reboot", ["reboot"]], + ])( + "runs %s: a refused word after the subcommand is an operand, and a plain reboot is not refused", + async (_name, args) => { + const { driver } = await world(); + + const command = await passthroughOf(driver, args); + + expect(command.args).toEqual(["-P", "5037", "-s", PHONE, ...args]); + }, + ); + + it("is refused with PassthroughRefusedError naming both versions when its check sees another version, and the process runner starts nothing", async () => { + const { driver, runner } = await world({ serverReply: VERSION_REPLY_40 }); + + const failure = await rejectionOf(passthroughOf(driver, ["shell", "getprop"])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + expect((failure as Error).message).toMatch(/40.*41|41.*40/s); + expect(runner.commands).toEqual([]); + }); + + it("is refused with PassthroughRefusedError naming which version it cannot read, and the process runner starts nothing", async () => { + const { driver, runner } = await world({ serverReply: "" }); + + const failure = await rejectionOf(passthroughOf(driver, ["shell", "getprop"])); + + expect(failure).toBeInstanceOf(PassthroughRefusedError); + expect((failure as Error).message).toMatch( + /the adb server's version on port 5037 is unreadable/, + ); + expect(runner.commands).toEqual([]); + }); + + it("returns unsetEnv ANDROID_ADB_SERVER_ADDRESS, ADB_SERVER_SOCKET and ANDROID_SERIAL for a physical device, and none with no device", async () => { + const { driver } = await world(); + + const physical = await passthroughOf(driver, ["shell", "getprop"]); + const none = await passthroughOf(driver, ["shell", "getprop"], NO_DEVICE); + + expect([...(physical.unsetEnv ?? [])].sort()).toEqual([ + "ADB_SERVER_SOCKET", + "ANDROID_ADB_SERVER_ADDRESS", + "ANDROID_SERIAL", + ]); + expect(none.unsetEnv ?? []).toEqual([]); + }); + + it("with no device, or with an emulator device, still runs on Simlock's own server and sends no host:version", async () => { + const { driver, probe } = await world(); + + const without = await passthroughOf(driver, ["shell", "getprop"], NO_DEVICE); + const emulator = await passthroughOf(driver, ["shell", "getprop"], EMULATOR_DEVICE); + + expect(without).toEqual({ + args: ["-P", "5038", "shell", "getprop"], + command: adb, + env: { ANDROID_ADB_SERVER_PORT: "5038" }, + }); + expect(emulator).toEqual(without); + expect(defaultServerSends(probe)).toEqual([]); + }); +}); + +describe("Android physical handler: the screen target", () => { + // why: the target a phone's scrcpy session runs on, built straight from the handler: the capture + // reaches it only through the driver, and what it must never see is the port or the check. + async function screenWorld(options: { readonly reply?: string } = {}): Promise<{ + readonly runner: PhoneAdb; + readonly target: ScreenTarget; + readonly tcp: FakeTcpStreamConnector; + }> { + const runner = new PhoneAdb([]); + const probe = new FakeTcpProbe([]); + probe.replyWith(options.reply ?? VERSION_REPLY_41); + const handler = new AndroidPhysicalHandler({ + adb, + // why: an env that aims adb elsewhere on purpose — a target that leaked these to a child + // would reach a server other than the one it just checked. + baseEnv: { + ADB_SERVER_SOCKET: "tcp:10.0.0.5:5037", + ANDROID_ADB_SERVER_ADDRESS: "10.0.0.5", + ANDROID_HOME: sdk, + ANDROID_SERIAL: "SOMEONE-ELSES-PHONE", + }, + filesystem: await androidFilesystem(), + isBareShell: () => false, + processRunner: runner, + tcpProbe: probe, + }); + const tcp = new FakeTcpStreamConnector(); + return { runner, target: handler.screenTargetFor(PHYSICAL, tcp), tcp }; + } + + it("sends nothing and refuses with the check's own message when the check is closed, for run, spawn and connect, at a mismatch and an unreadable server", async () => { + const closed = [ + { message: /speaks protocol 40/, name: "another protocol", reply: VERSION_REPLY_40 }, + { message: /unreadable/, name: "a server it cannot read", reply: "gibberish" }, + ] as const; + const verbs = [ + { + call: (target: ScreenTarget) => target.run(["shell", "dumpsys", "power"], 10_000), + name: "run", + }, + { + call: (target: ScreenTarget) => target.spawn(["shell", "app_process"]), + name: "spawn", + }, + { + call: (target: ScreenTarget) => target.connect(5_000), + name: "connect", + }, + ] as const; + for (const refusal of closed) { + for (const verb of verbs) { + const world = await screenWorld({ reply: refusal.reply }); + + const error = await rejectionOf(verb.call(world.target)); + + expect(error, `${refusal.name} via ${verb.name}`).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message, `${refusal.name} via ${verb.name}`).toMatch( + refusal.message, + ); + expect(world.runner.commands, `${refusal.name} via ${verb.name}`).toEqual([]); + expect(world.tcp.connections, `${refusal.name} via ${verb.name}`).toEqual([]); + } + } + }); + + it("runs adb -P 5037 with the shared server's env for run and spawn, and connects to 5037, when the check is open", async () => { + const world = await screenWorld(); + + await world.target.run(["shell", "dumpsys", "power"], 10_000); + await world.target.spawn(["shell", "app_process"]); + await world.target.connect(4_000); + + expect(world.runner.calls.map((call) => [...call.args])).toEqual([ + ["version"], + ["-P", "5037", "-s", PHONE, "shell", "dumpsys", "power"], + ["-P", "5037", "-s", PHONE, "shell", "app_process"], + ]); + for (const call of world.runner.commands) { + expect(call.options.env).toEqual({ + ANDROID_ADB_SERVER_PORT: "5037", + ANDROID_HOME: sdk, + }); + } + expect(world.runner.commands[0]?.options.timeoutMs).toBe(10_000); + expect(world.tcp.connections.map((connection) => connection.port)).toEqual([5037]); + }); +}); + +describe("Android physical handler: estimate", () => { + it("is 90 000 ms for a reclaim of a physical spec at both clean levels, and unchanged for an emulator spec", async () => { + const { driver } = await world(); + + expect(driver.estimate({ clean: "standard", operation: "reclaim" }, PHYSICAL_SPEC)).toBe( + 90_000, + ); + expect(driver.estimate({ clean: "full", operation: "reclaim" }, PHYSICAL_SPEC)).toBe(90_000); + expect(driver.estimate({ clean: "standard", operation: "reclaim" }, VIRTUAL_SPEC)).toBe(6_000); + expect(driver.estimate({ clean: "full", operation: "reclaim" }, VIRTUAL_SPEC)).toBe(32_000); + }); +}); diff --git a/src/drivers/android/physical-handler-windows.test.ts b/src/drivers/android/physical-handler-windows.test.ts new file mode 100644 index 000000000..9f00329fb --- /dev/null +++ b/src/drivers/android/physical-handler-windows.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { FakeTcpProbe } from "../../ports/index.js"; +import { AndroidPhysicalHandler } from "./physical-handler.js"; +import { + sdk, + adb, + PHONE, + VERSION_REPLY_41, + PhoneAdb, + androidFilesystem, + devicesListing, +} from "./test-physical-handler.js"; + +// why: the physical handler is a part of a decomposed module, so its tests reach it directly. +describe("Android physical handler on Windows", () => { + // why: adb on Windows lists no `usb:` path, which the driver surface cannot fake per host; + // the handler is built straight with `platform: "win32"`. + async function windowsHandler(listing: string, platform: NodeJS.Platform = "win32") { + const runner = new PhoneAdb([]); + runner.devicesOutput = listing; + const probe = new FakeTcpProbe([]); + probe.replyWith(VERSION_REPLY_41); + return new AndroidPhysicalHandler({ + adb, + baseEnv: { ANDROID_HOME: sdk }, + filesystem: await androidFilesystem(), + isBareShell: () => false, + platform, + processRunner: runner, + tcpProbe: probe, + }); + } + + const WINDOWS_LINE = `${PHONE} device product:a70qxtc model:SM_A705MN device:a70q transport_id:1`; + + it("enrols a phone adb lists with no usb: path, as adb on Windows lists every USB device", async () => { + const handler = await windowsHandler(devicesListing(WINDOWS_LINE)); + + await expect(handler.inspectPhysical(PHONE)).resolves.toMatchObject({ + driverDeviceId: PHONE, + enrollable: true, + }); + }); + + it("still refuses the same listing off Windows, where a USB device always has a usb: path", async () => { + const handler = await windowsHandler(devicesListing(WINDOWS_LINE), "linux"); + + await expect(handler.inspectPhysical(PHONE)).resolves.toMatchObject({ + enrollable: false, + reason: "network-only", + }); + }); + + it.each([ + ["win32", { driverDeviceId: PHONE, osVersion: "34", present: true }], + ["linux", { driverDeviceId: PHONE, present: false, reason: "not-present" }], + ] as const)( + "lists an enrolled phone with no usb: path on %s as the USB rule of that host says", + async (platform, expected) => { + const handler = await windowsHandler(devicesListing(WINDOWS_LINE), platform); + + await expect(handler.listPresentPhysical([PHONE])).resolves.toEqual([expected]); + }, + ); + + it.each([ + ["a host:port", "192.168.1.5:5555"], + ["an mDNS service", "adb-R58M123ABC-x1Yz2W._adb-tls-connect._tcp"], + ["an emulator", "emulator-5554"], + ])("refuses %s serial on Windows as network-only", async (_kind, serial) => { + const handler = await windowsHandler( + devicesListing(`${serial} device product:o1s transport_id:5`), + ); + + await expect(handler.inspectPhysical(serial)).resolves.toMatchObject({ + enrollable: false, + reason: "network-only", + }); + }); +}); diff --git a/src/drivers/android/physical-handler.test.ts b/src/drivers/android/physical-handler.test.ts index b3be418ae..f1b21ef9a 100644 --- a/src/drivers/android/physical-handler.test.ts +++ b/src/drivers/android/physical-handler.test.ts @@ -1,18 +1,8 @@ import { describe, expect, it } from "vitest"; - -import { - OWNED_ROOT_MARKER_FILE, - PassthroughRefusedError, - ScreenUnavailableError, -} from "../../core/index.js"; +import { PassthroughRefusedError } from "../../core/index.js"; import { - FakeClock, - FakeProcessSupervisor, FakeTcpListenerFactory, FakeTcpProbe, - FakeTcpStreamConnector, - type Filesystem, - MemoryFilesystem, type ProcessHandle, type ProcessResult, type ProcessRunner, @@ -23,216 +13,36 @@ import { type StreamingProcessHandle, } from "../../ports/index.js"; import { AndroidDriver } from "./index.js"; -import { emulatorHelpListing, memoryAndroidSourceDatabase } from "./testing.js"; -import { AndroidPhysicalHandler } from "./physical-handler.js"; -import type { ScreenTarget } from "./screen/index.js"; - -const sdk = "/android-sdk"; -const home = "/home/simlock"; -const simlockHome = "/home/simlock/.simlock"; -const instanceId = "instance-1"; -const adb = `${sdk}/platform-tools/adb`; -const avdDirectory = `${simlockHome}/devices/android`; - -const PHONE = "R58M123ABC"; -const VERSION_REPLY_41 = "OKAY00040029"; -const VERSION_REPLY_40 = "OKAY00040028"; -const HOST_VERSION = "000chost:version"; -const ADB_VERSION_41 = - "Android Debug Bridge version 1.0.41\nVersion 35.0.2-12147458\nInstalled as /android-sdk/platform-tools/adb\n"; -const ADB_VERSION_40 = "Android Debug Bridge version 1.0.40\nVersion 34.0.0-9570255\n"; - -const USB_LINE = `${PHONE} device usb:1-1 product:o1s model:SM_G991B device:o1s transport_id:3`; -const PHYSICAL = { - address: PHONE, - deviceId: PHONE, - driverData: {}, - enrolledApps: ["com.example.kept", "com.example.gone"], - physical: true, -} as const; -const EMULATOR_DEVICE = { - address: "", - deviceId: "device", - driverData: {}, - physical: false, -} as const; -const PHYSICAL_SPEC = { - model: "SM-G991B", - osVersion: "34", - physical: true, - platform: "android", -} as never; -const VIRTUAL_SPEC = { - model: "Pixel 8", - osVersion: "34", - physical: false, - platform: "android", -} as never; - -const REFUSED_SUBCOMMANDS = [ - "kill-server", - "start-server", - "server", - "nodaemon", - "connect", - "disconnect", - "pair", - "reconnect", - "tcpip", - "usb", - "detach", - "reboot-bootloader", -]; -const REFUSED_REBOOT_TARGETS = [ - "bootloader", - "sideload", - "sideload-auto-reboot", - "recovery", - "fastboot", -]; -const REFUSED_LEADING_FLAGS = [ - "-s", - "-t", - "-d", - "-e", - "-P", - "-H", - "-L", - "--one-device", - "--reply-fd", -]; - -interface Call { - readonly command: string; - readonly args: readonly string[]; - readonly options: ProcessRunOptions; -} - -type Override = (rest: readonly string[]) => ProcessResult | "hang" | undefined; - -const OK = (stdout = ""): ProcessResult => ({ code: 0, stderr: "", stdout }); -const FAILED: ProcessResult = { code: 1, stderr: "error: closed", stdout: "" }; -const TIMED_OUT: ProcessResult = { code: null, stderr: "", stdout: "" }; - -class PhoneAdb implements ProcessRunner { - readonly calls: Call[] = []; - readonly log: string[]; - readonly installed: string[] = ["com.example.kept", "com.example.stray", "com.example.other"]; - versionOutput = ADB_VERSION_41; - versionCode: number | null = 0; - versionFailure: Error | undefined; - devicesOutput = `List of devices attached\n${USB_LINE}\n\n`; - userOutput = "0\n"; - packagesOutput: string | undefined; - model = "SM-G991B"; - sdkLevel = "34"; - onCall: ((rest: readonly string[]) => void) | undefined; - readonly #overrides: Override[] = []; - - constructor(log: string[] = []) { - this.log = log; - } - - override(answer: Override): void { - this.#overrides.push(answer); - } - - get commands(): Call[] { - return this.calls.filter((call) => call.args[0] !== "version"); - } - - get rests(): string[][] { - return this.commands.map((call) => restOf(call)); - } - - async run( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): Promise { - this.calls.push({ args: [...args], command, options }); - this.log.push(`adb:${args.join(" ")}`); - if (args[0] === "version") { - if (this.versionFailure !== undefined) throw this.versionFailure; - return { code: this.versionCode, stderr: "", stdout: this.versionOutput }; - } - if (args[0] === "devices") { - return OK("List of devices attached\n"); - } - const rest = args[0] === "-P" ? args.slice(2) : args; - this.onCall?.(rest); - for (const answer of this.#overrides) { - const result = answer(rest); - if (result === "hang") return new Promise(() => undefined); - if (result !== undefined) return result; - } - return this.#answer(rest); - } - - #answer(rest: readonly string[]): ProcessResult { - const line = rest.join(" "); - if (line === "devices -l") return OK(this.devicesOutput); - // why: only a command scoped with -s reaches the phone; an unscoped one is unexpected. - const body = rest[0] === "-s" ? rest.slice(2).join(" ") : ""; - const fixed = new Map([ - ["shell am get-current-user", this.userOutput], - ["shell getprop ro.build.version.sdk", `${this.sdkLevel}\n`], - ["shell getprop ro.product.model", `${this.model}\n`], - ]); - const answered = fixed.get(body); - if (answered !== undefined) return OK(answered); - if (body.startsWith("shell pm list packages -3 --user ")) return OK(this.#packageListing()); - const uninstall = /^shell pm uninstall --user \d+ (\S+)$/.exec(body); - if (uninstall === null) return { code: 1, stderr: `fake adb: unexpected ${line}`, stdout: "" }; - this.installed.splice( - 0, - this.installed.length, - ...this.installed.filter((n) => n !== uninstall[1]), - ); - return OK("Success\n"); - } - - #packageListing(): string { - return this.packagesOutput ?? this.installed.map((name) => `package:${name}\n`).join(""); - } - - spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { - const result = this.run(command, args, options); - async function* none(): AsyncIterable { - yield* [] as string[]; - } - return { - kill: () => undefined, - pid: 1, - stderr: none(), - stdout: none(), - unref: () => undefined, - wait: () => result, - }; - } - - spawnStreaming( - command: string, - _args: readonly string[], - _options: ProcessStreamOptions, - ): StreamingProcessHandle { - throw new Error(`fake adb: spawnStreaming ${command} is not used by a driver`); - } -} - -class LoggingTcpProbe extends FakeTcpProbe { - readonly #log: string[]; +import { emulatorHelpListing } from "./testing.js"; +import { + sdk, + adb, + avdDirectory, + PHONE, + VERSION_REPLY_41, + VERSION_REPLY_40, + ADB_VERSION_41, + PHYSICAL, + PHYSICAL_SPEC, + OK, + FAILED, + TIMED_OUT, + PhoneAdb, + world, + androidFilesystem, + restOf, + isDevicesListing, + isGetprop, + isUserRead, + isPackageList, + defaultServerSends, + passthroughOf, + rejectionOf, +} from "./test-physical-handler.js"; - constructor(log: string[], listening: readonly number[]) { - super(listening); - this.#log = log; - } +const HOST_VERSION = "000chost:version"; - override async send(port: number, payload: string, timeoutMs?: number): Promise { - this.#log.push(`send:${String(port)}:${payload}`); - return super.send(port, payload, timeoutMs); - } -} +const ADB_VERSION_40 = "Android Debug Bridge version 1.0.40\nVersion 34.0.0-9570255\n"; class SplitRunner implements ProcessRunner { constructor( @@ -269,102 +79,10 @@ class SplitRunner implements ProcessRunner { } } -interface World { - readonly driver: AndroidDriver; - readonly runner: PhoneAdb; - readonly probe: LoggingTcpProbe; - readonly log: string[]; - readonly filesystem: Filesystem; -} - -async function world( - options: { - readonly env?: Readonly>; - readonly runner?: PhoneAdb; - readonly filesystem?: Filesystem; - readonly processRunner?: (phone: PhoneAdb) => ProcessRunner; - readonly serverReply?: string; - readonly listeners?: FakeTcpListenerFactory; - } = {}, -): Promise { - const log: string[] = []; - const runner = options.runner ?? new PhoneAdb(log); - const probe = new LoggingTcpProbe(runner.log, [5038]); - probe.replyWith(options.serverReply ?? VERSION_REPLY_41); - const filesystem = options.filesystem ?? (await androidFilesystem()); - await filesystem.mkdirp(simlockHome); - await filesystem.writeFileAtomic( - `${simlockHome}/adb-server.json`, - JSON.stringify({ pid: 4242, port: 5038, startedAt: 1 }), - ); - const driver = await AndroidDriver.create({ - clock: new FakeClock(), - database: await memoryAndroidSourceDatabase(), - driverConfig: {}, - env: { ANDROID_HOME: sdk, ...options.env }, - filesystem, - homeDirectory: home, - hostAbi: "arm64-v8a", - idGenerator: { generate: () => "one" }, - instanceId, - processRunner: options.processRunner?.(runner) ?? runner, - processSupervisor: new FakeProcessSupervisor([4242]), - simlockHome, - ...(options.listeners === undefined ? {} : { tcpListener: options.listeners }), - tcpProbe: probe, - }); - return { driver, filesystem, log: runner.log, probe, runner }; -} - -async function androidFilesystem(): Promise { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(`${sdk}/platform-tools`); - await filesystem.writeFileAtomic(adb, "binary"); - for (const binary of [ - `${sdk}/cmdline-tools/latest/bin/avdmanager`, - `${sdk}/cmdline-tools/latest/bin/sdkmanager`, - `${sdk}/emulator/emulator`, - ]) { - await filesystem.mkdirp(binary.slice(0, binary.lastIndexOf("/"))); - await filesystem.writeFileAtomic(binary, "binary"); - } - await filesystem.mkdirp(avdDirectory); - await filesystem.writeFileAtomic( - `${avdDirectory}/${OWNED_ROOT_MARKER_FILE}`, - JSON.stringify({ instanceId, owner: "simlock", platform: "android", schemaVersion: 1 }), - ); - await filesystem.mkdirp(`${sdk}/system-images/android-34/google_apis/arm64-v8a`); - return filesystem; -} - -function restOf(call: Call): string[] { - return call.args[0] === "-P" ? call.args.slice(2) : [...call.args]; -} - function econnrefused(): Error { return Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:5037"), { code: "ECONNREFUSED" }); } -function devicesListing(...lines: string[]): string { - return `List of devices attached\n${lines.join("\n")}\n\n`; -} - -function isDevicesListing(rest: readonly string[]): boolean { - return rest.join(" ") === "devices -l"; -} - -function isGetprop(rest: readonly string[]): boolean { - return rest.includes("getprop"); -} - -function isUserRead(rest: readonly string[]): boolean { - return rest.includes("get-current-user"); -} - -function isPackageList(rest: readonly string[]): boolean { - return rest.includes("list") && rest.includes("packages"); -} - function isUninstall(rest: readonly string[]): boolean { return rest.includes("uninstall"); } @@ -373,10 +91,6 @@ function isRuleCommand(rest: readonly string[]): boolean { return rest[2] === "reverse" || rest[2] === "forward"; } -function defaultServerSends(probe: FakeTcpProbe) { - return probe.sends.filter((send) => send.port === 5037); -} - function hostVersionSends(probe: FakeTcpProbe) { return defaultServerSends(probe).filter((send) => send.payload === HOST_VERSION); } @@ -387,35 +101,6 @@ async function advisoriesOf( return driver.advisories(); } -const NO_DEVICE = Symbol("no device"); - -interface PassthroughResult { - readonly command: string; - readonly args: readonly string[]; - readonly env: Readonly>; - readonly unsetEnv?: readonly string[]; -} - -function passthroughOf( - driver: AndroidDriver, - args: readonly string[], - device: unknown = PHYSICAL, - context: { readonly hasTerminal: boolean } = { hasTerminal: false }, -): Promise { - // why: an explicit `undefined` would take the default, so "no device" is a named value. - const target = device === NO_DEVICE ? undefined : device; - return ( - driver as unknown as { - passthrough( - tool: string, - device: unknown, - args: readonly string[], - context: { readonly hasTerminal: boolean }, - ): Promise; - } - ).passthrough("adb", target, args, context); -} - /** why: a refusal test starts from the same input accepted, so it cannot pass for another reason. */ async function expectReclaimAccepted(): Promise { const accepted = await world(); @@ -424,16 +109,14 @@ async function expectReclaimAccepted(): Promise { }); } -async function expectPassthroughAccepted(driver: AndroidDriver): Promise { - const accepted = await passthroughOf(driver, ["shell", "getprop"]); - expect(accepted.args).toEqual(["-P", "5037", "-s", PHONE, "shell", "getprop"]); +const pixelDevices = `Available devices:\nid: 0 or "pixel_8"\n Name: Pixel 8\n OEM : Google\n`; + +function sdkTool(name: "avdmanager" | "sdkmanager"): string { + return `${sdk}/cmdline-tools/latest/bin/${name}`; } -async function rejectionOf(promise: Promise): Promise { - return promise.then( - () => undefined, - (error: unknown) => error ?? new Error("rejected with nothing"), - ); +function scripted(command: string, args: readonly (string | RegExp)[], stdout = "") { + return { match: { args, command }, result: { code: 0, stderr: "", stdout } }; } describe("Android physical handler: the protocol gate", () => { @@ -872,321 +555,6 @@ describe("Android physical handler: scope", () => { }); }); -describe("Android physical handler: inspectPhysical", () => { - it("returns the serial, ro.product.model, the API level, class phone and the packages pm list packages -3 --user lists for the id am get-current-user prints", async () => { - const { driver, runner } = await world(); - runner.userOutput = "10\n"; - runner.installed.splice(0, runner.installed.length, "com.example.kept", "com.example.stray"); - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toEqual({ - apps: ["com.example.kept", "com.example.stray"], - class: "phone", - driverDeviceId: PHONE, - enrollable: true, - model: "SM-G991B", - osVersion: "34", - }); - expect(runner.rests).toContainEqual([ - "-s", - PHONE, - "shell", - "pm", - "list", - "packages", - "-3", - "--user", - "10", - ]); - expect(runner.rests).toContainEqual(["-s", PHONE, "shell", "getprop", "ro.product.model"]); - }); - - it("refuses a serial in state unauthorized as untrusted, and runs no command naming it", async () => { - const { driver, runner } = await world(); - expect(await driver.inspectPhysical(PHONE)).toMatchObject({ enrollable: true }); - runner.calls.length = 0; - runner.devicesOutput = devicesListing(`${PHONE} unauthorized usb:1-1 transport_id:3`); - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toMatchObject({ enrollable: false, reason: "untrusted" }); - expect(runner.commands.filter((call) => call.args.includes("-s"))).toEqual([]); - }); - - it.each([ - [ - "a network serial", - "192.168.1.5:5555 device product:o1s model:SM_G991B transport_id:4", - "192.168.1.5:5555", - ], - [ - "an mDNS name", - "adb-R58M123ABC-xyz._adb-tls-connect._tcp device product:o1s model:SM_G991B transport_id:4", - "adb-R58M123ABC-xyz._adb-tls-connect._tcp", - ], - [ - "a listed emulator", - "emulator-5554 device product:sdk_gphone model:sdk transport_id:1", - "emulator-5554", - ], - [ - "a USB-shaped serial on a network line", - `${PHONE} device product:o1s model:SM_G991B transport_id:3`, - PHONE, - ], - ])( - "refuses %s as network-only with the USB-only message, and runs no command naming it", - async (_name, line, id) => { - const { driver, runner } = await world(); - runner.devicesOutput = devicesListing(line); - - const inspection = await driver.inspectPhysical(id); - - expect(inspection).toMatchObject({ enrollable: false, reason: "network-only" }); - expect((inspection as { message: string }).message).toContain( - "only USB-connected devices can be enrolled", - ); - expect(runner.commands.filter((call) => call.args.includes("-s"))).toEqual([]); - }, - ); - - it("inspects a host:port-shaped serial on a USB transport: the transport decides, not the shape", async () => { - const { driver, runner } = await world(); - runner.devicesOutput = devicesListing( - "192.168.1.5:5555 device usb:1-1 product:o1s model:SM_G991B transport_id:5", - ); - - const inspection = await driver.inspectPhysical("192.168.1.5:5555"); - - expect(inspection).toMatchObject({ driverDeviceId: "192.168.1.5:5555", enrollable: true }); - }); - - it.each([ - ["not listed", devicesListing()], - ["listed offline", devicesListing(`${PHONE} offline usb:1-1 transport_id:3`)], - ])("refuses a serial that is %s as not-connected", async (_name, listing) => { - const { driver, runner } = await world(); - expect(await driver.inspectPhysical(PHONE)).toMatchObject({ enrollable: true }); - runner.devicesOutput = listing; - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toMatchObject({ enrollable: false, reason: "not-connected" }); - }); - - it.each([ - [ - "adb devices -l fails", - (rest: readonly string[]) => (isDevicesListing(rest) ? FAILED : undefined), - /could not read the device list \(adb devices -l\)/, - ], - [ - "adb devices -l passes 30 s", - (rest: readonly string[]) => (isDevicesListing(rest) ? TIMED_OUT : undefined), - /could not read the device list \(adb devices -l\)/, - ], - [ - "getprop fails", - (rest: readonly string[]) => (isGetprop(rest) ? FAILED : undefined), - /could not read the model/, - ], - [ - "the user id read fails", - (rest: readonly string[]) => (isUserRead(rest) ? FAILED : undefined), - /could not read the current user id/, - ], - [ - "the package list fails", - (rest: readonly string[]) => (isPackageList(rest) ? FAILED : undefined), - /could not read the user apps/, - ], - ])( - "is refused unreadable naming the read, and does not reject, when %s", - async (_name, override, named) => { - const { driver, runner } = await world(); - runner.override(override); - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); - expect((inspection as { message: string }).message).toMatch(named); - }, - ); - - it("is refused unreadable, and does not reject, when the process runner cannot spawn adb", async () => { - const { driver, runner } = await world(); - runner.override((rest) => { - if (isDevicesListing(rest)) throw new ProcessSpawnError(adb, rest); - return undefined; - }); - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); - }); - - it("lists nothing present, and does not reject, when the process runner cannot spawn adb", async () => { - const { driver, runner } = await world(); - runner.override((rest) => { - if (isDevicesListing(rest)) throw new ProcessSpawnError(adb, rest); - return undefined; - }); - - const present = await driver.listPresentPhysical([PHONE]); - - expect(present).toEqual([{ driverDeviceId: PHONE, present: false, reason: "not-present" }]); - }); - - it.each([ - ["a shell injection", "0; rm -r /sdcard\n"], - ["nothing", ""], - ["seven digits", "1234567\n"], - ])( - "is refused unreadable and runs no pm list packages when am get-current-user prints %s", - async (_name, printed) => { - const { driver, runner } = await world(); - runner.userOutput = printed; - - const inspection = await driver.inspectPhysical(PHONE); - - expect(inspection).toMatchObject({ enrollable: false, reason: "unreadable" }); - expect(runner.rests.some(isPackageList)).toBe(false); - }, - ); -}); - -describe("Android physical handler: listPresentPhysical", () => { - it("answers a USB serial in state device present with its API level, an unauthorized one not present untrusted, and an offline, network-only or missing one not present not-present", async () => { - const { driver, runner } = await world(); - runner.devicesOutput = devicesListing( - USB_LINE, - "UNAUTH1 unauthorized usb:1-2 transport_id:4", - "OFFLINE1 offline usb:1-3 transport_id:5", - "192.168.1.5:5555 device product:o1s transport_id:6", - ); - - const present = await driver.listPresentPhysical([ - PHONE, - "UNAUTH1", - "OFFLINE1", - "192.168.1.5:5555", - "MISSING1", - ]); - - expect(present).toEqual([ - { driverDeviceId: PHONE, osVersion: "34", present: true }, - { driverDeviceId: "UNAUTH1", present: false, reason: "untrusted" }, - { driverDeviceId: "OFFLINE1", present: false, reason: "not-present" }, - { driverDeviceId: "192.168.1.5:5555", present: false, reason: "not-present" }, - { driverDeviceId: "MISSING1", present: false, reason: "not-present" }, - ]); - }); - - it("gives no entry, and runs no command, for a USB phone that is not among the given serials", async () => { - const { driver, runner } = await world(); - runner.devicesOutput = devicesListing(USB_LINE, "OTHER99 device usb:1-2 transport_id:4"); - - const present = await driver.listPresentPhysical([PHONE]); - - expect(present).toEqual([{ driverDeviceId: PHONE, osVersion: "34", present: true }]); - expect(runner.rests.flat()).not.toContain("OTHER99"); - }); - - it.each([ - ["fails", FAILED], - ["has no exit code", TIMED_OUT], - ])( - "answers every given serial not present not-present, and does not reject, when adb devices -l %s", - async (_name, result) => { - const { driver, runner } = await world(); - runner.override((rest) => (isDevicesListing(rest) ? result : undefined)); - - const present = await driver.listPresentPhysical([PHONE, "SERIAL2"]); - - expect(present).toEqual([ - { driverDeviceId: PHONE, present: false, reason: "not-present" }, - { driverDeviceId: "SERIAL2", present: false, reason: "not-present" }, - ]); - }, - ); - - it("answers a present serial whose getprop fails not present not-present", async () => { - const { driver, runner } = await world(); - runner.override((rest) => (isGetprop(rest) ? FAILED : undefined)); - - const present = await driver.listPresentPhysical([PHONE]); - - expect(present).toEqual([{ driverDeviceId: PHONE, present: false, reason: "not-present" }]); - }); -}); - -describe("Android physical handler: listAttachedPhysical", () => { - // why: the Windows rule needs `platform: "win32"`, which the driver surface cannot fake per host; - // the handler is built straight, as the Windows describe below does. - async function handlerFor( - listing: string, - options: { readonly platform?: NodeJS.Platform; readonly reply?: string } = {}, - ) { - const runner = new PhoneAdb([]); - runner.devicesOutput = listing; - const probe = new FakeTcpProbe([]); - probe.replyWith(options.reply ?? VERSION_REPLY_41); - const handler = new AndroidPhysicalHandler({ - adb, - baseEnv: { ANDROID_HOME: sdk }, - filesystem: await androidFilesystem(), - isBareShell: () => false, - platform: options.platform ?? "linux", - processRunner: runner, - tcpProbe: probe, - }); - return { handler, runner }; - } - - it("lists the phones on USB in any state, none on the network or an emulator, throws when the list cannot be read, and runs no adb command at a protocol mismatch", async () => { - const listings = [ - { - listing: devicesListing( - USB_LINE, - "UNAUTH1 unauthorized usb:1-2 transport_id:4", - "OFFLINE1 offline usb:1-3 transport_id:5", - "192.168.1.5:5555 device product:o1s transport_id:6", - "emulator-5554 device product:sdk transport_id:7", - ), - listed: [PHONE, "UNAUTH1", "OFFLINE1"], - name: "linux", - platform: "linux", - }, - { - listing: devicesListing( - `${PHONE} device product:a70qxtc transport_id:1`, - "192.168.1.5:5555 device product:o1s transport_id:6", - ), - listed: [PHONE], - name: "windows, with no usb: path and the header line", - platform: "win32", - }, - ] as const; - for (const { listing, listed, name, platform } of listings) { - const { handler, runner } = await handlerFor(listing, { platform }); - - await expect(handler.listAttachedPhysical(), name).resolves.toEqual([...listed].sort()); - expect(runner.rests, name).toEqual([["devices", "-l"]]); - } - - const failing = await handlerFor(devicesListing(USB_LINE)); - failing.runner.override((rest) => (isDevicesListing(rest) ? FAILED : undefined)); - await expect(failing.handler.listAttachedPhysical()).rejects.toThrow( - /could not read the device list \(adb devices -l\)/, - ); - - const mismatch = await handlerFor(devicesListing(USB_LINE), { reply: VERSION_REPLY_40 }); - await expect(mismatch.handler.listAttachedPhysical()).rejects.toThrow(/speaks protocol 40/); - expect(mismatch.runner.commands).toEqual([]); - }); -}); - describe("Android physical handler: reclaim", () => { it("uninstalls, for the current user, each package off the enrollment list, uninstalls no listed package, reports listed packages not installed as missingApps, and returns strategy uninstall", async () => { const { driver, runner } = await world(); @@ -1491,300 +859,3 @@ describe("Android physical handler: independence from emulator calls", () => { expect(command.args).toEqual(["-P", "5037", "-s", PHONE, "shell", "getprop"]); }); }); - -describe("Android physical handler: passthrough", () => { - it("resolves passthrough of adb shell getprop on a physical device to adb -P 5037 -s shell getprop with ANDROID_ADB_SERVER_PORT=5037", async () => { - const { driver } = await world(); - - const command = await passthroughOf(driver, ["shell", "getprop"]); - - expect(command).toMatchObject({ - args: ["-P", "5037", "-s", PHONE, "shell", "getprop"], - command: adb, - env: { ANDROID_ADB_SERVER_PORT: "5037" }, - }); - }); - - it.each(REFUSED_LEADING_FLAGS)( - "refuses the caller argument %s before the subcommand", - async (flag) => { - const { driver } = await world(); - await expectPassthroughAccepted(driver); - - const failure = await rejectionOf(passthroughOf(driver, [flag, "x", "shell", "getprop"])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - }, - ); - - it.each(REFUSED_SUBCOMMANDS)("refuses %s as the subcommand", async (subcommand) => { - const { driver } = await world(); - await expectPassthroughAccepted(driver); - - const failure = await rejectionOf(passthroughOf(driver, [subcommand])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - }); - - it.each(REFUSED_REBOOT_TARGETS)("refuses reboot followed by %s", async (target) => { - const { driver } = await world(); - expect(await passthroughOf(driver, ["reboot"])).toBeDefined(); - - const failure = await rejectionOf(passthroughOf(driver, ["reboot", target])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - }); - - it("refuses a bare shell with no terminal", async () => { - const { driver } = await world(); - await expectPassthroughAccepted(driver); - - const failure = await rejectionOf(passthroughOf(driver, ["shell"])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - }); - - it.each([ - ["shell dumpsys usb", ["shell", "dumpsys", "usb"]], - ["shell echo connect", ["shell", "echo", "connect"]], - ["reboot", ["reboot"]], - ])( - "runs %s: a refused word after the subcommand is an operand, and a plain reboot is not refused", - async (_name, args) => { - const { driver } = await world(); - - const command = await passthroughOf(driver, args); - - expect(command.args).toEqual(["-P", "5037", "-s", PHONE, ...args]); - }, - ); - - it("is refused with PassthroughRefusedError naming both versions when its check sees another version, and the process runner starts nothing", async () => { - const { driver, runner } = await world({ serverReply: VERSION_REPLY_40 }); - - const failure = await rejectionOf(passthroughOf(driver, ["shell", "getprop"])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - expect((failure as Error).message).toMatch(/40.*41|41.*40/s); - expect(runner.commands).toEqual([]); - }); - - it("is refused with PassthroughRefusedError naming which version it cannot read, and the process runner starts nothing", async () => { - const { driver, runner } = await world({ serverReply: "" }); - - const failure = await rejectionOf(passthroughOf(driver, ["shell", "getprop"])); - - expect(failure).toBeInstanceOf(PassthroughRefusedError); - expect((failure as Error).message).toMatch( - /the adb server's version on port 5037 is unreadable/, - ); - expect(runner.commands).toEqual([]); - }); - - it("returns unsetEnv ANDROID_ADB_SERVER_ADDRESS, ADB_SERVER_SOCKET and ANDROID_SERIAL for a physical device, and none with no device", async () => { - const { driver } = await world(); - - const physical = await passthroughOf(driver, ["shell", "getprop"]); - const none = await passthroughOf(driver, ["shell", "getprop"], NO_DEVICE); - - expect([...(physical.unsetEnv ?? [])].sort()).toEqual([ - "ADB_SERVER_SOCKET", - "ANDROID_ADB_SERVER_ADDRESS", - "ANDROID_SERIAL", - ]); - expect(none.unsetEnv ?? []).toEqual([]); - }); - - it("with no device, or with an emulator device, still runs on Simlock's own server and sends no host:version", async () => { - const { driver, probe } = await world(); - - const without = await passthroughOf(driver, ["shell", "getprop"], NO_DEVICE); - const emulator = await passthroughOf(driver, ["shell", "getprop"], EMULATOR_DEVICE); - - expect(without).toEqual({ - args: ["-P", "5038", "shell", "getprop"], - command: adb, - env: { ANDROID_ADB_SERVER_PORT: "5038" }, - }); - expect(emulator).toEqual(without); - expect(defaultServerSends(probe)).toEqual([]); - }); -}); - -describe("Android physical handler: the screen target", () => { - // why: the target a phone's scrcpy session runs on, built straight from the handler: the capture - // reaches it only through the driver, and what it must never see is the port or the check. - async function screenWorld(options: { readonly reply?: string } = {}): Promise<{ - readonly runner: PhoneAdb; - readonly target: ScreenTarget; - readonly tcp: FakeTcpStreamConnector; - }> { - const runner = new PhoneAdb([]); - const probe = new FakeTcpProbe([]); - probe.replyWith(options.reply ?? VERSION_REPLY_41); - const handler = new AndroidPhysicalHandler({ - adb, - // why: an env that aims adb elsewhere on purpose — a target that leaked these to a child - // would reach a server other than the one it just checked. - baseEnv: { - ADB_SERVER_SOCKET: "tcp:10.0.0.5:5037", - ANDROID_ADB_SERVER_ADDRESS: "10.0.0.5", - ANDROID_HOME: sdk, - ANDROID_SERIAL: "SOMEONE-ELSES-PHONE", - }, - filesystem: await androidFilesystem(), - isBareShell: () => false, - processRunner: runner, - tcpProbe: probe, - }); - const tcp = new FakeTcpStreamConnector(); - return { runner, target: handler.screenTargetFor(PHYSICAL, tcp), tcp }; - } - - it("sends nothing and refuses with the check's own message when the check is closed, for run, spawn and connect, at a mismatch and an unreadable server", async () => { - const closed = [ - { message: /speaks protocol 40/, name: "another protocol", reply: VERSION_REPLY_40 }, - { message: /unreadable/, name: "a server it cannot read", reply: "gibberish" }, - ] as const; - const verbs = [ - { - call: (target: ScreenTarget) => target.run(["shell", "dumpsys", "power"], 10_000), - name: "run", - }, - { - call: (target: ScreenTarget) => target.spawn(["shell", "app_process"]), - name: "spawn", - }, - { - call: (target: ScreenTarget) => target.connect(5_000), - name: "connect", - }, - ] as const; - for (const refusal of closed) { - for (const verb of verbs) { - const world = await screenWorld({ reply: refusal.reply }); - - const error = await rejectionOf(verb.call(world.target)); - - expect(error, `${refusal.name} via ${verb.name}`).toBeInstanceOf(ScreenUnavailableError); - expect((error as Error).message, `${refusal.name} via ${verb.name}`).toMatch( - refusal.message, - ); - expect(world.runner.commands, `${refusal.name} via ${verb.name}`).toEqual([]); - expect(world.tcp.connections, `${refusal.name} via ${verb.name}`).toEqual([]); - } - } - }); - - it("runs adb -P 5037 with the shared server's env for run and spawn, and connects to 5037, when the check is open", async () => { - const world = await screenWorld(); - - await world.target.run(["shell", "dumpsys", "power"], 10_000); - await world.target.spawn(["shell", "app_process"]); - await world.target.connect(4_000); - - expect(world.runner.calls.map((call) => [...call.args])).toEqual([ - ["version"], - ["-P", "5037", "-s", PHONE, "shell", "dumpsys", "power"], - ["-P", "5037", "-s", PHONE, "shell", "app_process"], - ]); - for (const call of world.runner.commands) { - expect(call.options.env).toEqual({ - ANDROID_ADB_SERVER_PORT: "5037", - ANDROID_HOME: sdk, - }); - } - expect(world.runner.commands[0]?.options.timeoutMs).toBe(10_000); - expect(world.tcp.connections.map((connection) => connection.port)).toEqual([5037]); - }); -}); - -describe("Android physical handler: estimate", () => { - it("is 90 000 ms for a reclaim of a physical spec at both clean levels, and unchanged for an emulator spec", async () => { - const { driver } = await world(); - - expect(driver.estimate({ clean: "standard", operation: "reclaim" }, PHYSICAL_SPEC)).toBe( - 90_000, - ); - expect(driver.estimate({ clean: "full", operation: "reclaim" }, PHYSICAL_SPEC)).toBe(90_000); - expect(driver.estimate({ clean: "standard", operation: "reclaim" }, VIRTUAL_SPEC)).toBe(6_000); - expect(driver.estimate({ clean: "full", operation: "reclaim" }, VIRTUAL_SPEC)).toBe(32_000); - }); -}); - -const pixelDevices = `Available devices:\nid: 0 or "pixel_8"\n Name: Pixel 8\n OEM : Google\n`; - -function sdkTool(name: "avdmanager" | "sdkmanager"): string { - return `${sdk}/cmdline-tools/latest/bin/${name}`; -} - -function scripted(command: string, args: readonly (string | RegExp)[], stdout = "") { - return { match: { args, command }, result: { code: 0, stderr: "", stdout } }; -} - -describe("Android physical handler on Windows", () => { - // why: adb on Windows lists no `usb:` path, which the driver surface cannot fake per host; - // the handler is built straight with `platform: "win32"`. - async function windowsHandler(listing: string, platform: NodeJS.Platform = "win32") { - const runner = new PhoneAdb([]); - runner.devicesOutput = listing; - const probe = new FakeTcpProbe([]); - probe.replyWith(VERSION_REPLY_41); - return new AndroidPhysicalHandler({ - adb, - baseEnv: { ANDROID_HOME: sdk }, - filesystem: await androidFilesystem(), - isBareShell: () => false, - platform, - processRunner: runner, - tcpProbe: probe, - }); - } - - const WINDOWS_LINE = `${PHONE} device product:a70qxtc model:SM_A705MN device:a70q transport_id:1`; - - it("enrols a phone adb lists with no usb: path, as adb on Windows lists every USB device", async () => { - const handler = await windowsHandler(devicesListing(WINDOWS_LINE)); - - await expect(handler.inspectPhysical(PHONE)).resolves.toMatchObject({ - driverDeviceId: PHONE, - enrollable: true, - }); - }); - - it("still refuses the same listing off Windows, where a USB device always has a usb: path", async () => { - const handler = await windowsHandler(devicesListing(WINDOWS_LINE), "linux"); - - await expect(handler.inspectPhysical(PHONE)).resolves.toMatchObject({ - enrollable: false, - reason: "network-only", - }); - }); - - it.each([ - ["win32", { driverDeviceId: PHONE, osVersion: "34", present: true }], - ["linux", { driverDeviceId: PHONE, present: false, reason: "not-present" }], - ] as const)( - "lists an enrolled phone with no usb: path on %s as the USB rule of that host says", - async (platform, expected) => { - const handler = await windowsHandler(devicesListing(WINDOWS_LINE), platform); - - await expect(handler.listPresentPhysical([PHONE])).resolves.toEqual([expected]); - }, - ); - - it.each([ - ["a host:port", "192.168.1.5:5555"], - ["an mDNS service", "adb-R58M123ABC-x1Yz2W._adb-tls-connect._tcp"], - ["an emulator", "emulator-5554"], - ])("refuses %s serial on Windows as network-only", async (_kind, serial) => { - const handler = await windowsHandler( - devicesListing(`${serial} device product:o1s transport_id:5`), - ); - - await expect(handler.inspectPhysical(serial)).resolves.toMatchObject({ - enrollable: false, - reason: "network-only", - }); - }); -}); diff --git a/src/drivers/android/test-physical-handler.ts b/src/drivers/android/test-physical-handler.ts new file mode 100644 index 000000000..a76462ff1 --- /dev/null +++ b/src/drivers/android/test-physical-handler.ts @@ -0,0 +1,321 @@ +import { OWNED_ROOT_MARKER_FILE } from "../../core/index.js"; +import { + FakeClock, + FakeProcessSupervisor, + FakeTcpListenerFactory, + FakeTcpProbe, + type Filesystem, + MemoryFilesystem, + type ProcessHandle, + type ProcessResult, + type ProcessRunner, + type ProcessRunOptions, + type ProcessStreamOptions, + type StreamingProcessHandle, +} from "../../ports/index.js"; +import { AndroidDriver } from "./index.js"; +import { memoryAndroidSourceDatabase } from "./testing.js"; + +export const sdk = "/android-sdk"; + +export const home = "/home/simlock"; + +export const simlockHome = "/home/simlock/.simlock"; + +export const instanceId = "instance-1"; + +export const adb = `${sdk}/platform-tools/adb`; + +export const avdDirectory = `${simlockHome}/devices/android`; + +export const PHONE = "R58M123ABC"; + +export const VERSION_REPLY_41 = "OKAY00040029"; + +export const VERSION_REPLY_40 = "OKAY00040028"; + +export const ADB_VERSION_41 = + "Android Debug Bridge version 1.0.41\nVersion 35.0.2-12147458\nInstalled as /android-sdk/platform-tools/adb\n"; + +export const USB_LINE = `${PHONE} device usb:1-1 product:o1s model:SM_G991B device:o1s transport_id:3`; + +export const PHYSICAL = { + address: PHONE, + deviceId: PHONE, + driverData: {}, + enrolledApps: ["com.example.kept", "com.example.gone"], + physical: true, +} as const; + +export const PHYSICAL_SPEC = { + model: "SM-G991B", + osVersion: "34", + physical: true, + platform: "android", +} as never; + +export interface Call { + readonly command: string; + readonly args: readonly string[]; + readonly options: ProcessRunOptions; +} + +export type Override = (rest: readonly string[]) => ProcessResult | "hang" | undefined; + +export const OK = (stdout = ""): ProcessResult => ({ code: 0, stderr: "", stdout }); + +export const FAILED: ProcessResult = { code: 1, stderr: "error: closed", stdout: "" }; + +export const TIMED_OUT: ProcessResult = { code: null, stderr: "", stdout: "" }; + +export class PhoneAdb implements ProcessRunner { + readonly calls: Call[] = []; + readonly log: string[]; + readonly installed: string[] = ["com.example.kept", "com.example.stray", "com.example.other"]; + versionOutput = ADB_VERSION_41; + versionCode: number | null = 0; + versionFailure: Error | undefined; + devicesOutput = `List of devices attached\n${USB_LINE}\n\n`; + userOutput = "0\n"; + packagesOutput: string | undefined; + model = "SM-G991B"; + sdkLevel = "34"; + onCall: ((rest: readonly string[]) => void) | undefined; + readonly #overrides: Override[] = []; + + constructor(log: string[] = []) { + this.log = log; + } + + override(answer: Override): void { + this.#overrides.push(answer); + } + + get commands(): Call[] { + return this.calls.filter((call) => call.args[0] !== "version"); + } + + get rests(): string[][] { + return this.commands.map((call) => restOf(call)); + } + + async run( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): Promise { + this.calls.push({ args: [...args], command, options }); + this.log.push(`adb:${args.join(" ")}`); + if (args[0] === "version") { + if (this.versionFailure !== undefined) throw this.versionFailure; + return { code: this.versionCode, stderr: "", stdout: this.versionOutput }; + } + if (args[0] === "devices") { + return OK("List of devices attached\n"); + } + const rest = args[0] === "-P" ? args.slice(2) : args; + this.onCall?.(rest); + for (const answer of this.#overrides) { + const result = answer(rest); + if (result === "hang") return new Promise(() => undefined); + if (result !== undefined) return result; + } + return this.#answer(rest); + } + + #answer(rest: readonly string[]): ProcessResult { + const line = rest.join(" "); + if (line === "devices -l") return OK(this.devicesOutput); + // why: only a command scoped with -s reaches the phone; an unscoped one is unexpected. + const body = rest[0] === "-s" ? rest.slice(2).join(" ") : ""; + const fixed = new Map([ + ["shell am get-current-user", this.userOutput], + ["shell getprop ro.build.version.sdk", `${this.sdkLevel}\n`], + ["shell getprop ro.product.model", `${this.model}\n`], + ]); + const answered = fixed.get(body); + if (answered !== undefined) return OK(answered); + if (body.startsWith("shell pm list packages -3 --user ")) return OK(this.#packageListing()); + const uninstall = /^shell pm uninstall --user \d+ (\S+)$/.exec(body); + if (uninstall === null) return { code: 1, stderr: `fake adb: unexpected ${line}`, stdout: "" }; + this.installed.splice( + 0, + this.installed.length, + ...this.installed.filter((n) => n !== uninstall[1]), + ); + return OK("Success\n"); + } + + #packageListing(): string { + return this.packagesOutput ?? this.installed.map((name) => `package:${name}\n`).join(""); + } + + spawn(command: string, args: readonly string[], options: ProcessRunOptions = {}): ProcessHandle { + const result = this.run(command, args, options); + async function* none(): AsyncIterable { + yield* [] as string[]; + } + return { + kill: () => undefined, + pid: 1, + stderr: none(), + stdout: none(), + unref: () => undefined, + wait: () => result, + }; + } + + spawnStreaming( + command: string, + _args: readonly string[], + _options: ProcessStreamOptions, + ): StreamingProcessHandle { + throw new Error(`fake adb: spawnStreaming ${command} is not used by a driver`); + } +} + +export class LoggingTcpProbe extends FakeTcpProbe { + readonly #log: string[]; + + constructor(log: string[], listening: readonly number[]) { + super(listening); + this.#log = log; + } + + override async send(port: number, payload: string, timeoutMs?: number): Promise { + this.#log.push(`send:${String(port)}:${payload}`); + return super.send(port, payload, timeoutMs); + } +} + +export interface World { + readonly driver: AndroidDriver; + readonly runner: PhoneAdb; + readonly probe: LoggingTcpProbe; + readonly log: string[]; + readonly filesystem: Filesystem; +} + +export async function world( + options: { + readonly env?: Readonly>; + readonly runner?: PhoneAdb; + readonly filesystem?: Filesystem; + readonly processRunner?: (phone: PhoneAdb) => ProcessRunner; + readonly serverReply?: string; + readonly listeners?: FakeTcpListenerFactory; + } = {}, +): Promise { + const log: string[] = []; + const runner = options.runner ?? new PhoneAdb(log); + const probe = new LoggingTcpProbe(runner.log, [5038]); + probe.replyWith(options.serverReply ?? VERSION_REPLY_41); + const filesystem = options.filesystem ?? (await androidFilesystem()); + await filesystem.mkdirp(simlockHome); + await filesystem.writeFileAtomic( + `${simlockHome}/adb-server.json`, + JSON.stringify({ pid: 4242, port: 5038, startedAt: 1 }), + ); + const driver = await AndroidDriver.create({ + clock: new FakeClock(), + database: await memoryAndroidSourceDatabase(), + driverConfig: {}, + env: { ANDROID_HOME: sdk, ...options.env }, + filesystem, + homeDirectory: home, + hostAbi: "arm64-v8a", + idGenerator: { generate: () => "one" }, + instanceId, + processRunner: options.processRunner?.(runner) ?? runner, + processSupervisor: new FakeProcessSupervisor([4242]), + simlockHome, + ...(options.listeners === undefined ? {} : { tcpListener: options.listeners }), + tcpProbe: probe, + }); + return { driver, filesystem, log: runner.log, probe, runner }; +} + +export async function androidFilesystem(): Promise { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(`${sdk}/platform-tools`); + await filesystem.writeFileAtomic(adb, "binary"); + for (const binary of [ + `${sdk}/cmdline-tools/latest/bin/avdmanager`, + `${sdk}/cmdline-tools/latest/bin/sdkmanager`, + `${sdk}/emulator/emulator`, + ]) { + await filesystem.mkdirp(binary.slice(0, binary.lastIndexOf("/"))); + await filesystem.writeFileAtomic(binary, "binary"); + } + await filesystem.mkdirp(avdDirectory); + await filesystem.writeFileAtomic( + `${avdDirectory}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "android", schemaVersion: 1 }), + ); + await filesystem.mkdirp(`${sdk}/system-images/android-34/google_apis/arm64-v8a`); + return filesystem; +} + +export function restOf(call: Call): string[] { + return call.args[0] === "-P" ? call.args.slice(2) : [...call.args]; +} + +export function devicesListing(...lines: string[]): string { + return `List of devices attached\n${lines.join("\n")}\n\n`; +} + +export function isDevicesListing(rest: readonly string[]): boolean { + return rest.join(" ") === "devices -l"; +} + +export function isGetprop(rest: readonly string[]): boolean { + return rest.includes("getprop"); +} + +export function isUserRead(rest: readonly string[]): boolean { + return rest.includes("get-current-user"); +} + +export function isPackageList(rest: readonly string[]): boolean { + return rest.includes("list") && rest.includes("packages"); +} + +export function defaultServerSends(probe: FakeTcpProbe) { + return probe.sends.filter((send) => send.port === 5037); +} + +export const NO_DEVICE = Symbol("no device"); + +export interface PassthroughResult { + readonly command: string; + readonly args: readonly string[]; + readonly env: Readonly>; + readonly unsetEnv?: readonly string[]; +} + +export function passthroughOf( + driver: AndroidDriver, + args: readonly string[], + device: unknown = PHYSICAL, + context: { readonly hasTerminal: boolean } = { hasTerminal: false }, +): Promise { + // why: an explicit `undefined` would take the default, so "no device" is a named value. + const target = device === NO_DEVICE ? undefined : device; + return ( + driver as unknown as { + passthrough( + tool: string, + device: unknown, + args: readonly string[], + context: { readonly hasTerminal: boolean }, + ): Promise; + } + ).passthrough("adb", target, args, context); +} + +export async function rejectionOf(promise: Promise): Promise { + return promise.then( + () => undefined, + (error: unknown) => error ?? new Error("rejected with nothing"), + ); +} From c3050bf31a9fa350d485c4b7eb4a390ad5418db8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:26:01 +0100 Subject: [PATCH 36/83] test(daemon): split dispatcher-leases.test.ts by topic --- src/daemon/dispatcher-leases-capacity.test.ts | 152 +++ .../dispatcher-leases-device-mode.test.ts | 425 ++++++++ ...dispatcher-leases-request-outcomes.test.ts | 409 ++++++++ src/daemon/dispatcher-leases.test.ts | 989 +----------------- 4 files changed, 990 insertions(+), 985 deletions(-) create mode 100644 src/daemon/dispatcher-leases-capacity.test.ts create mode 100644 src/daemon/dispatcher-leases-device-mode.test.ts create mode 100644 src/daemon/dispatcher-leases-request-outcomes.test.ts diff --git a/src/daemon/dispatcher-leases-capacity.test.ts b/src/daemon/dispatcher-leases-capacity.test.ts new file mode 100644 index 000000000..195202fbf --- /dev/null +++ b/src/daemon/dispatcher-leases-capacity.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { type Config } from "../core/index.js"; +import { classifyError } from "./error-code.js"; +import { HOST_SYSTEM, buildDispatcher, flush, gibibyte, session } from "./test-dispatcher.js"; + +describe("Dispatcher: a noWait lease.request refused for capacity names the check that refused it", () => { + const sizes = { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }; + const limits = { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 4, maxRunning: 4 }, + maxRunning: 6, + }; + const ios = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; + const resource = ( + overrides: { limits?: typeof limits; ramBudget?: typeof sizes } = {}, + ): Config["capacity"] => ({ + strategy: "resource", + config: { limits: overrides.limits ?? limits, ramBudget: overrides.ramBudget ?? sizes }, + }); + const holder = () => session({ principal: "tok_holder" }); + const asker = () => session({ principal: "tok_asker" }); + const refusal = (promise: Promise): Promise => + promise.then( + () => { + throw new Error("expected the request to be refused"); + }, + (error: unknown) => error, + ); + + it.each([ + { + name: "the RAM budget", + reason: "ram-budget", + capacity: resource({ ramBudget: { ...sizes, iosBytesPerDevice: 30 * gibibyte } }), + holds: false, + }, + { + name: "the device limit", + reason: "device-limit", + capacity: resource({ limits: { ...limits, ios: { maxDevices: 1, maxRunning: 4 } } }), + holds: true, + }, + { + name: "the platform running limit", + reason: "platform-running-limit", + capacity: resource({ limits: { ...limits, ios: { maxDevices: 4, maxRunning: 1 } } }), + holds: true, + }, + ] as const)("rejects NO_CAPACITY with details.capacityReason $reason at $name", async (row) => { + const { dispatcher } = await buildDispatcher({ capacity: row.capacity }); + if (row.holds) await dispatcher.dispatch("lease.request", ios, holder()); + + await expect( + dispatcher.dispatch("lease.request", { ...ios, noWait: true }, asker()), + ).rejects.toMatchObject({ code: "NO_CAPACITY", details: { capacityReason: row.reason } }); + }); + + it("rejects NO_CAPACITY with no details for a request behind an earlier one", async () => { + const { dispatcher } = await buildDispatcher({ + capacity: resource({ limits: { ...limits, ios: { maxDevices: 1, maxRunning: 4 } } }), + }); + await dispatcher.dispatch("lease.request", ios, holder()); + const waiting = dispatcher.dispatch("lease.request", ios, session({ principal: "tok_head" })); + void waiting.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 0)); + + const error = await refusal( + dispatcher.dispatch("lease.request", { ...ios, noWait: true }, asker()), + ); + + expect(classifyError(error)).toBe("NO_CAPACITY"); + expect((error as { details?: unknown }).details).toBeUndefined(); + }); + + it("answers a replay of the RAM-budget request by its idempotency key with NO_CAPACITY and no details", async () => { + const { dispatcher } = await buildDispatcher({ + capacity: resource({ ramBudget: { ...sizes, iosBytesPerDevice: 30 * gibibyte } }), + }); + const input = { ...ios, idempotencyKey: "key-1", noWait: true } as const; + await expect(dispatcher.dispatch("lease.request", input, asker())).rejects.toMatchObject({ + details: { capacityReason: "ram-budget" }, + }); + + const replay = await refusal(dispatcher.dispatch("lease.request", input, asker())); + + expect(classifyError(replay)).toBe("NO_CAPACITY"); + expect((replay as { details?: unknown }).details).toBeUndefined(); + }); +}); + +describe("Dispatcher: startup-readiness parking", () => { + it("answers status.get with host facts while other operations are parked on startup", async () => { + const { dispatcher } = await buildDispatcher({ + awaitReady: () => new Promise(() => undefined), + hostFacts: () => ({ + ...HOST_SYSTEM, + tools: [{ name: "xcode", platform: "ios", version: "16.4" }], + }), + }); + + let parked = true; + void dispatcher.dispatch("catalog.get", {}, session()).then(() => { + parked = false; + }); + const status = (await dispatcher.dispatch("status.get", {}, session())) as { + readonly host: { readonly os: string; readonly tools: readonly unknown[] }; + }; + await flush(); + + expect(parked).toBe(true); + expect(status.host.os).toBe("macOS"); + expect(status.host.tools).toHaveLength(1); + }); + + it("parks every operation but status.get on awaitReady", async () => { + let releaseReady: () => void = () => {}; + const readyPromise = new Promise((resolve) => { + releaseReady = resolve; + }); + const { dispatcher } = await buildDispatcher({ awaitReady: () => readyPromise }); + + let statusSettled = false; + void dispatcher.dispatch("status.get", {}, session()).then(() => { + statusSettled = true; + }); + await flush(); + expect(statusSettled).toBe(true); + + const parked = { + "catalog.get": dispatcher.dispatch("catalog.get", {}, session()), + "lease.request": dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ), + "list.get": dispatcher.dispatch("list.get", { kind: "devices" }, session({ role: "admin" })), + }; + const settled = new Set(); + for (const [operation, answer] of Object.entries(parked)) { + void answer.then( + () => settled.add(operation), + () => settled.add(operation), + ); + } + await flush(); + expect([...settled]).toEqual([]); + + releaseReady(); + await flush(); + expect([...settled].sort()).toEqual(["catalog.get", "lease.request", "list.get"]); + }); +}); diff --git a/src/daemon/dispatcher-leases-device-mode.test.ts b/src/daemon/dispatcher-leases-device-mode.test.ts new file mode 100644 index 000000000..f5dfcae02 --- /dev/null +++ b/src/daemon/dispatcher-leases-device-mode.test.ts @@ -0,0 +1,425 @@ +import { describe, expect, it } from "vitest"; +import { capacityChangedPayload, FakeDriver } from "../core/testing.js"; +import { type LatestPresence } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import { buildDispatcher, session } from "./test-dispatcher.js"; + +describe("Dispatcher: device mode on every surface", () => { + const request = { model: "iPhone 17 Pro", platform: "ios" } as const; + + it.each([ + ["slim", "a slim boot", "slim"], + ["full", "no mode", undefined], + ] as const)( + "a lease grant's device carries mode: %s when the driver reported %s", + async (expected, _reported, driverMode) => { + const { dispatcher } = await buildDispatcher({ driverOptions: { mode: driverMode } }); + + const grant = await dispatcher.dispatch("lease.request", request, session()); + + expect(grant.device.mode).toBe(expected); + }, + ); + + it("status.get reports the capacity figures the last capacity.changed carries", async () => { + const { dispatcher, engine, eventBus } = await buildDispatcher(); + await engine.convergeRunningCapacity(); + await dispatcher.dispatch("lease.request", request, session()); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + const last = eventBus + .replay() + .filter((event) => event.event === "capacity.changed") + .at(-1); + const capacity = status.capacity as NonNullable; + expect(last?.payload).toEqual(capacityChangedPayload(capacity)); + expect(status.capacity?.ios).toMatchObject({ running: 1, warm: 0, used: 1 }); + }); + + it("lease.request grants a physical device with physical true and its class on the device", async () => { + const { dispatcher, driver, engine } = await buildDispatcher({ + physicalDevices: [ + { + createdAt: 1, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: [], + id: "dev_phys", + spec: { + class: "tablet", + model: "iPad Pro", + osVersion: "18.4", + }, + state: "ready", + }, + ], + }); + await engine.convergeRunningCapacity(); + driver.setPhysicalPresence("00008110-001A2B3C4D5E", true); + + const grant = await dispatcher.dispatch( + "lease.request", + { model: "iPad Pro", physical: true, platform: "ios" }, + session(), + ); + + expect(grant.device).toMatchObject({ class: "tablet", id: "dev_phys", physical: true }); + }); + + it("status.get and list.get show a physical record with physical, its class and no mode or enrolledApps, and count nothing of it in the capacity figures", async () => { + const { dispatcher, engine } = await buildDispatcher({ + physicalDevices: [ + { + createdAt: 1, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: ["com.example.kept"], + id: "dev_phys", + spec: { + class: "tablet", + model: "iPad Pro", + osVersion: "18.4", + }, + state: "ready", + }, + ], + }); + await engine.convergeRunningCapacity(); + const lease = await dispatcher.dispatch("lease.request", request, session()); + + const status = await dispatcher.dispatch("status.get", {}, session()); + const listed = (await dispatcher.dispatch( + "list.get", + { kind: "devices" }, + session({ role: "admin" }), + )) as { id: string; class?: string; mode?: string; physical: boolean }[]; + + for (const devices of [status.devices ?? [], listed]) { + const phys = devices.find((device) => device.id === "dev_phys"); + expect(phys).toMatchObject({ class: "tablet", physical: true, state: "ready" }); + expect(phys).not.toHaveProperty("mode"); + expect(phys).not.toHaveProperty("enrolledApps"); + const simulator = devices.find((device) => device.id === lease.device.id); + expect(simulator).toMatchObject({ mode: "full", physical: false }); + expect(simulator).not.toHaveProperty("class"); + } + expect(status.capacity?.ios).toMatchObject({ running: 1, used: 1, warm: 0 }); + }); + + it("status.get and list.get carry the latest presence read as presence on a physical device; a virtual device, and a physical device not yet read, carry none", async () => { + const physical = (id: string, driverDeviceId: string) => ({ + createdAt: 1, + driverDeviceId, + enrolledApps: [], + id, + physical: true, + spec: { + class: "phone", + model: "iPhone 15 Pro", + osVersion: "18.4", + physical: true, + platform: "ios", + }, + state: "ready", + }); + const reads = new Map([ + ["UDID-READ", { present: false, reason: "locked" }], + ["UDID-PRESENT", { present: true }], + ]); + const { dispatcher, engine } = await buildDispatcher({ + // why: every ID but the unread one answers a read, so a virtual device's own driver ID does too and only the physical guard keeps it off the wire. + presence: (driverDeviceId) => + driverDeviceId === "UDID-UNREAD" + ? undefined + : (reads.get(driverDeviceId) ?? { present: true }), + physicalDevices: [ + physical("dev_read", "UDID-READ"), + physical("dev_present", "UDID-PRESENT"), + physical("dev_unread", "UDID-UNREAD"), + ], + }); + await engine.convergeRunningCapacity(); + const lease = await dispatcher.dispatch("lease.request", request, session()); + + const status = await dispatcher.dispatch("status.get", {}, session()); + const listed = (await dispatcher.dispatch( + "list.get", + { kind: "devices" }, + session({ role: "admin" }), + )) as { id: string; presence?: unknown }[]; + + for (const devices of [status.devices ?? [], listed]) { + const byId = (id: string) => devices.find((device) => device.id === id); + expect(byId("dev_read")).toMatchObject({ presence: { present: false, reason: "locked" } }); + expect(byId("dev_present")).toMatchObject({ presence: { present: true } }); + expect(byId("dev_unread")).not.toHaveProperty("presence"); + expect(byId(lease.device.id)).not.toHaveProperty("presence"); + } + }); + + it("a reclaiming physical device's status entry has no reclaimingSince", async () => { + const { dispatcher } = await buildDispatcher({ + physicalDevices: [ + { + createdAt: 1, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: [], + id: "dev_phys", + lastLeaseEndedAt: 200, + reclaimingSince: 600, + spec: { + class: "phone", + model: "iPhone 15 Pro", + osVersion: "18.4", + }, + state: "reclaiming", + }, + ], + }); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + const entry = status.devices?.find((device) => device.id === "dev_phys"); + expect(entry).toMatchObject({ state: "reclaiming", transitionAgeMs: 400 }); + expect(entry).not.toHaveProperty("reclaimingSince"); + }); + + it("status.get carries the warm pool's figures: the reserve, and a target the pass could not meet with its reason", async () => { + const { dispatcher, engine } = await buildDispatcher({ + warmPool: { + reserveRunning: { android: 0, ios: 1 }, + targets: [{ count: 1, model: "iPhone 17", osVersion: "27.0", platform: "ios" }], + }, + }); + await engine.convergeRunningCapacity(); + await engine.core.passWarmPool(); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + expect(status.warmPool).toStrictEqual({ + enabled: true, + reserveRunning: { android: 0, ios: 1 }, + targets: [ + { + booting: 0, + count: 1, + mode: "full", + model: "iPhone 17", + origin: "configured", + osVersion: "27.0", + platform: "ios", + ready: 0, + short: "runtime-missing", + }, + ], + }); + }); + + it("status.get on a daemon whose pool is off carries enabled false, the reserve and every configured target with ready 0, booting 0 and short disabled", async () => { + const { dispatcher } = await buildDispatcher({ + warmPool: { + enabled: false, + reserveRunning: { android: 1, ios: 0 }, + targets: [ + { count: 2, model: "iPhone 17", osVersion: "26.5", platform: "ios" }, + { count: 1, mode: "slim", model: "Pixel 8", platform: "android" }, + ], + }, + }); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + expect(status.warmPool).toStrictEqual({ + enabled: false, + reserveRunning: { android: 1, ios: 0 }, + targets: [ + { + booting: 0, + count: 2, + mode: "full", + model: "iPhone 17", + origin: "configured", + osVersion: "26.5", + platform: "ios", + ready: 0, + short: "disabled", + }, + { + booting: 0, + count: 1, + mode: "slim", + model: "Pixel 8", + origin: "configured", + platform: "android", + ready: 0, + short: "disabled", + }, + ], + }); + }); + + it("status.get and list.get return mode for every device, including one still provisioning", async () => { + const { dispatcher, registry } = await buildDispatcher({ driverOptions: { mode: "slim" } }); + const granted = await dispatcher.dispatch("lease.request", request, session()); + const provisioning = await registry.registerDevice({ + driverData: { fakeDeviceId: "driver-provisioning" }, + driverDeviceId: "driver-provisioning", + provisionDuration: 0, + spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + }); + + const status = await dispatcher.dispatch("status.get", {}, session()); + const list = await dispatcher.dispatch( + "list.get", + { kind: "devices" }, + session({ role: "admin" }), + ); + + const expected = [ + { id: granted.device.id, mode: "slim" }, + { id: provisioning.id, mode: "full" }, + ]; + expect(status.devices?.map(({ id, mode }) => ({ id, mode }))).toEqual(expected); + expect((list as { id: string; mode: string }[]).map(({ id, mode }) => ({ id, mode }))).toEqual( + expected, + ); + }); + + it.each([ + ["slim", true, false], + ["full", false, true], + ] as const)( + "status.get and list.get report servesDefaultMode for every device: on a worker whose default is %s, a slim pool is %s and a full pool is %s", + async (defaultMode, slimServes, fullServes) => { + const { dispatcher, registry } = await buildDispatcher({ + engine: { defaultModes: { ios: defaultMode } }, + }); + const register = (driverDeviceId: string, slim: boolean) => + registry.registerDevice({ + driverData: { fakeDeviceId: driverDeviceId }, + driverDeviceId, + provisionDuration: 0, + spec: { + physical: false, + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + ...(slim ? { mode: "slim" as const } : {}), + }, + }); + const slim = await register("driver-slim", true); + const full = await register("driver-full", false); + + const status = await dispatcher.dispatch("status.get", {}, session()); + const list = await dispatcher.dispatch( + "list.get", + { kind: "devices" }, + session({ role: "admin" }), + ); + + const expected = [ + { id: slim.id, servesDefaultMode: slimServes }, + { id: full.id, servesDefaultMode: fullServes }, + ]; + const project = (devices: readonly { id: string; servesDefaultMode?: boolean }[]) => + devices.map(({ id, servesDefaultMode }) => ({ id, servesDefaultMode })); + expect(project(status.devices ?? [])).toEqual(expected); + expect(project(list as { id: string; servesDefaultMode?: boolean }[])).toEqual(expected); + }, + ); + + it("status marks a stalled device with stalled: true and leaves others without it", async () => { + const { clock, dispatcher, registry } = await buildDispatcher(); + const leased = await dispatcher.dispatch("lease.request", request, session()); + const register = (driverDeviceId: string) => + registry.registerDevice({ + driverData: { fakeDeviceId: driverDeviceId }, + driverDeviceId, + provisionDuration: 0, + spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + }); + const stuck = await register("driver-stuck"); + clock.advance(60_001); + const fresh = await register("driver-fresh"); + + const status = await dispatcher.dispatch("status.get", {}, session()); + const list = (await dispatcher.dispatch( + "list.get", + { kind: "devices" }, + session({ role: "admin" }), + )) as { id: string; stalled?: boolean }[]; + const { workers } = await dispatcher.dispatch( + "worker.list", + {}, + session({ principal: "operator", role: "admin" }), + ); + + for (const devices of [status.devices, list, workers[0]?.devices ?? []]) { + const byId = new Map(devices?.map((device) => [device.id, device])); + expect(byId.get(stuck.id)?.stalled).toBe(true); + expect(byId.get(fresh.id)).toBeDefined(); + expect(byId.get(fresh.id)).not.toHaveProperty("stalled"); + expect(byId.get(leased.device.id)).toBeDefined(); + expect(byId.get(leased.device.id)).not.toHaveProperty("stalled"); + } + }); + + it("status leaves stalled off a device past its threshold that a live operation holds", async () => { + const claimed = new Set(); + const { clock, dispatcher, registry } = await buildDispatcher({ + claims: { isClaimed: (deviceId) => claimed.has(deviceId) }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "driver-booting" }, + driverDeviceId: "driver-booting", + provisionDuration: 0, + spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + }); + clock.advance(60_001); + const stalled = async () => + (await dispatcher.dispatch("status.get", {}, session())).devices?.find( + (entry) => entry.id === device.id, + )?.stalled; + + claimed.add(device.id); + expect(await stalled()).toBeUndefined(); + claimed.delete(device.id); + expect(await stalled()).toBe(true); + }); + + it("status measures a stall against the driver for the device's own platform", async () => { + const android = new FakeDriver({ + clock: new FakeClock(1_000), + estimateMs: { boot: 300_000, provision: 300_000 }, + platform: "android", + }); + const { clock, dispatcher, registry } = await buildDispatcher({ otherStallDrivers: [android] }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "driver-ios-stuck" }, + driverDeviceId: "driver-ios-stuck", + provisionDuration: 0, + spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + }); + clock.advance(60_001); + + const status = await dispatcher.dispatch("status.get", {}, session()); + + expect(status.devices?.find((entry) => entry.id === device.id)?.stalled).toBe(true); + }); + + it("no lease.request, list.get, or status.get response carries featureProfile or slim", async () => { + const { dispatcher } = await buildDispatcher({ driverOptions: { mode: "slim" } }); + + const responses = [ + await dispatcher.dispatch("lease.request", request, session()), + await dispatcher.dispatch("list.get", { kind: "devices" }, session({ role: "admin" })), + await dispatcher.dispatch("status.get", {}, session()), + ]; + + for (const response of responses) { + const json = JSON.stringify(response); + expect(json).toContain('"mode":"slim"'); + expect(json).not.toContain('"featureProfile"'); + expect(json).not.toContain('"slim":'); + } + }); +}); diff --git a/src/daemon/dispatcher-leases-request-outcomes.test.ts b/src/daemon/dispatcher-leases-request-outcomes.test.ts new file mode 100644 index 000000000..a24d87c7e --- /dev/null +++ b/src/daemon/dispatcher-leases-request-outcomes.test.ts @@ -0,0 +1,409 @@ +import { describe, expect, it, vi } from "vitest"; +import { type Config, BootTimeoutError, DriverCrashError } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import type { DispatcherOptions } from "./dispatcher.js"; +import { classifyError } from "./error-code.js"; +import { buildDispatcher, flush, gibibyte, seedSlimDevice, session } from "./test-dispatcher.js"; + +describe("Dispatcher: lease.request requires", () => { + const IOS = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; + const REQUIRES = { ...IOS, requires: ["agent-device"] } as const; + type SourceState = { state: "available" | "off" | "unavailable" }; + + function sources(initial: SourceState["state"]) { + const current: SourceState = { state: initial }; + return { + current, + list: [{ name: "agent-device", read: async () => ({ state: current.state }) }] as NonNullable< + DispatcherOptions["capabilitySources"] + >, + }; + } + + it("refuses a request on a worker whose agent-device is off with CAPABILITY_MISSING, message `This worker cannot serve agent-device`, details naming it, stores nothing and emits no event", async () => { + const capability = sources("available"); + const { dispatcher, eventBus, registry } = await buildDispatcher({ + capabilitySources: capability.list, + }); + const grant = await dispatcher.dispatch( + "lease.request", + { ...REQUIRES, requesterId: "a" }, + session(), + ); + await dispatcher.dispatch("lease.release", { leaseId: grant.lease.id }, session()); + // why: the release deletes the device in the background; its event must land before the baseline. + await vi.waitFor(() => + expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["deleted"]), + ); + capability.current.state = "off"; + const requestsBefore = registry.leaseRequests().length; + const eventsBefore = eventBus.replay().length; + + await expect( + dispatcher.dispatch("lease.request", { ...REQUIRES, requesterId: "b" }, session()), + ).rejects.toMatchObject({ + code: "CAPABILITY_MISSING", + details: { capabilities: ["agent-device"] }, + message: "This worker cannot serve agent-device", + }); + + expect(registry.leaseRequests()).toHaveLength(requestsBefore); + expect(eventBus.replay()).toHaveLength(eventsBefore); + }); + + it("refuses the same request from the gateway uplink as a probe, the same way, with no lease.declined", async () => { + const capability = sources("unavailable"); + const { dispatcher, eventBus, registry } = await buildDispatcher({ + capabilitySources: capability.list, + }); + + await expect( + dispatcher.dispatch( + "lease.request", + { ...REQUIRES, fleetRequestId: "req_gw1", noWait: true }, + session({ isGatewayUplink: true, principal: "tok_gateway", role: "admin" }), + ), + ).rejects.toMatchObject({ + code: "CAPABILITY_MISSING", + details: { capabilities: ["agent-device"] }, + message: "This worker cannot serve agent-device", + }); + + expect(registry.leaseRequests()).toEqual([]); + expect(eventBus.replay().map((event) => event.event)).not.toContain("lease.declined"); + }); + + it("names only x-other when requires is [x-other, agent-device] and only agent-device is available", async () => { + const { dispatcher } = await buildDispatcher({ + capabilitySources: sources("available").list, + }); + + await expect( + dispatcher.dispatch( + "lease.request", + { ...IOS, requires: ["x-other", "agent-device"] }, + session(), + ), + ).rejects.toMatchObject({ + code: "CAPABILITY_MISSING", + details: { capabilities: ["x-other"] }, + message: "This worker cannot serve x-other", + }); + }); + + it("refuses within 1.5 s with CAPABILITY_MISSING when the capability source takes 2 s", async () => { + const clock = new FakeClock(1_000); + const { dispatcher } = await buildDispatcher({ + capabilitySources: [ + { + name: "agent-device", + read: () => + new Promise((resolve) => clock.setTimer(2_000, () => resolve({ state: "available" }))), + }, + ], + clock, + }); + let settled = false; + const answer = dispatcher.dispatch("lease.request", REQUIRES, session()).then( + () => "granted", + (error: unknown) => error, + ); + void answer.then(() => { + settled = true; + }); + + for (let elapsed = 0; elapsed < 1_500; elapsed += 100) { + await new Promise((resolve) => setImmediate(resolve)); + clock.advance(100); + } + await new Promise((resolve) => setImmediate(resolve)); + + expect(settled).toBe(true); + expect(await answer).toMatchObject({ code: "CAPABILITY_MISSING" }); + }); + + it("grants a queued request with requires once a device frees, even though the capability turned unavailable while it waited", async () => { + const capability = sources("available"); + const { dispatcher, engine } = await buildDispatcher({ capabilitySources: capability.list }); + const first = await dispatcher.dispatch( + "lease.request", + { ...IOS, requesterId: "h1" }, + session(), + ); + await dispatcher.dispatch("lease.request", { ...IOS, requesterId: "h2" }, session()); + const waiting = dispatcher.dispatch( + "lease.request", + { ...REQUIRES, requesterId: "w" }, + session(), + ); + await expect.poll(() => engine.queueDepth).toBe(1); + capability.current.state = "unavailable"; + + await dispatcher.dispatch("lease.release", { leaseId: first.lease.id }, session()); + + await expect(waiting).resolves.toMatchObject({ lease: { requesterId: "w" } }); + }); + + it("refuses requires together with idempotencyKey with BAD_REQUEST and stores nothing", async () => { + const { dispatcher, registry } = await buildDispatcher({ + capabilitySources: sources("available").list, + }); + + await expect( + dispatcher.dispatch("lease.request", { ...REQUIRES, idempotencyKey: "key-1" }, session()), + ).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining("requires cannot be combined with idempotencyKey"), + }); + + expect(registry.leaseRequests()).toEqual([]); + expect(registry.snapshot.leases).toEqual([]); + }); + + it.each([ + ["absent", {}], + ["[]", { requires: [] }], + ])( + "a request with requires %s is granted on a worker whose agent-device is off, and lease.requested's requestSpec has no requires", + async (_label, extra) => { + const { dispatcher, eventBus } = await buildDispatcher({ + capabilitySources: sources("off").list, + }); + + await dispatcher.dispatch("lease.request", { ...IOS, ...extra }, session()); + + const requested = eventBus.replay().find((event) => event.event === "lease.requested"); + expect(requested?.payload).toMatchObject({ requestSpec: IOS }); + expect(JSON.stringify(requested?.payload)).not.toContain("requires"); + }, + ); + + it("a request with requires is granted as today on a worker with the capability, and its requestSpec has no requires", async () => { + const { dispatcher, eventBus } = await buildDispatcher({ + capabilitySources: sources("available").list, + }); + + const grant = await dispatcher.dispatch("lease.request", REQUIRES, session()); + + expect(grant.device.spec).toMatchObject({ model: "iPhone 17 Pro", platform: "ios" }); + const requested = eventBus.replay().find((event) => event.event === "lease.requested"); + expect(requested?.payload).toMatchObject({ requestSpec: IOS }); + expect(JSON.stringify(requested?.payload)).not.toContain("requires"); + }); +}); + +describe("Dispatcher: lease.request at new-devices-paused", () => { + const slimmable = { slimmableOsVersions: ["26.5"] }; + const ask = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; + const slim = { ...ask, mode: "slim" } as const; + const asker = () => session({ principal: "tok_asker" }); + const waiting = (built: Awaited>) => + built.dispatcher.dispatch("list.get", { kind: "requests" }, session({ role: "admin" })); + const fixed: Config["capacity"] = { strategy: "fixed", config: { maxRunning: 4 } }; + + it.each([ + { name: "needs a new device", seed: undefined, input: ask, options: {} }, + { name: "boots a shut-down device", seed: "shutdown", input: slim, options: {} }, + { + name: "needs the reboot of an idle device for its except list", + seed: "ready", + input: { ...slim, except: ["store"] }, + options: {}, + }, + { + name: "is refused for RAM too", + seed: undefined, + input: ask, + options: { + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 4, maxRunning: 4 }, + maxRunning: 6, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: 30 * gibibyte }, + }, + }, + }, + }, + { + name: "runs under the fixed strategy", + seed: undefined, + input: ask, + options: { capacity: fixed }, + }, + ] as const)( + "a request that $name waits, or gets NO_CAPACITY under noWait, with memory-pressure", + async (row) => { + const start = async () => { + const built = await buildDispatcher({ + driverOptions: slimmable, + memoryPressureStep: "new-devices-paused", + ...row.options, + }); + if (row.seed !== undefined) await seedSlimDevice(built, row.seed); + return { built, seededCalls: built.driver.calls.length }; + }; + const driven = (world: Awaited>) => + world.built.driver.calls + .slice(world.seededCalls) + .filter((call) => ["provision", "makeReady", "shutdown"].includes(call.operation)); + + const waiter = await start(); + void waiter.built.dispatcher + .dispatch("lease.request", row.input, asker()) + .catch(() => undefined); + await flush(); + const listed = await waiting(waiter.built); + const refused = await start(); + + await expect( + refused.built.dispatcher.dispatch("lease.request", { ...row.input, noWait: true }, asker()), + ).rejects.toMatchObject({ + code: "NO_CAPACITY", + details: { capacityReason: "memory-pressure" }, + }); + expect(listed).toMatchObject([{ reason: "memory-pressure", stage: "queued" }]); + expect(driven(waiter)).toEqual([]); + expect(driven(refused)).toEqual([]); + }, + ); + + it("grants a request that fits an idle booted device, and does not evict an idle device of another kind", async () => { + const granted = await buildDispatcher({ + driverOptions: slimmable, + memoryPressureStep: "new-devices-paused", + }); + const registered = await seedSlimDevice(granted, "ready"); + await expect( + granted.dispatcher.dispatch("lease.request", slim, asker()), + ).resolves.toMatchObject({ device: { id: registered.id } }); + + const limited = await buildDispatcher({ + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 4, maxRunning: 1 }, + maxRunning: 6, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + driverOptions: slimmable, + memoryPressureStep: "new-devices-paused", + }); + const idle = await seedSlimDevice(limited, "ready"); + await expect( + limited.dispatcher.dispatch("lease.request", { ...ask, mode: "full", noWait: true }, asker()), + ).rejects.toMatchObject({ details: { capacityReason: "memory-pressure" } }); + expect(limited.registry.snapshot.virtualDevices.find((d) => d.id === idle.id)?.state).toBe( + "ready", + ); + }); + + it.each(["none", "warming-paused"] as const)( + "a request waiting on memory pressure reads memory-pressure, then is served when the step lifts to %s with no other event", + async (lifted) => { + const built = await buildDispatcher({ memoryPressureStep: "new-devices-paused" }); + let granted: unknown; + void built.dispatcher.dispatch("lease.request", ask, asker()).then((grant) => { + granted = grant; + }); + await flush(); + const before = await waiting(built); + + built.setMemoryPressure(lifted); + built.eventBus.emit( + "memory.pressure-changed", + { from: "new-devices-paused", to: lifted }, + "test", + ); + + await flush(); + + expect(before).toMatchObject([{ reason: "memory-pressure" }]); + expect(granted).toMatchObject({ lease: { ownerId: "tok_asker" } }); + }, + ); +}); + +describe("Dispatcher: lease.request whose device fails to boot", () => { + it("answers a boot timeout or failed boot of a new or shut-down device as PROVISION_FAILED with the boot's message", async () => { + const ask = { + mode: "slim", + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + } as const; + const timedOut = /^Timed out waiting for device to boot: /; + const failed = /^Device failed to boot: .+: simulator exited$/; + const cases = [ + ["a new device", new BootTimeoutError("sim-1"), timedOut], + ["a new device", new DriverCrashError("simulator exited"), failed], + ["a shut-down device", new BootTimeoutError("sim-1"), timedOut], + ["a shut-down device", new DriverCrashError("simulator exited"), failed], + ] as const; + + for (const [target, error, message] of cases) { + const name = `${target}: ${error.name}`; + const built = await buildDispatcher({ driverOptions: { slimmableOsVersions: ["26.5"] } }); + if (target === "a shut-down device") { + await seedSlimDevice(built, "shutdown"); + // why: call 1 of makeReady is the seed's own boot; the waiter's boot is call 2. + built.driver.failOn("makeReady", 2, error); + } else { + built.driver.failOn("makeReady", 1, error); + } + + const refusal = await built.dispatcher + .dispatch("lease.request", ask, session()) + .catch((caught: unknown) => caught); + + expect( + built.eventBus.replay().find((event) => event.event === "lease.rejected")?.payload, + name, + ).toMatchObject({ reason: "boot-timeout" }); + expect(classifyError(refusal) ?? "INTERNAL", name).toBe("PROVISION_FAILED"); + expect((refusal as Error).message, name).toMatch(message); + } + }); +}); + +describe("Dispatcher: a replay of a failed lease.request", () => { + it("answers a classified failure with its stored code and message, and an unclassified one as INTERNAL with a generic message", async () => { + const ask = { + idempotencyKey: "key-1", + mode: "slim", + model: "iPhone 17 Pro", + osVersion: "26.5", + platform: "ios", + } as const; + async function replayAfter(operation: "makeReady" | "resolveSpec", error: Error) { + const built = await buildDispatcher({ driverOptions: { slimmableOsVersions: ["26.5"] } }); + built.driver.failOn(operation, 1, error); + await built.dispatcher.dispatch("lease.request", ask, session()).catch(() => undefined); + const replay = await built.dispatcher + .dispatch("lease.request", ask, session()) + .catch((caught: unknown) => caught); + return { code: classifyError(replay), message: (replay as Error).message }; + } + + const classified = await replayAfter("makeReady", new BootTimeoutError("sim-1")); + const unclassified = await replayAfter( + "resolveSpec", + new Error("/Users/someone/secret/path exploded"), + ); + + expect([classified, unclassified]).toEqual([ + { + code: "PROVISION_FAILED", + message: expect.stringMatching(/^Timed out waiting for device to boot: /), + }, + { code: "INTERNAL", message: "Internal error" }, + ]); + }); +}); diff --git a/src/daemon/dispatcher-leases.test.ts b/src/daemon/dispatcher-leases.test.ts index 6e2839a2c..1b3fe5616 100644 --- a/src/daemon/dispatcher-leases.test.ts +++ b/src/daemon/dispatcher-leases.test.ts @@ -1,19 +1,9 @@ -import { describe, expect, it, vi } from "vitest"; -import { capacityChangedPayload, FakeDriver } from "../core/testing.js"; -import { type Config, type LatestPresence } from "../core/index.js"; +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; import { FakeClock } from "../ports/index.js"; -import type { DispatcherOptions } from "./dispatcher.js"; -import { classifyError } from "./error-code.js"; -import { BootTimeoutError, DriverCrashError, UnknownLeaseError } from "../core/index.js"; +import { UnknownLeaseError } from "../core/index.js"; import { RequestCancelledError } from "../leasing/index.js"; -import { - HOST_SYSTEM, - buildDispatcher, - flush, - gibibyte, - seedSlimDevice, - session, -} from "./test-dispatcher.js"; +import { buildDispatcher, gibibyte, session } from "./test-dispatcher.js"; describe("Dispatcher: ownership", () => { async function grantLease(dispatcher: Awaited>["dispatcher"]) { @@ -565,974 +555,3 @@ describe("Dispatcher: lease.release-all", () => { ).toEqual(["killed", "killed"]); }); }); - -describe("Dispatcher: device mode on every surface", () => { - const request = { model: "iPhone 17 Pro", platform: "ios" } as const; - - it.each([ - ["slim", "a slim boot", "slim"], - ["full", "no mode", undefined], - ] as const)( - "a lease grant's device carries mode: %s when the driver reported %s", - async (expected, _reported, driverMode) => { - const { dispatcher } = await buildDispatcher({ driverOptions: { mode: driverMode } }); - - const grant = await dispatcher.dispatch("lease.request", request, session()); - - expect(grant.device.mode).toBe(expected); - }, - ); - - it("status.get reports the capacity figures the last capacity.changed carries", async () => { - const { dispatcher, engine, eventBus } = await buildDispatcher(); - await engine.convergeRunningCapacity(); - await dispatcher.dispatch("lease.request", request, session()); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - const last = eventBus - .replay() - .filter((event) => event.event === "capacity.changed") - .at(-1); - const capacity = status.capacity as NonNullable; - expect(last?.payload).toEqual(capacityChangedPayload(capacity)); - expect(status.capacity?.ios).toMatchObject({ running: 1, warm: 0, used: 1 }); - }); - - it("lease.request grants a physical device with physical true and its class on the device", async () => { - const { dispatcher, driver, engine } = await buildDispatcher({ - physicalDevices: [ - { - createdAt: 1, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: [], - id: "dev_phys", - spec: { - class: "tablet", - model: "iPad Pro", - osVersion: "18.4", - }, - state: "ready", - }, - ], - }); - await engine.convergeRunningCapacity(); - driver.setPhysicalPresence("00008110-001A2B3C4D5E", true); - - const grant = await dispatcher.dispatch( - "lease.request", - { model: "iPad Pro", physical: true, platform: "ios" }, - session(), - ); - - expect(grant.device).toMatchObject({ class: "tablet", id: "dev_phys", physical: true }); - }); - - it("status.get and list.get show a physical record with physical, its class and no mode or enrolledApps, and count nothing of it in the capacity figures", async () => { - const { dispatcher, engine } = await buildDispatcher({ - physicalDevices: [ - { - createdAt: 1, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: ["com.example.kept"], - id: "dev_phys", - spec: { - class: "tablet", - model: "iPad Pro", - osVersion: "18.4", - }, - state: "ready", - }, - ], - }); - await engine.convergeRunningCapacity(); - const lease = await dispatcher.dispatch("lease.request", request, session()); - - const status = await dispatcher.dispatch("status.get", {}, session()); - const listed = (await dispatcher.dispatch( - "list.get", - { kind: "devices" }, - session({ role: "admin" }), - )) as { id: string; class?: string; mode?: string; physical: boolean }[]; - - for (const devices of [status.devices ?? [], listed]) { - const phys = devices.find((device) => device.id === "dev_phys"); - expect(phys).toMatchObject({ class: "tablet", physical: true, state: "ready" }); - expect(phys).not.toHaveProperty("mode"); - expect(phys).not.toHaveProperty("enrolledApps"); - const simulator = devices.find((device) => device.id === lease.device.id); - expect(simulator).toMatchObject({ mode: "full", physical: false }); - expect(simulator).not.toHaveProperty("class"); - } - expect(status.capacity?.ios).toMatchObject({ running: 1, used: 1, warm: 0 }); - }); - - it("status.get and list.get carry the latest presence read as presence on a physical device; a virtual device, and a physical device not yet read, carry none", async () => { - const physical = (id: string, driverDeviceId: string) => ({ - createdAt: 1, - driverDeviceId, - enrolledApps: [], - id, - physical: true, - spec: { - class: "phone", - model: "iPhone 15 Pro", - osVersion: "18.4", - physical: true, - platform: "ios", - }, - state: "ready", - }); - const reads = new Map([ - ["UDID-READ", { present: false, reason: "locked" }], - ["UDID-PRESENT", { present: true }], - ]); - const { dispatcher, engine } = await buildDispatcher({ - // why: every ID but the unread one answers a read, so a virtual device's own driver ID does too and only the physical guard keeps it off the wire. - presence: (driverDeviceId) => - driverDeviceId === "UDID-UNREAD" - ? undefined - : (reads.get(driverDeviceId) ?? { present: true }), - physicalDevices: [ - physical("dev_read", "UDID-READ"), - physical("dev_present", "UDID-PRESENT"), - physical("dev_unread", "UDID-UNREAD"), - ], - }); - await engine.convergeRunningCapacity(); - const lease = await dispatcher.dispatch("lease.request", request, session()); - - const status = await dispatcher.dispatch("status.get", {}, session()); - const listed = (await dispatcher.dispatch( - "list.get", - { kind: "devices" }, - session({ role: "admin" }), - )) as { id: string; presence?: unknown }[]; - - for (const devices of [status.devices ?? [], listed]) { - const byId = (id: string) => devices.find((device) => device.id === id); - expect(byId("dev_read")).toMatchObject({ presence: { present: false, reason: "locked" } }); - expect(byId("dev_present")).toMatchObject({ presence: { present: true } }); - expect(byId("dev_unread")).not.toHaveProperty("presence"); - expect(byId(lease.device.id)).not.toHaveProperty("presence"); - } - }); - - it("a reclaiming physical device's status entry has no reclaimingSince", async () => { - const { dispatcher } = await buildDispatcher({ - physicalDevices: [ - { - createdAt: 1, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: [], - id: "dev_phys", - lastLeaseEndedAt: 200, - reclaimingSince: 600, - spec: { - class: "phone", - model: "iPhone 15 Pro", - osVersion: "18.4", - }, - state: "reclaiming", - }, - ], - }); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - const entry = status.devices?.find((device) => device.id === "dev_phys"); - expect(entry).toMatchObject({ state: "reclaiming", transitionAgeMs: 400 }); - expect(entry).not.toHaveProperty("reclaimingSince"); - }); - - it("status.get carries the warm pool's figures: the reserve, and a target the pass could not meet with its reason", async () => { - const { dispatcher, engine } = await buildDispatcher({ - warmPool: { - reserveRunning: { android: 0, ios: 1 }, - targets: [{ count: 1, model: "iPhone 17", osVersion: "27.0", platform: "ios" }], - }, - }); - await engine.convergeRunningCapacity(); - await engine.core.passWarmPool(); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - expect(status.warmPool).toStrictEqual({ - enabled: true, - reserveRunning: { android: 0, ios: 1 }, - targets: [ - { - booting: 0, - count: 1, - mode: "full", - model: "iPhone 17", - origin: "configured", - osVersion: "27.0", - platform: "ios", - ready: 0, - short: "runtime-missing", - }, - ], - }); - }); - - it("status.get on a daemon whose pool is off carries enabled false, the reserve and every configured target with ready 0, booting 0 and short disabled", async () => { - const { dispatcher } = await buildDispatcher({ - warmPool: { - enabled: false, - reserveRunning: { android: 1, ios: 0 }, - targets: [ - { count: 2, model: "iPhone 17", osVersion: "26.5", platform: "ios" }, - { count: 1, mode: "slim", model: "Pixel 8", platform: "android" }, - ], - }, - }); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - expect(status.warmPool).toStrictEqual({ - enabled: false, - reserveRunning: { android: 1, ios: 0 }, - targets: [ - { - booting: 0, - count: 2, - mode: "full", - model: "iPhone 17", - origin: "configured", - osVersion: "26.5", - platform: "ios", - ready: 0, - short: "disabled", - }, - { - booting: 0, - count: 1, - mode: "slim", - model: "Pixel 8", - origin: "configured", - platform: "android", - ready: 0, - short: "disabled", - }, - ], - }); - }); - - it("status.get and list.get return mode for every device, including one still provisioning", async () => { - const { dispatcher, registry } = await buildDispatcher({ driverOptions: { mode: "slim" } }); - const granted = await dispatcher.dispatch("lease.request", request, session()); - const provisioning = await registry.registerDevice({ - driverData: { fakeDeviceId: "driver-provisioning" }, - driverDeviceId: "driver-provisioning", - provisionDuration: 0, - spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - }); - - const status = await dispatcher.dispatch("status.get", {}, session()); - const list = await dispatcher.dispatch( - "list.get", - { kind: "devices" }, - session({ role: "admin" }), - ); - - const expected = [ - { id: granted.device.id, mode: "slim" }, - { id: provisioning.id, mode: "full" }, - ]; - expect(status.devices?.map(({ id, mode }) => ({ id, mode }))).toEqual(expected); - expect((list as { id: string; mode: string }[]).map(({ id, mode }) => ({ id, mode }))).toEqual( - expected, - ); - }); - - it.each([ - ["slim", true, false], - ["full", false, true], - ] as const)( - "status.get and list.get report servesDefaultMode for every device: on a worker whose default is %s, a slim pool is %s and a full pool is %s", - async (defaultMode, slimServes, fullServes) => { - const { dispatcher, registry } = await buildDispatcher({ - engine: { defaultModes: { ios: defaultMode } }, - }); - const register = (driverDeviceId: string, slim: boolean) => - registry.registerDevice({ - driverData: { fakeDeviceId: driverDeviceId }, - driverDeviceId, - provisionDuration: 0, - spec: { - physical: false, - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - ...(slim ? { mode: "slim" as const } : {}), - }, - }); - const slim = await register("driver-slim", true); - const full = await register("driver-full", false); - - const status = await dispatcher.dispatch("status.get", {}, session()); - const list = await dispatcher.dispatch( - "list.get", - { kind: "devices" }, - session({ role: "admin" }), - ); - - const expected = [ - { id: slim.id, servesDefaultMode: slimServes }, - { id: full.id, servesDefaultMode: fullServes }, - ]; - const project = (devices: readonly { id: string; servesDefaultMode?: boolean }[]) => - devices.map(({ id, servesDefaultMode }) => ({ id, servesDefaultMode })); - expect(project(status.devices ?? [])).toEqual(expected); - expect(project(list as { id: string; servesDefaultMode?: boolean }[])).toEqual(expected); - }, - ); - - it("status marks a stalled device with stalled: true and leaves others without it", async () => { - const { clock, dispatcher, registry } = await buildDispatcher(); - const leased = await dispatcher.dispatch("lease.request", request, session()); - const register = (driverDeviceId: string) => - registry.registerDevice({ - driverData: { fakeDeviceId: driverDeviceId }, - driverDeviceId, - provisionDuration: 0, - spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - }); - const stuck = await register("driver-stuck"); - clock.advance(60_001); - const fresh = await register("driver-fresh"); - - const status = await dispatcher.dispatch("status.get", {}, session()); - const list = (await dispatcher.dispatch( - "list.get", - { kind: "devices" }, - session({ role: "admin" }), - )) as { id: string; stalled?: boolean }[]; - const { workers } = await dispatcher.dispatch( - "worker.list", - {}, - session({ principal: "operator", role: "admin" }), - ); - - for (const devices of [status.devices, list, workers[0]?.devices ?? []]) { - const byId = new Map(devices?.map((device) => [device.id, device])); - expect(byId.get(stuck.id)?.stalled).toBe(true); - expect(byId.get(fresh.id)).toBeDefined(); - expect(byId.get(fresh.id)).not.toHaveProperty("stalled"); - expect(byId.get(leased.device.id)).toBeDefined(); - expect(byId.get(leased.device.id)).not.toHaveProperty("stalled"); - } - }); - - it("status leaves stalled off a device past its threshold that a live operation holds", async () => { - const claimed = new Set(); - const { clock, dispatcher, registry } = await buildDispatcher({ - claims: { isClaimed: (deviceId) => claimed.has(deviceId) }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "driver-booting" }, - driverDeviceId: "driver-booting", - provisionDuration: 0, - spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - }); - clock.advance(60_001); - const stalled = async () => - (await dispatcher.dispatch("status.get", {}, session())).devices?.find( - (entry) => entry.id === device.id, - )?.stalled; - - claimed.add(device.id); - expect(await stalled()).toBeUndefined(); - claimed.delete(device.id); - expect(await stalled()).toBe(true); - }); - - it("status measures a stall against the driver for the device's own platform", async () => { - const android = new FakeDriver({ - clock: new FakeClock(1_000), - estimateMs: { boot: 300_000, provision: 300_000 }, - platform: "android", - }); - const { clock, dispatcher, registry } = await buildDispatcher({ otherStallDrivers: [android] }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "driver-ios-stuck" }, - driverDeviceId: "driver-ios-stuck", - provisionDuration: 0, - spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - }); - clock.advance(60_001); - - const status = await dispatcher.dispatch("status.get", {}, session()); - - expect(status.devices?.find((entry) => entry.id === device.id)?.stalled).toBe(true); - }); - - it("no lease.request, list.get, or status.get response carries featureProfile or slim", async () => { - const { dispatcher } = await buildDispatcher({ driverOptions: { mode: "slim" } }); - - const responses = [ - await dispatcher.dispatch("lease.request", request, session()), - await dispatcher.dispatch("list.get", { kind: "devices" }, session({ role: "admin" })), - await dispatcher.dispatch("status.get", {}, session()), - ]; - - for (const response of responses) { - const json = JSON.stringify(response); - expect(json).toContain('"mode":"slim"'); - expect(json).not.toContain('"featureProfile"'); - expect(json).not.toContain('"slim":'); - } - }); -}); - -describe("Dispatcher: a noWait lease.request refused for capacity names the check that refused it", () => { - const sizes = { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }; - const limits = { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 4, maxRunning: 4 }, - maxRunning: 6, - }; - const ios = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; - const resource = ( - overrides: { limits?: typeof limits; ramBudget?: typeof sizes } = {}, - ): Config["capacity"] => ({ - strategy: "resource", - config: { limits: overrides.limits ?? limits, ramBudget: overrides.ramBudget ?? sizes }, - }); - const holder = () => session({ principal: "tok_holder" }); - const asker = () => session({ principal: "tok_asker" }); - const refusal = (promise: Promise): Promise => - promise.then( - () => { - throw new Error("expected the request to be refused"); - }, - (error: unknown) => error, - ); - - it.each([ - { - name: "the RAM budget", - reason: "ram-budget", - capacity: resource({ ramBudget: { ...sizes, iosBytesPerDevice: 30 * gibibyte } }), - holds: false, - }, - { - name: "the device limit", - reason: "device-limit", - capacity: resource({ limits: { ...limits, ios: { maxDevices: 1, maxRunning: 4 } } }), - holds: true, - }, - { - name: "the platform running limit", - reason: "platform-running-limit", - capacity: resource({ limits: { ...limits, ios: { maxDevices: 4, maxRunning: 1 } } }), - holds: true, - }, - ] as const)("rejects NO_CAPACITY with details.capacityReason $reason at $name", async (row) => { - const { dispatcher } = await buildDispatcher({ capacity: row.capacity }); - if (row.holds) await dispatcher.dispatch("lease.request", ios, holder()); - - await expect( - dispatcher.dispatch("lease.request", { ...ios, noWait: true }, asker()), - ).rejects.toMatchObject({ code: "NO_CAPACITY", details: { capacityReason: row.reason } }); - }); - - it("rejects NO_CAPACITY with no details for a request behind an earlier one", async () => { - const { dispatcher } = await buildDispatcher({ - capacity: resource({ limits: { ...limits, ios: { maxDevices: 1, maxRunning: 4 } } }), - }); - await dispatcher.dispatch("lease.request", ios, holder()); - const waiting = dispatcher.dispatch("lease.request", ios, session({ principal: "tok_head" })); - void waiting.catch(() => undefined); - await new Promise((resolve) => setTimeout(resolve, 0)); - - const error = await refusal( - dispatcher.dispatch("lease.request", { ...ios, noWait: true }, asker()), - ); - - expect(classifyError(error)).toBe("NO_CAPACITY"); - expect((error as { details?: unknown }).details).toBeUndefined(); - }); - - it("answers a replay of the RAM-budget request by its idempotency key with NO_CAPACITY and no details", async () => { - const { dispatcher } = await buildDispatcher({ - capacity: resource({ ramBudget: { ...sizes, iosBytesPerDevice: 30 * gibibyte } }), - }); - const input = { ...ios, idempotencyKey: "key-1", noWait: true } as const; - await expect(dispatcher.dispatch("lease.request", input, asker())).rejects.toMatchObject({ - details: { capacityReason: "ram-budget" }, - }); - - const replay = await refusal(dispatcher.dispatch("lease.request", input, asker())); - - expect(classifyError(replay)).toBe("NO_CAPACITY"); - expect((replay as { details?: unknown }).details).toBeUndefined(); - }); -}); - -describe("Dispatcher: startup-readiness parking", () => { - it("answers status.get with host facts while other operations are parked on startup", async () => { - const { dispatcher } = await buildDispatcher({ - awaitReady: () => new Promise(() => undefined), - hostFacts: () => ({ - ...HOST_SYSTEM, - tools: [{ name: "xcode", platform: "ios", version: "16.4" }], - }), - }); - - let parked = true; - void dispatcher.dispatch("catalog.get", {}, session()).then(() => { - parked = false; - }); - const status = (await dispatcher.dispatch("status.get", {}, session())) as { - readonly host: { readonly os: string; readonly tools: readonly unknown[] }; - }; - await flush(); - - expect(parked).toBe(true); - expect(status.host.os).toBe("macOS"); - expect(status.host.tools).toHaveLength(1); - }); - - it("parks every operation but status.get on awaitReady", async () => { - let releaseReady: () => void = () => {}; - const readyPromise = new Promise((resolve) => { - releaseReady = resolve; - }); - const { dispatcher } = await buildDispatcher({ awaitReady: () => readyPromise }); - - let statusSettled = false; - void dispatcher.dispatch("status.get", {}, session()).then(() => { - statusSettled = true; - }); - await flush(); - expect(statusSettled).toBe(true); - - const parked = { - "catalog.get": dispatcher.dispatch("catalog.get", {}, session()), - "lease.request": dispatcher.dispatch( - "lease.request", - { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - session(), - ), - "list.get": dispatcher.dispatch("list.get", { kind: "devices" }, session({ role: "admin" })), - }; - const settled = new Set(); - for (const [operation, answer] of Object.entries(parked)) { - void answer.then( - () => settled.add(operation), - () => settled.add(operation), - ); - } - await flush(); - expect([...settled]).toEqual([]); - - releaseReady(); - await flush(); - expect([...settled].sort()).toEqual(["catalog.get", "lease.request", "list.get"]); - }); -}); - -describe("Dispatcher: lease.request requires", () => { - const IOS = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; - const REQUIRES = { ...IOS, requires: ["agent-device"] } as const; - type SourceState = { state: "available" | "off" | "unavailable" }; - - function sources(initial: SourceState["state"]) { - const current: SourceState = { state: initial }; - return { - current, - list: [{ name: "agent-device", read: async () => ({ state: current.state }) }] as NonNullable< - DispatcherOptions["capabilitySources"] - >, - }; - } - - it("refuses a request on a worker whose agent-device is off with CAPABILITY_MISSING, message `This worker cannot serve agent-device`, details naming it, stores nothing and emits no event", async () => { - const capability = sources("available"); - const { dispatcher, eventBus, registry } = await buildDispatcher({ - capabilitySources: capability.list, - }); - const grant = await dispatcher.dispatch( - "lease.request", - { ...REQUIRES, requesterId: "a" }, - session(), - ); - await dispatcher.dispatch("lease.release", { leaseId: grant.lease.id }, session()); - // why: the release deletes the device in the background; its event must land before the baseline. - await vi.waitFor(() => - expect(registry.snapshot.devices.map((device) => device.state)).toEqual(["deleted"]), - ); - capability.current.state = "off"; - const requestsBefore = registry.leaseRequests().length; - const eventsBefore = eventBus.replay().length; - - await expect( - dispatcher.dispatch("lease.request", { ...REQUIRES, requesterId: "b" }, session()), - ).rejects.toMatchObject({ - code: "CAPABILITY_MISSING", - details: { capabilities: ["agent-device"] }, - message: "This worker cannot serve agent-device", - }); - - expect(registry.leaseRequests()).toHaveLength(requestsBefore); - expect(eventBus.replay()).toHaveLength(eventsBefore); - }); - - it("refuses the same request from the gateway uplink as a probe, the same way, with no lease.declined", async () => { - const capability = sources("unavailable"); - const { dispatcher, eventBus, registry } = await buildDispatcher({ - capabilitySources: capability.list, - }); - - await expect( - dispatcher.dispatch( - "lease.request", - { ...REQUIRES, fleetRequestId: "req_gw1", noWait: true }, - session({ isGatewayUplink: true, principal: "tok_gateway", role: "admin" }), - ), - ).rejects.toMatchObject({ - code: "CAPABILITY_MISSING", - details: { capabilities: ["agent-device"] }, - message: "This worker cannot serve agent-device", - }); - - expect(registry.leaseRequests()).toEqual([]); - expect(eventBus.replay().map((event) => event.event)).not.toContain("lease.declined"); - }); - - it("names only x-other when requires is [x-other, agent-device] and only agent-device is available", async () => { - const { dispatcher } = await buildDispatcher({ - capabilitySources: sources("available").list, - }); - - await expect( - dispatcher.dispatch( - "lease.request", - { ...IOS, requires: ["x-other", "agent-device"] }, - session(), - ), - ).rejects.toMatchObject({ - code: "CAPABILITY_MISSING", - details: { capabilities: ["x-other"] }, - message: "This worker cannot serve x-other", - }); - }); - - it("refuses within 1.5 s with CAPABILITY_MISSING when the capability source takes 2 s", async () => { - const clock = new FakeClock(1_000); - const { dispatcher } = await buildDispatcher({ - capabilitySources: [ - { - name: "agent-device", - read: () => - new Promise((resolve) => clock.setTimer(2_000, () => resolve({ state: "available" }))), - }, - ], - clock, - }); - let settled = false; - const answer = dispatcher.dispatch("lease.request", REQUIRES, session()).then( - () => "granted", - (error: unknown) => error, - ); - void answer.then(() => { - settled = true; - }); - - for (let elapsed = 0; elapsed < 1_500; elapsed += 100) { - await new Promise((resolve) => setImmediate(resolve)); - clock.advance(100); - } - await new Promise((resolve) => setImmediate(resolve)); - - expect(settled).toBe(true); - expect(await answer).toMatchObject({ code: "CAPABILITY_MISSING" }); - }); - - it("grants a queued request with requires once a device frees, even though the capability turned unavailable while it waited", async () => { - const capability = sources("available"); - const { dispatcher, engine } = await buildDispatcher({ capabilitySources: capability.list }); - const first = await dispatcher.dispatch( - "lease.request", - { ...IOS, requesterId: "h1" }, - session(), - ); - await dispatcher.dispatch("lease.request", { ...IOS, requesterId: "h2" }, session()); - const waiting = dispatcher.dispatch( - "lease.request", - { ...REQUIRES, requesterId: "w" }, - session(), - ); - await expect.poll(() => engine.queueDepth).toBe(1); - capability.current.state = "unavailable"; - - await dispatcher.dispatch("lease.release", { leaseId: first.lease.id }, session()); - - await expect(waiting).resolves.toMatchObject({ lease: { requesterId: "w" } }); - }); - - it("refuses requires together with idempotencyKey with BAD_REQUEST and stores nothing", async () => { - const { dispatcher, registry } = await buildDispatcher({ - capabilitySources: sources("available").list, - }); - - await expect( - dispatcher.dispatch("lease.request", { ...REQUIRES, idempotencyKey: "key-1" }, session()), - ).rejects.toMatchObject({ - code: "BAD_REQUEST", - message: expect.stringContaining("requires cannot be combined with idempotencyKey"), - }); - - expect(registry.leaseRequests()).toEqual([]); - expect(registry.snapshot.leases).toEqual([]); - }); - - it.each([ - ["absent", {}], - ["[]", { requires: [] }], - ])( - "a request with requires %s is granted on a worker whose agent-device is off, and lease.requested's requestSpec has no requires", - async (_label, extra) => { - const { dispatcher, eventBus } = await buildDispatcher({ - capabilitySources: sources("off").list, - }); - - await dispatcher.dispatch("lease.request", { ...IOS, ...extra }, session()); - - const requested = eventBus.replay().find((event) => event.event === "lease.requested"); - expect(requested?.payload).toMatchObject({ requestSpec: IOS }); - expect(JSON.stringify(requested?.payload)).not.toContain("requires"); - }, - ); - - it("a request with requires is granted as today on a worker with the capability, and its requestSpec has no requires", async () => { - const { dispatcher, eventBus } = await buildDispatcher({ - capabilitySources: sources("available").list, - }); - - const grant = await dispatcher.dispatch("lease.request", REQUIRES, session()); - - expect(grant.device.spec).toMatchObject({ model: "iPhone 17 Pro", platform: "ios" }); - const requested = eventBus.replay().find((event) => event.event === "lease.requested"); - expect(requested?.payload).toMatchObject({ requestSpec: IOS }); - expect(JSON.stringify(requested?.payload)).not.toContain("requires"); - }); -}); - -describe("Dispatcher: lease.request at new-devices-paused", () => { - const slimmable = { slimmableOsVersions: ["26.5"] }; - const ask = { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" } as const; - const slim = { ...ask, mode: "slim" } as const; - const asker = () => session({ principal: "tok_asker" }); - const waiting = (built: Awaited>) => - built.dispatcher.dispatch("list.get", { kind: "requests" }, session({ role: "admin" })); - const fixed: Config["capacity"] = { strategy: "fixed", config: { maxRunning: 4 } }; - - it.each([ - { name: "needs a new device", seed: undefined, input: ask, options: {} }, - { name: "boots a shut-down device", seed: "shutdown", input: slim, options: {} }, - { - name: "needs the reboot of an idle device for its except list", - seed: "ready", - input: { ...slim, except: ["store"] }, - options: {}, - }, - { - name: "is refused for RAM too", - seed: undefined, - input: ask, - options: { - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 4, maxRunning: 4 }, - maxRunning: 6, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: 30 * gibibyte }, - }, - }, - }, - }, - { - name: "runs under the fixed strategy", - seed: undefined, - input: ask, - options: { capacity: fixed }, - }, - ] as const)( - "a request that $name waits, or gets NO_CAPACITY under noWait, with memory-pressure", - async (row) => { - const start = async () => { - const built = await buildDispatcher({ - driverOptions: slimmable, - memoryPressureStep: "new-devices-paused", - ...row.options, - }); - if (row.seed !== undefined) await seedSlimDevice(built, row.seed); - return { built, seededCalls: built.driver.calls.length }; - }; - const driven = (world: Awaited>) => - world.built.driver.calls - .slice(world.seededCalls) - .filter((call) => ["provision", "makeReady", "shutdown"].includes(call.operation)); - - const waiter = await start(); - void waiter.built.dispatcher - .dispatch("lease.request", row.input, asker()) - .catch(() => undefined); - await flush(); - const listed = await waiting(waiter.built); - const refused = await start(); - - await expect( - refused.built.dispatcher.dispatch("lease.request", { ...row.input, noWait: true }, asker()), - ).rejects.toMatchObject({ - code: "NO_CAPACITY", - details: { capacityReason: "memory-pressure" }, - }); - expect(listed).toMatchObject([{ reason: "memory-pressure", stage: "queued" }]); - expect(driven(waiter)).toEqual([]); - expect(driven(refused)).toEqual([]); - }, - ); - - it("grants a request that fits an idle booted device, and does not evict an idle device of another kind", async () => { - const granted = await buildDispatcher({ - driverOptions: slimmable, - memoryPressureStep: "new-devices-paused", - }); - const registered = await seedSlimDevice(granted, "ready"); - await expect( - granted.dispatcher.dispatch("lease.request", slim, asker()), - ).resolves.toMatchObject({ device: { id: registered.id } }); - - const limited = await buildDispatcher({ - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 4, maxRunning: 1 }, - maxRunning: 6, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - }, - driverOptions: slimmable, - memoryPressureStep: "new-devices-paused", - }); - const idle = await seedSlimDevice(limited, "ready"); - await expect( - limited.dispatcher.dispatch("lease.request", { ...ask, mode: "full", noWait: true }, asker()), - ).rejects.toMatchObject({ details: { capacityReason: "memory-pressure" } }); - expect(limited.registry.snapshot.virtualDevices.find((d) => d.id === idle.id)?.state).toBe( - "ready", - ); - }); - - it.each(["none", "warming-paused"] as const)( - "a request waiting on memory pressure reads memory-pressure, then is served when the step lifts to %s with no other event", - async (lifted) => { - const built = await buildDispatcher({ memoryPressureStep: "new-devices-paused" }); - let granted: unknown; - void built.dispatcher.dispatch("lease.request", ask, asker()).then((grant) => { - granted = grant; - }); - await flush(); - const before = await waiting(built); - - built.setMemoryPressure(lifted); - built.eventBus.emit( - "memory.pressure-changed", - { from: "new-devices-paused", to: lifted }, - "test", - ); - - await flush(); - - expect(before).toMatchObject([{ reason: "memory-pressure" }]); - expect(granted).toMatchObject({ lease: { ownerId: "tok_asker" } }); - }, - ); -}); - -describe("Dispatcher: lease.request whose device fails to boot", () => { - it("answers a boot timeout or failed boot of a new or shut-down device as PROVISION_FAILED with the boot's message", async () => { - const ask = { - mode: "slim", - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - } as const; - const timedOut = /^Timed out waiting for device to boot: /; - const failed = /^Device failed to boot: .+: simulator exited$/; - const cases = [ - ["a new device", new BootTimeoutError("sim-1"), timedOut], - ["a new device", new DriverCrashError("simulator exited"), failed], - ["a shut-down device", new BootTimeoutError("sim-1"), timedOut], - ["a shut-down device", new DriverCrashError("simulator exited"), failed], - ] as const; - - for (const [target, error, message] of cases) { - const name = `${target}: ${error.name}`; - const built = await buildDispatcher({ driverOptions: { slimmableOsVersions: ["26.5"] } }); - if (target === "a shut-down device") { - await seedSlimDevice(built, "shutdown"); - // why: call 1 of makeReady is the seed's own boot; the waiter's boot is call 2. - built.driver.failOn("makeReady", 2, error); - } else { - built.driver.failOn("makeReady", 1, error); - } - - const refusal = await built.dispatcher - .dispatch("lease.request", ask, session()) - .catch((caught: unknown) => caught); - - expect( - built.eventBus.replay().find((event) => event.event === "lease.rejected")?.payload, - name, - ).toMatchObject({ reason: "boot-timeout" }); - expect(classifyError(refusal) ?? "INTERNAL", name).toBe("PROVISION_FAILED"); - expect((refusal as Error).message, name).toMatch(message); - } - }); -}); - -describe("Dispatcher: a replay of a failed lease.request", () => { - it("answers a classified failure with its stored code and message, and an unclassified one as INTERNAL with a generic message", async () => { - const ask = { - idempotencyKey: "key-1", - mode: "slim", - model: "iPhone 17 Pro", - osVersion: "26.5", - platform: "ios", - } as const; - async function replayAfter(operation: "makeReady" | "resolveSpec", error: Error) { - const built = await buildDispatcher({ driverOptions: { slimmableOsVersions: ["26.5"] } }); - built.driver.failOn(operation, 1, error); - await built.dispatcher.dispatch("lease.request", ask, session()).catch(() => undefined); - const replay = await built.dispatcher - .dispatch("lease.request", ask, session()) - .catch((caught: unknown) => caught); - return { code: classifyError(replay), message: (replay as Error).message }; - } - - const classified = await replayAfter("makeReady", new BootTimeoutError("sim-1")); - const unclassified = await replayAfter( - "resolveSpec", - new Error("/Users/someone/secret/path exploded"), - ); - - expect([classified, unclassified]).toEqual([ - { - code: "PROVISION_FAILED", - message: expect.stringMatching(/^Timed out waiting for device to boot: /), - }, - { code: "INTERNAL", message: "Internal error" }, - ]); - }); -}); From 9a0be83c5f26ee32f86d2c381f2656b78f5085a9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:28:20 +0100 Subject: [PATCH 37/83] test(daemon): split relay-gateway.e2e.test.ts by topic --- src/daemon/relay-gateway-bodies.e2e.test.ts | 331 +++++ src/daemon/relay-gateway-cancel.e2e.test.ts | 293 +++++ src/daemon/relay-gateway-capacity.e2e.test.ts | 322 +++++ src/daemon/relay-gateway-files.e2e.test.ts | 447 +++++++ src/daemon/relay-gateway.e2e.test.ts | 1138 +---------------- src/daemon/test-relay-gateway.ts | 76 ++ 6 files changed, 1478 insertions(+), 1129 deletions(-) create mode 100644 src/daemon/relay-gateway-bodies.e2e.test.ts create mode 100644 src/daemon/relay-gateway-cancel.e2e.test.ts create mode 100644 src/daemon/relay-gateway-capacity.e2e.test.ts create mode 100644 src/daemon/relay-gateway-files.e2e.test.ts create mode 100644 src/daemon/test-relay-gateway.ts diff --git a/src/daemon/relay-gateway-bodies.e2e.test.ts b/src/daemon/relay-gateway-bodies.e2e.test.ts new file mode 100644 index 000000000..3b4e2a6fd --- /dev/null +++ b/src/daemon/relay-gateway-bodies.e2e.test.ts @@ -0,0 +1,331 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { GatewayRelay, type GatewayWorkerClient, type WorkerDirectory } from "../gateway/index.js"; +import { linkPair, TYPE } from "../link/testing.js"; +import { connectSimlockClient } from "../simlock-client/client.js"; +import type { DispatchSession } from "./dispatch.js"; +import type { RelayTarget, TwoWayRelayTarget } from "./relay.js"; +import type { StartDaemonOptions } from "./main.js"; +import type { DaemonServer } from "./server.js"; +import { + cleanUp, + gate, + MIB, + KIB, + patternBytes, + patternStream, + settleWithin, + sleep, + startFleet as startFleet_, + startWorker, + type FleetExec, + type WorkerFixture, +} from "./test-relay-support.js"; +import { WORKER_ID, buildCall, spyOn } from "./test-relay-gateway.js"; + +describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { + const daemons: DaemonServer[] = []; + + const directories: string[] = []; + + let behavior: RelayTarget = () => Promise.resolve(new Response("ok")); + + const targetCalls: { + url: string; + method: string; + headers: Headers; + tenant: string; + leaseId: string | undefined; + }[] = []; + + let twoWayBehavior: TwoWayRelayTarget = () => + Promise.resolve(new Response(null, { status: 200 })); + + const twoWayTarget: TwoWayRelayTarget = (request, tenant, leaseId, pipe) => + twoWayBehavior(request, tenant, leaseId, pipe); + + const target: RelayTarget = (request, tenant, leaseId) => { + targetCalls.push({ + headers: request.headers, + leaseId, + method: request.method, + tenant, + url: request.url, + }); + return behavior(request, tenant, leaseId); + }; + + afterEach(async () => { + behavior = () => Promise.resolve(new Response("ok")); + twoWayBehavior = () => Promise.resolve(new Response(null, { status: 200 })); + targetCalls.length = 0; + await cleanUp(daemons, directories); + }); + + async function startPair( + options: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) { + const fixture: WorkerFixture = await startWorker( + directories, + daemons, + { "agent-device": target, echo: target }, + { relayTwoWayTargets: { echo: twoWayTarget } }, + ); + const pair = linkPair(); + const gatewayEnd = pair.gatewayEnd; + fixture.worker.acceptUplink(pair.worker); + const connected = (await connectSimlockClient( + { connection: pair.gateway, principal: "gw:test" }, + true, + )) as GatewayWorkerClient; + const { client, spy } = spyOn(connected); + const directory: WorkerDirectory = { + target: (workerId) => + workerId === WORKER_ID + ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } + : undefined, + }; + const makeRelay = ( + relayOptions: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) => + new GatewayRelay({ + directory, + ...(relayOptions.deadlineMs === undefined ? {} : { deadlineMs: relayOptions.deadlineMs }), + ...(relayOptions.idleMs === undefined ? {} : { idleMs: relayOptions.idleMs }), + }); + const relay = makeRelay(options); + const opened = () => pair.gatewayTap.binaries(TYPE.OPEN).length; + return { fixture, gatewayEnd, makeRelay, opened, pair, relay, spy }; + } + + const startFleet = ( + execCommand?: FleetExec, + extra: { + readonly proxyAuth?: Record; + readonly worker?: Partial; + readonly gateway?: Partial; + } = {}, + ) => + startFleet_( + { daemons, directories, target, twoWayTarget }, + { ...(execCommand === undefined ? {} : { execCommand }), ...extra }, + ); + + const agent = (overrides: Partial = {}): DispatchSession => ({ + manageEventSubscription: () => undefined, + via: "socket", + principal: "fleet-agent-1", + role: "agent", + ...overrides, + }); + + it("while a 20 MiB relay runs, a lease.renew forwarded to the same worker answers", async () => { + const midway = gate(); + let finished = false; + behavior = async (request) => { + const reader = request.body?.getReader(); + let total = 0; + for (;;) { + const step = await reader?.read(); + if (step === undefined || step.done) break; + total += step.value.length; + if (total >= 5 * MIB) midway.open(); + } + finished = true; + return new Response("done"); + }; + const { gateway, relay, workerId } = await startFleet(); + const grant = await gateway.dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + agent(), + ); + + const pending = relay.relay( + workerId, + buildCall({ body: patternStream(20 * MIB, 5), method: "PUT" }), + ); + await midway.promise; + const renewed = await settleWithin( + gateway.dispatch("lease.renew", { leaseId: grant.lease.id }, agent()), + 10_000, + ); + const relayFinishedBeforeRenew = finished; + const response = await pending; + + expect(renewed).toMatchObject({ state: "fulfilled", value: { id: grant.lease.id } }); + expect(relayFinishedBeforeRenew).toBe(false); + expect(await response.text()).toBe("done"); + }, 120_000); + + it("a relay and a device.exec with output running at once on one worker each get only their own pushes", async () => { + const execStarted = gate(); + behavior = () => { + const encoder = new TextEncoder(); + return Promise.resolve( + new Response( + new ReadableStream({ + async pull(controller) { + controller.enqueue(encoder.encode("r1")); + await execStarted.promise; + controller.enqueue(encoder.encode("r2")); + controller.close(); + }, + }), + ), + ); + }; + const { gateway, relay, workerId } = await startFleet({ + args: ["-e", 'process.stdout.write("e1");setTimeout(()=>process.stdout.write("e2"),300);'], + command: process.execPath, + env: {}, + }); + const grant = await gateway.dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + agent(), + ); + const output: string[] = []; + + const exec = gateway.dispatch( + "device.exec", + { args: [], leaseId: grant.lease.id, tool: "adb" }, + agent({ + onOutput: (_stream, chunk) => { + output.push(chunk); + execStarted.open(); + }, + }), + ); + const response = await relay.relay(workerId, buildCall({})); + const body = await response.text(); + await exec; + + expect(body).toBe("r1r2"); + expect(output.join("")).toBe("e1e2"); + }, 30_000); + + it("a target that answers 401 without reading its 20 MiB upload: the caller gets the 401 and its body, and the gateway stops reading the caller's body", async () => { + behavior = async () => { + // why: the 401 comes after the upload has begun, so the gateway is reading the caller's body when the target answers + await sleep(300); + return new Response("nope", { status: 401 }); + }; + const { relay } = await startPair(); + let pulls = 0; + let cancelled = false; + const body = new ReadableStream({ + async pull(controller) { + pulls += 1; + await sleep(5); + controller.enqueue(patternBytes(256 * KIB, 4)); + if (pulls >= 100) controller.close(); + }, + cancel() { + cancelled = true; + }, + }); + + const response = await relay.relay(WORKER_ID, buildCall({ body, method: "PUT" })); + const text = await response.text(); + await sleep(300); + const pullsAfterAnswer = pulls; + await sleep(300); + + expect(response.status).toBe(401); + expect(text).toBe("nope"); + expect(pulls).toBe(pullsAfterAnswer); + expect(pulls).toBeLessThan(100); + expect(cancelled).toBe(true); + }, 30_000); + + it("a response the target streams in parts reaches the gateway caller part by part, before the target's body ends", async () => { + const second = gate(); + behavior = () => { + const encoder = new TextEncoder(); + let step = 0; + return Promise.resolve( + new Response( + new ReadableStream({ + async pull(controller) { + step += 1; + if (step === 1) controller.enqueue(encoder.encode("part-1")); + else { + await second.promise; + controller.enqueue(encoder.encode("part-2")); + controller.close(); + } + }, + }), + ), + ); + }; + const { relay } = await startPair(); + + const response = await relay.relay(WORKER_ID, buildCall({})); + const reader = response.body?.getReader(); + const first = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body")), 3_000); + second.open(); + const rest = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body")), 3_000); + + expect( + first.state === "fulfilled" ? new TextDecoder().decode(first.value.value) : first.state, + ).toBe("part-1"); + expect( + rest.state === "fulfilled" ? new TextDecoder().decode(rest.value.value) : rest.state, + ).toBe("part-2"); + }, 30_000); + + it.each([ + { method: "GET", status: 204 }, + { method: "GET", status: 205 }, + { method: "GET", status: 304 }, + { method: "HEAD", status: 200 }, + ])( + "a $status answer to a $method reaches the caller with a null body", + async ({ method, status }) => { + behavior = () => Promise.resolve(new Response(method === "HEAD" ? "x" : null, { status })); + const { relay } = await startPair(); + + const response = await relay.relay(WORKER_ID, buildCall({ method })); + + expect(response.status).toBe(status); + expect(response.body).toBeNull(); + }, + ); + + it("a caller that reads nothing slows only its own transfer: another relay to the same worker completes while the first's response is unread, and the target has not sent all 80 MiB", async () => { + let sent = 0; + behavior = (request) => { + if (new URL(request.url).searchParams.get("slow") === null) { + return Promise.resolve(new Response("quick")); + } + return Promise.resolve( + new Response( + new ReadableStream({ + async pull(controller) { + await sleep(5); + controller.enqueue(patternBytes(MIB, 6, sent)); + sent += MIB; + if (sent >= 80 * MIB) controller.close(); + }, + }), + ), + ); + }; + const { relay } = await startPair(); + + const slow = await relay.relay( + WORKER_ID, + buildCall({ url: "http://gateway.test:4321/agent-device/rpc?slow=1" }), + ); + await sleep(500); + const quick = await settleWithin( + relay.relay(WORKER_ID, buildCall({})).then((response) => response.text()), + 5_000, + ); + + expect(quick).toEqual({ state: "fulfilled", value: "quick" }); + expect(sent).toBeLessThan(80 * MIB); + expect(slow.status).toBe(200); + }, 60_000); +}); diff --git a/src/daemon/relay-gateway-cancel.e2e.test.ts b/src/daemon/relay-gateway-cancel.e2e.test.ts new file mode 100644 index 000000000..344dfd6dc --- /dev/null +++ b/src/daemon/relay-gateway-cancel.e2e.test.ts @@ -0,0 +1,293 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + GatewayRelay, + type GatewayWorkerClient, + type TwoWayRelay, + type WorkerDirectory, +} from "../gateway/index.js"; +import { linkPair, TYPE } from "../link/testing.js"; +import { connectSimlockClient } from "../simlock-client/client.js"; +import type { RelayTarget, TwoWayRelayTarget } from "./relay.js"; +import type { DaemonServer } from "./server.js"; +import { + aborted, + cleanUp, + patternBytes, + readParts, + settleWithin, + sleep, + startWorker, + type WorkerFixture, +} from "./test-relay-support.js"; +import { WORKER_ID, buildCall, spyOn } from "./test-relay-gateway.js"; + +const poll = (read: () => T | Promise, timeout = 5_000) => expect.poll(read, { timeout }); + +describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { + const daemons: DaemonServer[] = []; + + const directories: string[] = []; + + let behavior: RelayTarget = () => Promise.resolve(new Response("ok")); + + const targetCalls: { + url: string; + method: string; + headers: Headers; + tenant: string; + leaseId: string | undefined; + }[] = []; + + let twoWayBehavior: TwoWayRelayTarget = () => + Promise.resolve(new Response(null, { status: 200 })); + + const twoWayTarget: TwoWayRelayTarget = (request, tenant, leaseId, pipe) => + twoWayBehavior(request, tenant, leaseId, pipe); + + const target: RelayTarget = (request, tenant, leaseId) => { + targetCalls.push({ + headers: request.headers, + leaseId, + method: request.method, + tenant, + url: request.url, + }); + return behavior(request, tenant, leaseId); + }; + + afterEach(async () => { + behavior = () => Promise.resolve(new Response("ok")); + twoWayBehavior = () => Promise.resolve(new Response(null, { status: 200 })); + targetCalls.length = 0; + await cleanUp(daemons, directories); + }); + + async function startPair( + options: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) { + const fixture: WorkerFixture = await startWorker( + directories, + daemons, + { "agent-device": target, echo: target }, + { relayTwoWayTargets: { echo: twoWayTarget } }, + ); + const pair = linkPair(); + const gatewayEnd = pair.gatewayEnd; + fixture.worker.acceptUplink(pair.worker); + const connected = (await connectSimlockClient( + { connection: pair.gateway, principal: "gw:test" }, + true, + )) as GatewayWorkerClient; + const { client, spy } = spyOn(connected); + const directory: WorkerDirectory = { + target: (workerId) => + workerId === WORKER_ID + ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } + : undefined, + }; + const makeRelay = ( + relayOptions: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) => + new GatewayRelay({ + directory, + ...(relayOptions.deadlineMs === undefined ? {} : { deadlineMs: relayOptions.deadlineMs }), + ...(relayOptions.idleMs === undefined ? {} : { idleMs: relayOptions.idleMs }), + }); + const relay = makeRelay(options); + const opened = () => pair.gatewayTap.binaries(TYPE.OPEN).length; + return { fixture, gatewayEnd, makeRelay, opened, pair, relay, spy }; + } + + it("aborting the caller's request signal before the head comes fails the call; after it, it errors the caller's body stream; either way the target's request signal aborts and relay.request answers { cancelled: true } with no push after the cancel", async () => { + let signal: AbortSignal | undefined; + behavior = async (request) => { + signal = request.signal; + await aborted(request.signal); + return new Response("late"); + }; + const before = await startPair(); + const controller = new AbortController(); + const pendingBefore = before.relay.relay(WORKER_ID, buildCall({ signal: controller.signal })); + const settledBefore = settleWithin(pendingBefore); + await poll(() => signal !== undefined).toBe(true); + controller.abort(new Error("caller left before the head")); + const failure = await settledBefore; + await poll(() => signal?.aborted).toBe(true); + await poll(() => + before.spy.events.filter((event) => event.startsWith("request-answered")), + ).toEqual(['request-answered:{"cancelled":true}']); + + signal = undefined; + behavior = (request) => { + signal = request.signal; + return Promise.resolve( + new Response( + new ReadableStream({ + async pull(streamController) { + await sleep(10); + streamController.enqueue(patternBytes(512, 1)); + }, + }), + ), + ); + }; + const after = await startPair(); + const secondController = new AbortController(); + const response = await after.relay.relay( + WORKER_ID, + buildCall({ signal: secondController.signal }), + ); + const reader = response.body?.getReader(); + await reader?.read(); + secondController.abort(new Error("caller left after the head")); + const bodyFailure = await settleWithin( + (async () => { + for (;;) { + const step = await reader?.read(); + if (step === undefined || step.done) return; + } + })(), + ); + await poll(() => signal?.aborted).toBe(true); + await poll(() => + after.spy.events.filter((event) => event.startsWith("request-answered")), + ).toEqual(['request-answered:{"cancelled":true}']); + await sleep(150); + const eventsAfterCancel = after.spy.events.slice( + after.spy.events.indexOf("cancel-answered") + 1, + ); + + expect(failure).toMatchObject({ message: "caller left before the head", state: "rejected" }); + expect(bodyFailure).toMatchObject({ message: "caller left after the head", state: "rejected" }); + expect(after.spy.events).toContain("cancel-answered"); + expect(eventsAfterCancel.filter((event) => event.startsWith("push:"))).toEqual([]); + }, 30_000); + + it("a caller's request body that errors mid-upload aborts the target's request signal", async () => { + let signal: AbortSignal | undefined; + behavior = async (request) => { + signal = request.signal; + await readParts(request.body).catch(() => undefined); + await aborted(request.signal); + return new Response("late"); + }; + const { relay } = await startPair(); + let step = 0; + const body = new ReadableStream({ + pull(controller) { + step += 1; + if (step === 1) controller.enqueue(patternBytes(1024, 8)); + else controller.error(new Error("upload broke")); + }, + }); + + const outcome = await settleWithin(relay.relay(WORKER_ID, buildCall({ body, method: "PUT" }))); + + expect(outcome).toMatchObject({ message: "upload broke", state: "rejected" }); + await poll(() => signal?.aborted).toBe(true); + }, 30_000); + + it("a target that never answers is cancelled after a 200 ms deadline, and the caller gets WORKER_UNREACHABLE with message `worker did not finish the relayed request within 200 ms`; one that stalls after its head, under a 10 s deadline and a 200 ms idle bound, gets its body stream errored with WORKER_UNREACHABLE `worker sent nothing on the relayed response for 200 ms`", async () => { + let signal: AbortSignal | undefined; + behavior = async (request) => { + signal = request.signal; + await aborted(request.signal); + return new Response("late"); + }; + const never = await startPair({ deadlineMs: 200 }); + const noHead = await settleWithin(never.relay.relay(WORKER_ID, buildCall({}))); + await poll(() => signal?.aborted).toBe(true); + + behavior = () => + Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("x")); + }, + }), + ), + ); + const stalling = await startPair({ deadlineMs: 10_000, idleMs: 200 }); + const response = await stalling.relay.relay(WORKER_ID, buildCall({})); + const reader = response.body?.getReader(); + await reader?.read(); + const stalled = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body"))); + + const message = `worker ${WORKER_ID} did not finish the relayed request within 200 ms`; + expect(noHead).toMatchObject({ code: "WORKER_UNREACHABLE", message, state: "rejected" }); + expect(stalled).toMatchObject({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} sent nothing on the relayed response for 200 ms`, + state: "rejected", + }); + }, 30_000); + + it("closing the uplink mid-relay fails the caller with WORKER_UNREACHABLE, or errors its body stream after the head, and aborts the target's request signal", async () => { + let signal: AbortSignal | undefined; + behavior = async (request) => { + signal = request.signal; + await aborted(request.signal); + return new Response("late"); + }; + const before = await startPair(); + const noHead = settleWithin(before.relay.relay(WORKER_ID, buildCall({}))); + await poll(() => signal !== undefined).toBe(true); + await before.pair.gatewayEnd.close(); + const noHeadOutcome = await noHead; + await poll(() => signal?.aborted).toBe(true); + + signal = undefined; + behavior = (request) => { + signal = request.signal; + return Promise.resolve( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("x")); + }, + }), + ), + ); + }; + const after = await startPair(); + const response = await after.relay.relay(WORKER_ID, buildCall({})); + const reader = response.body?.getReader(); + await reader?.read(); + await after.pair.gatewayEnd.close(); + const bodyOutcome = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body"))); + + expect(noHeadOutcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + expect(bodyOutcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + await poll(() => signal?.aborted).toBe(true); + }, 30_000); + + it("closing the uplink ends every two-way relay: the gateway side gets WORKER_UNREACHABLE and the target's signal aborts", async () => { + const signals: AbortSignal[] = []; + twoWayBehavior = (_request, _tenant, _leaseId, pipe) => { + signals.push(pipe.signal); + return Promise.resolve(new Response(null, { status: 200 })); + }; + const { gatewayEnd, relay } = await startPair(); + const open = async () => { + const opened = (await relay.relayTwoWay(WORKER_ID, buildCall({}))) as TwoWayRelay; + return { reader: opened.readable.getReader(), writer: opened.writable.getWriter() }; + }; + const first = await open(); + const second = await open(); + await poll(() => signals.map((signal) => signal.aborted)).toEqual([false, false]); + + await gatewayEnd.close(); + const outcomes = await Promise.all([ + settleWithin(first.reader.read()), + settleWithin(first.writer.write(new Uint8Array(1))), + settleWithin(second.reader.read()), + settleWithin(second.writer.write(new Uint8Array(1))), + ]); + + for (const settled of outcomes) { + expect(settled).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + } + await poll(() => signals.map((signal) => signal.aborted)).toEqual([true, true]); + }, 30_000); +}); diff --git a/src/daemon/relay-gateway-capacity.e2e.test.ts b/src/daemon/relay-gateway-capacity.e2e.test.ts new file mode 100644 index 000000000..143d426bb --- /dev/null +++ b/src/daemon/relay-gateway-capacity.e2e.test.ts @@ -0,0 +1,322 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { GatewayRelay, type GatewayWorkerClient, type WorkerDirectory } from "../gateway/index.js"; +import { linkPair, TYPE } from "../link/testing.js"; +import { connectSimlockClient } from "../simlock-client/client.js"; +import type { RelayTarget, TwoWayRelayTarget } from "./relay.js"; +import type { DaemonServer } from "./server.js"; +import { + aborted, + cleanUp, + digestOf, + gate, + MIB, + patternDigest, + patternStream, + readParts, + settleWithin, + sleep, + startWorker, + type WorkerFixture, +} from "./test-relay-support.js"; +import { WORKER_ID, buildCall, spyOn } from "./test-relay-gateway.js"; + +const poll = (read: () => T | Promise, timeout = 5_000) => expect.poll(read, { timeout }); + +describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { + const daemons: DaemonServer[] = []; + + const directories: string[] = []; + + let behavior: RelayTarget = () => Promise.resolve(new Response("ok")); + + const targetCalls: { + url: string; + method: string; + headers: Headers; + tenant: string; + leaseId: string | undefined; + }[] = []; + + let twoWayBehavior: TwoWayRelayTarget = () => + Promise.resolve(new Response(null, { status: 200 })); + + const twoWayTarget: TwoWayRelayTarget = (request, tenant, leaseId, pipe) => + twoWayBehavior(request, tenant, leaseId, pipe); + + const target: RelayTarget = (request, tenant, leaseId) => { + targetCalls.push({ + headers: request.headers, + leaseId, + method: request.method, + tenant, + url: request.url, + }); + return behavior(request, tenant, leaseId); + }; + + afterEach(async () => { + behavior = () => Promise.resolve(new Response("ok")); + twoWayBehavior = () => Promise.resolve(new Response(null, { status: 200 })); + targetCalls.length = 0; + await cleanUp(daemons, directories); + }); + + async function startPair( + options: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) { + const fixture: WorkerFixture = await startWorker( + directories, + daemons, + { "agent-device": target, echo: target }, + { relayTwoWayTargets: { echo: twoWayTarget } }, + ); + const pair = linkPair(); + const gatewayEnd = pair.gatewayEnd; + fixture.worker.acceptUplink(pair.worker); + const connected = (await connectSimlockClient( + { connection: pair.gateway, principal: "gw:test" }, + true, + )) as GatewayWorkerClient; + const { client, spy } = spyOn(connected); + const directory: WorkerDirectory = { + target: (workerId) => + workerId === WORKER_ID + ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } + : undefined, + }; + const makeRelay = ( + relayOptions: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, + ) => + new GatewayRelay({ + directory, + ...(relayOptions.deadlineMs === undefined ? {} : { deadlineMs: relayOptions.deadlineMs }), + ...(relayOptions.idleMs === undefined ? {} : { idleMs: relayOptions.idleMs }), + }); + const relay = makeRelay(options); + const opened = () => pair.gatewayTap.binaries(TYPE.OPEN).length; + return { fixture, gatewayEnd, makeRelay, opened, pair, relay, spy }; + } + + it("a relay to a worker with no uplink fails with WORKER_UNREACHABLE and the target sees nothing", async () => { + const { relay } = await startPair(); + + const outcome = await settleWithin(relay.relay("wrk_nobody", buildCall({}))); + await sleep(100); + + expect(outcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); + expect(targetCalls).toEqual([]); + }); + + it("a relay naming a target nobody registered fails with RELAY_TARGET_MISSING", async () => { + const { relay } = await startPair(); + + const outcome = await settleWithin(relay.relay(WORKER_ID, buildCall({ target: "nobody" }))); + + expect(outcome).toMatchObject({ code: "RELAY_TARGET_MISSING", state: "rejected" }); + expect(targetCalls).toEqual([]); + }); + + it("the caller's authorization, x-agent-device-token, connection and transfer-encoding headers never reach the target, and a target's content-length and connection headers never reach the caller", async () => { + behavior = () => + Promise.resolve( + new Response("hello", { + headers: { connection: "close", "content-length": "5", "x-kept": "yes" }, + }), + ); + const { relay } = await startPair(); + + const response = await relay.relay( + WORKER_ID, + buildCall({ + headers: { + authorization: "Bearer gateway-secret", + connection: "keep-alive", + "transfer-encoding": "chunked", + "x-agent-device-token": "device-secret", + "x-keep-me": "yes", + }, + }), + ); + await response.text(); + + const seen = targetCalls[0]?.headers; + expect(seen?.get("x-keep-me")).toBe("yes"); + expect(seen?.get("authorization")).toBeNull(); + expect(seen?.get("x-agent-device-token")).toBeNull(); + expect(seen?.get("connection")).toBeNull(); + expect(seen?.get("transfer-encoding")).toBeNull(); + expect(response.headers.get("x-kept")).toBe("yes"); + expect(response.headers.get("content-length")).toBeNull(); + expect(response.headers.get("connection")).toBeNull(); + }); + + it("a 65th relay to one worker starts once one of 64 ends, and not before", async () => { + const releases: (() => void)[] = []; + behavior = async () => { + const held = gate(); + releases.push(held.open); + await held.promise; + return new Response("ok"); + }; + const { opened, relay } = await startPair(); + + const calls = Array.from({ length: 65 }, () => + settleWithin(relay.relay(WORKER_ID, buildCall({})), 20_000), + ); + await poll(() => targetCalls.length, 10_000).toBe(64); + await sleep(200); + const startedBeforeRelease = targetCalls.length; + releases[0]?.(); + await poll(() => targetCalls.length, 10_000).toBe(65); + for (const release of releases) release(); + const outcomes = await Promise.all(calls); + + expect(startedBeforeRelease).toBe(64); + expect(opened()).toBe(65); + expect(outcomes.map((outcome) => outcome.state)).toEqual( + Array.from({ length: 65 }, () => "fulfilled"), + ); + }, 60_000); + + it("a 65th relay whose 200 ms deadline passes first fails WORKER_UNREACHABLE with the deadline message, and its target is never called", async () => { + const release = gate(); + behavior = async () => { + await release.promise; + return new Response("ok"); + }; + const { makeRelay } = await startPair(); + const long = makeRelay(); + const short = makeRelay({ deadlineMs: 200 }); + const holders = Array.from({ length: 64 }, () => + settleWithin(long.relay(WORKER_ID, buildCall({})), 20_000), + ); + await poll(() => targetCalls.length, 10_000).toBe(64); + + const outcome = await settleWithin(short.relay(WORKER_ID, buildCall({}))); + release.open(); + await Promise.all(holders); + + expect(outcome).toEqual({ + code: "WORKER_UNREACHABLE", + message: `worker ${WORKER_ID} did not finish the relayed request within 200 ms`, + state: "rejected", + }); + expect(targetCalls).toHaveLength(64); + }, 60_000); + + it("a 65th relay whose caller aborts while it waits for a channel fails at once with the abort reason", async () => { + const release = gate(); + behavior = async () => { + await release.promise; + return new Response("ok"); + }; + const { relay } = await startPair(); + const holders = Array.from({ length: 64 }, () => + settleWithin(relay.relay(WORKER_ID, buildCall({})), 20_000), + ); + await poll(() => targetCalls.length, 10_000).toBe(64); + const controller = new AbortController(); + const waiting = settleWithin( + relay.relay(WORKER_ID, buildCall({ signal: controller.signal })), + 3_000, + ); + await sleep(100); + + controller.abort(new Error("caller left while waiting")); + const outcome = await waiting; + release.open(); + await Promise.all(holders); + + expect(outcome).toMatchObject({ message: "caller left while waiting", state: "rejected" }); + expect(targetCalls).toHaveLength(64); + }, 60_000); + + const leaving: readonly { + readonly name: string; + readonly run: ( + relay: GatewayRelay, + makeRelay: (o?: { deadlineMs?: number }) => GatewayRelay, + ) => Promise; + }[] = [ + { + name: "were cancelled", + run: async (relay) => { + const controller = new AbortController(); + const pending = settleWithin( + relay.relay(WORKER_ID, buildCall({ signal: controller.signal })), + ); + await sleep(20); + controller.abort(new Error("cancelled")); + return pending; + }, + }, + { + name: "timed out", + run: (_relay, makeRelay) => + settleWithin(makeRelay({ deadlineMs: 100 }).relay(WORKER_ID, buildCall({}))), + }, + { + name: "failed with a missing target", + run: (relay) => settleWithin(relay.relay(WORKER_ID, buildCall({ target: "nobody" }))), + }, + { + name: "had a target that answered without reading its upload", + run: async (relay) => { + const response = await relay.relay( + WORKER_ID, + buildCall({ body: patternStream(MIB, 3), method: "PUT" }), + ); + return settleWithin(response.text()); + }, + }, + ]; + + it.each(leaving)( + "after 64 relays to one worker that $name, a 65th starts at once", + async ({ run }) => { + const held = gate(); + behavior = async (request) => { + if (request.headers.get("x-marker") === "last") { + await held.promise; + return new Response("last"); + } + if (request.method === "PUT") return new Response("not reading"); + await aborted(request.signal); + return new Response("late"); + }; + const { makeRelay, opened, relay } = await startPair(); + + for (let batch = 0; batch < 4; batch += 1) { + await Promise.all(Array.from({ length: 16 }, () => run(relay, makeRelay))); + } + const last = settleWithin( + relay.relay(WORKER_ID, buildCall({ headers: { "x-marker": "last" } })), + 10_000, + ); + await poll(() => targetCalls.some((call) => call.headers.get("x-marker") === "last")).toBe( + true, + ); + const channels = opened(); + held.open(); + await last; + + expect(channels).toBe(65); + }, + 60_000, + ); + + it("with the worker's transport held so the {} answer overtakes a 20 MiB response, the caller still gets every byte, and its body ends only at the channel's end", async () => { + behavior = () => Promise.resolve(new Response(patternStream(20 * MIB, 9))); + const { pair, relay, spy } = await startPair(); + pair.workerEnd.hold(); + + const pending = relay.relay(WORKER_ID, buildCall({})); + await sleep(300); + pair.workerEnd.release(); + const response = await pending; + const digest = digestOf(await readParts(response.body)); + + expect(digest).toBe(patternDigest(20 * MIB, 9)); + expect(spy.events).toContain("request-answered:{}"); + }, 60_000); +}); diff --git a/src/daemon/relay-gateway-files.e2e.test.ts b/src/daemon/relay-gateway-files.e2e.test.ts new file mode 100644 index 000000000..adeb6bfdd --- /dev/null +++ b/src/daemon/relay-gateway-files.e2e.test.ts @@ -0,0 +1,447 @@ +import { request as httpRequest } from "node:http"; +import { readdir } from "node:fs/promises"; +import { connect, createServer } from "node:net"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, onTestFinished } from "vitest"; +import type { LeaseFilesLimits } from "../lease-files/index.js"; +import { TYPE } from "../link/testing.js"; +import type { DispatchSession } from "./dispatch.js"; +import type { RelayTarget, TwoWayRelayTarget } from "./relay.js"; +import type { StartDaemonOptions } from "./main.js"; +import type { DaemonServer } from "./server.js"; +import { + relayAdminSession, + cleanUp, + digestOf, + MIB, + KIB, + patternBytes, + patternDigest, + patternStream, + readParts, + settleWithin, + sleep, + startFleet as startFleet_, + type FleetExec, +} from "./test-relay-support.js"; +import { buildCall } from "./test-relay-gateway.js"; + +const poll = (read: () => T | Promise, timeout = 5_000) => expect.poll(read, { timeout }); + +describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { + const daemons: DaemonServer[] = []; + + const directories: string[] = []; + + let behavior: RelayTarget = () => Promise.resolve(new Response("ok")); + + const targetCalls: { + url: string; + method: string; + headers: Headers; + tenant: string; + leaseId: string | undefined; + }[] = []; + + let twoWayBehavior: TwoWayRelayTarget = () => + Promise.resolve(new Response(null, { status: 200 })); + + const twoWayTarget: TwoWayRelayTarget = (request, tenant, leaseId, pipe) => + twoWayBehavior(request, tenant, leaseId, pipe); + + const target: RelayTarget = (request, tenant, leaseId) => { + targetCalls.push({ + headers: request.headers, + leaseId, + method: request.method, + tenant, + url: request.url, + }); + return behavior(request, tenant, leaseId); + }; + + afterEach(async () => { + behavior = () => Promise.resolve(new Response("ok")); + twoWayBehavior = () => Promise.resolve(new Response(null, { status: 200 })); + targetCalls.length = 0; + await cleanUp(daemons, directories); + }); + + const startFleet = ( + execCommand?: FleetExec, + extra: { + readonly proxyAuth?: Record; + readonly worker?: Partial; + readonly gateway?: Partial; + } = {}, + ) => + startFleet_( + { daemons, directories, target, twoWayTarget }, + { ...(execCommand === undefined ? {} : { execCommand }), ...extra }, + ); + + const agent = (overrides: Partial = {}): DispatchSession => ({ + manageEventSubscription: () => undefined, + via: "socket", + principal: "fleet-agent-1", + role: "agent", + ...overrides, + }); + + async function fileRig(limits?: LeaseFilesLimits, fileSilenceMs?: number) { + const fleet = await startFleet(undefined, { + ...(limits === undefined ? {} : { worker: { leaseFilesLimits: limits } }), + ...(fileSilenceMs === undefined ? {} : { gateway: { fileSilenceMs } }), + }); + const { secret, token } = await fleet.gateway.dispatch( + "token.create", + { label: "holder", role: "agent" }, + relayAdminSession, + ); + const holder = agent({ principal: token.id }); + const grant = await fleet.gateway.dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + holder, + ); + const callOn = + (leaseId: string, bearer: string) => + (path: string, init: RequestInit = {}) => + fetch(`http://127.0.0.1:${String(fleet.port)}/v1/leases/${leaseId}/files${path}`, { + ...init, + headers: { authorization: `Bearer ${bearer}`, ...init.headers }, + }); + const call = callOn(grant.lease.id, secret); + const base = `http://127.0.0.1:${String(fleet.port)}/v1/leases/${grant.lease.id}/files`; + // why: `fetch` refuses a `Content-Length` that does not match its body, and the 2 GiB refusal is decided on the header alone. + const declare = (path: string, bytes: number) => + new Promise<{ status: number; body: string }>((resolve, reject) => { + const request = httpRequest( + `${base}${path}`, + { + headers: { authorization: `Bearer ${secret}`, "content-length": String(bytes) }, + method: "PUT", + }, + (response) => { + const parts: Buffer[] = []; + response.on("data", (part: Buffer) => parts.push(part)); + response.on("end", () => + resolve({ + body: Buffer.concat(parts).toString(), + status: response.statusCode ?? 0, + }), + ); + }, + ); + request.on("error", reject); + // why: a gateway that waits for the body instead of refusing the header must fail the assertion, not the test's timeout. + request.setTimeout(5_000, () => { + request.destroy(); + resolve({ body: "no answer within 5 s", status: 0 }); + }); + request.flushHeaders(); + }); + const channelsOpened = () => fleet.taps.flatMap((tap) => tap.binaries(TYPE.OPEN)).length; + // why: CI saw NO_CAPACITY for a request right after the release, so the helper retries until the fleet can grant. + const newLease = async () => { + let granted: string | undefined; + await poll(async () => { + granted = await fleet.gateway + .dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + holder, + ) + .then( + (next) => next.lease.id, + () => undefined, + ); + return granted !== undefined; + }, 15_000).toBe(true); + const leaseId = granted as string; + return { call: callOn(leaseId, secret), leaseId }; + }; + const stranger = async () => { + const other = await fleet.gateway.dispatch( + "token.create", + { label: "stranger", role: "agent" }, + relayAdminSession, + ); + return callOn(grant.lease.id, other.secret); + }; + return { + ...fleet, + call, + channelsOpened, + declare, + holder, + leaseId: grant.lease.id, + newLease, + secret, + stranger, + }; + } + + // why: a loaded machine can hand a request to the worker late; holding a connection's bytes back makes the upload reach the worker late, so the test must wait for it before it probes the name. + async function lateProxy(port: number, holdMs: number) { + const sockets = new Set<{ destroy: () => void }>(); + const server = createServer((client) => { + sockets.add(client); + client.pause(); + setTimeout(() => { + const upstream = connect(port, "127.0.0.1"); + sockets.add(upstream); + client.pipe(upstream).pipe(client); + client.resume(); + client.on("error", () => upstream.destroy()); + upstream.on("error", () => client.destroy()); + }, holdMs); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + onTestFinished( + () => + new Promise((resolve) => { + server.close(() => resolve()); + for (const socket of sockets) socket.destroy(); + }), + ); + return (server.address() as { port: number }).port; + } + + // why: `feed.write` resolving only means fetch took the first byte; the worker's store holds the name once its `.upload-` file exists. + async function uploadsOnWorker(directory: string) { + const root = join(directory, "lease-files"); + const folders = await readdir(root).catch(() => []); + const entries = await Promise.all(folders.map((folder) => readdir(join(root, folder)))); + return entries.flat().filter((entry) => entry.startsWith(".upload-")).length; + } + + it("through a gateway, a lease holder's PUT, GET, list and DELETE reach the lease's worker and answer as on a worker", async () => { + const { call, channelsOpened, declare } = await fileRig({ + fileBytes: 32 * MIB, + leaseBytes: 512 * MIB, + silenceMs: 330_000, + }); + const bytes = patternBytes(20 * MIB, 5); + + const put = await call("/fixture.tar", { + body: patternStream(20 * MIB, 5), + duplex: "half", + method: "PUT", + } as RequestInit); + const get = await call("/fixture.tar"); + const ranged = await call("/fixture.tar", { headers: { range: "bytes=10-19" } }); + const listed = await call(""); + const deleted = await call("/fixture.tar", { method: "DELETE" }); + const after = await call(""); + const names = Array.from({ length: 10 }, (_, index) => `many-${String(index)}.bin`); + const many = await Promise.all( + names.map((name, index) => + call(`/${name}`, { + body: patternStream(20 * MIB, 20 + index), + duplex: "half", + method: "PUT", + } as RequestInit), + ), + ); + const stored = await (await call("")).json(); + const missing = await call("/missing.bin"); + const oversized = await call("/large.bin", { + body: patternStream(33 * MIB, 6), + duplex: "half", + method: "PUT", + } as RequestInit); + const opened = channelsOpened(); + const tooLarge = await declare("/huge.bin", 2 * 1024 * MIB + 1); + + expect(put.status).toBe(201); + expect(await put.json()).toEqual({ name: "fixture.tar", size: 20 * MIB }); + expect(get.status).toBe(200); + expect(digestOf([new Uint8Array(await get.arrayBuffer())])).toBe(digestOf([bytes])); + expect(get.headers.get("content-length")).toBe(String(20 * MIB)); + expect(ranged.status).toBe(206); + expect(ranged.headers.get("content-length")).toBe("10"); + expect(ranged.headers.get("content-range")).toBe(`bytes 10-19/${String(20 * MIB)}`); + expect(new Uint8Array(await ranged.arrayBuffer())).toEqual(bytes.subarray(10, 20)); + expect(await listed.json()).toMatchObject({ files: [{ name: "fixture.tar", size: 20 * MIB }] }); + expect(deleted.status).toBe(204); + expect(await after.json()).toEqual({ files: [] }); + expect(many.map((response) => response.status)).toEqual(names.map(() => 201)); + expect(stored).toMatchObject({ files: names.map((name) => ({ name, size: 20 * MIB })) }); + expect(missing.status).toBe(404); + expect(await missing.json()).toMatchObject({ error: { code: "UNKNOWN_FILE" } }); + expect(oversized.status).toBe(413); + expect(await oversized.json()).toMatchObject({ + error: { code: "FILE_TOO_LARGE", name: "large.bin" }, + }); + expect(tooLarge.status).toBe(413); + expect(JSON.parse(tooLarge.body)).toMatchObject({ + error: { code: "FILE_TOO_LARGE", name: "huge.bin" }, + }); + expect(channelsOpened()).toBe(opened); + }, 120_000); + + it("through a gateway, a PUT of a name whose upload is still arriving gets 409 FILE_BUSY, once that upload has reached the worker late and is still arriving", async () => { + const rig = await fileRig({ fileBytes: 32 * MIB, leaseBytes: 512 * MIB, silenceMs: 330_000 }); + const late = await lateProxy(rig.port, 500); + const busy = new TransformStream(); + const holding = fetch( + `http://127.0.0.1:${String(late)}/v1/leases/${rig.leaseId}/files/busy.bin`, + { + body: busy.readable, + duplex: "half", + headers: { authorization: `Bearer ${rig.secret}` }, + method: "PUT", + } as RequestInit, + ); + const feed = busy.writable.getWriter(); + await feed.write(new Uint8Array(1)); + await poll(() => uploadsOnWorker(rig.worker.directory)).toBe(1); + + const second = await rig.call("/busy.bin", { body: "x", method: "PUT" }); + await feed.close(); + const first = await holding; + + expect(second.status).toBe(409); + expect(await second.json()).toMatchObject({ error: { code: "FILE_BUSY", name: "busy.bin" } }); + expect(first.status).toBe(201); + expect(await first.json()).toEqual({ name: "busy.bin", size: 1 }); + }, 30_000); + + it("through a gateway, a caller that does not hold the lease gets 404 UNKNOWN_LEASE, and a lease whose worker is gone gets 503 WORKER_UNREACHABLE", async () => { + const rig = await fileRig({ fileBytes: 32 * MIB, leaseBytes: MIB, silenceMs: 330_000 }); + const asStranger = await rig.stranger(); + const routes = (call: typeof rig.call) => [ + () => call("/fixture.tar", { body: "x", method: "PUT" }), + () => call("/fixture.tar"), + () => call("/fixture.tar", { method: "DELETE" }), + () => call(""), + ]; + + const refused = await Promise.all(routes(asStranger).map((send) => send())); + const full = await rig.call("/big.bin", { body: new Uint8Array(MIB + 1), method: "PUT" }); + const feed = new TransformStream(); + const writer = feed.writable.getWriter(); + const running = rig.call("/running.bin", { + body: feed.readable, + duplex: "half", + method: "PUT", + } as RequestInit); + await writer.write(new Uint8Array(1)); + await poll(async () => (await rig.call("")).status).toBe(200); + await rig.gateway.dispatch("lease.release", { leaseId: rig.leaseId }, rig.holder); + await writer.write(new Uint8Array(1)); + await writer.close(); + const ended = await running; + const next = await rig.newLease(); + await rig.worker.worker.stop("test"); + await poll(async () => (await next.call("")).status).toBe(503); + + expect(refused.map((response) => response.status)).toEqual([404, 404, 404, 404]); + for (const response of refused) { + expect(await response.json()).toMatchObject({ + error: { code: "UNKNOWN_LEASE", leaseId: rig.leaseId }, + }); + } + expect(full.status).toBe(413); + const fullBody = (await full.json()) as { + error: { code: string; message: string; leaseId: string }; + }; + expect(fullBody.error).toMatchObject({ code: "LEASE_FILES_FULL", leaseId: rig.leaseId }); + expect(fullBody.error.message).toContain(`Lease ${rig.leaseId} holds`); + expect(ended.status).toBe(404); + const endedBody = (await ended.json()) as { + error: { code: string; message: string; leaseId: string }; + }; + expect(endedBody.error).toMatchObject({ code: "UNKNOWN_LEASE", leaseId: rig.leaseId }); + expect(endedBody.error.message).toBe(`Unknown lease: ${rig.leaseId}`); + expect(await (await next.call("")).json()).toMatchObject({ + error: { code: "WORKER_UNREACHABLE" }, + }); + }, 60_000); + + it("through a gateway, a file upload is cut only by 330 s of silence, not by 330 s in total", async () => { + const { call } = await fileRig(undefined, 1_000); + const pieces = 12; + let sent = 0; + const flowing = new ReadableStream({ + async pull(controller) { + await sleep(300); + controller.enqueue(new Uint8Array(KIB)); + sent += 1; + if (sent === pieces) controller.close(); + }, + }); + const silent = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(KIB)); + }, + }); + + const slow = await call("/slow.bin", { + body: flowing, + duplex: "half", + method: "PUT", + } as RequestInit); + const stalled = await settleWithin( + call("/stalled.bin", { body: silent, duplex: "half", method: "PUT" } as RequestInit), + 6_000, + ); + + expect(slow.status).toBe(201); + expect(await slow.json()).toEqual({ name: "slow.bin", size: pieces * KIB }); + expect(stalled).toMatchObject({ state: "fulfilled", value: { status: 408 } }); + const body = await (stalled as { value: Response }).value.json(); + expect(body).toEqual({ + error: { + code: "UPLOAD_STALLED", + message: "No bytes of stalled.bin arrived for 1 s", + name: "stalled.bin", + }, + }); + expect((await (await call("")).json()) as { files: { name: string }[] }).toEqual({ + files: [{ name: "slow.bin", size: pieces * KIB }], + }); + }, 30_000); + + it("a 20 MiB PUT body arrives at the target byte for byte, and a 20 MiB response body comes back byte for byte", async () => { + let received = ""; + behavior = async (request) => { + received = digestOf(await readParts(request.body)); + return new Response(patternStream(20 * MIB, 9)); + }; + const { relay, taps, workerId } = await startFleet(); + + const response = await relay.relay( + workerId, + buildCall({ body: patternStream(20 * MIB, 5), method: "PUT" }), + ); + const back = digestOf(await readParts(response.body)); + const dataBytes = taps + .flatMap((tap) => tap.binaries(TYPE.DATA)) + .reduce((total, envelope) => total + envelope.payload.length, 0); + + expect(received).toBe(patternDigest(20 * MIB, 5)); + expect(back).toBe(patternDigest(20 * MIB, 9)); + expect(dataBytes).toBeGreaterThanOrEqual(20 * MIB); + }, 120_000); + + it("an empty upload (0 bytes) reaches the target as an empty body, and a GET relays as before", async () => { + const lengths: number[] = []; + behavior = async (request) => { + lengths.push((await request.arrayBuffer()).byteLength); + return new Response("fine"); + }; + const { relay, workerId } = await startFleet(); + + const put = await relay.relay( + workerId, + buildCall({ body: patternStream(0, 1), method: "PUT" }), + ); + const get = await relay.relay(workerId, buildCall({})); + + expect(await put.text()).toBe("fine"); + expect(await get.text()).toBe("fine"); + expect(lengths).toEqual([0, 0]); + }, 30_000); +}); diff --git a/src/daemon/relay-gateway.e2e.test.ts b/src/daemon/relay-gateway.e2e.test.ts index a4ec7d4c3..42fa44445 100644 --- a/src/daemon/relay-gateway.e2e.test.ts +++ b/src/daemon/relay-gateway.e2e.test.ts @@ -1,121 +1,28 @@ -import { request as httpRequest } from "node:http"; -import { readdir } from "node:fs/promises"; -import { connect, createServer } from "node:net"; -import { join } from "node:path"; - -import { afterEach, describe, expect, it, onTestFinished } from "vitest"; - -import { - GatewayRelay, - type GatewayWorkerClient, - type TwoWayRelay, - type WorkerDirectory, -} from "../gateway/index.js"; -import type { LeaseFilesLimits } from "../lease-files/index.js"; -import { linkPair, TYPE } from "../link/testing.js"; -import { connectSimlockClient } from "../simlock-client/client.js"; -import type { DispatchSession } from "./dispatch.js"; +import { createServer } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { type TwoWayRelay } from "../gateway/index.js"; +import { TYPE } from "../link/testing.js"; import type { RelayTarget, TwoWayRelayTarget } from "./relay.js"; import type { StartDaemonOptions } from "./main.js"; import type { DaemonServer } from "./server.js"; import { - aborted, - relayAdminSession, cleanUp, - digestOf, gate, - MIB, - KIB, - patternBytes, - patternDigest, - patternStream, - readParts, settleWithin, - sleep, startFleet as startFleet_, - startWorker, type FleetExec, - type WorkerFixture, } from "./test-relay-support.js"; +import { buildCall } from "./test-relay-gateway.js"; const poll = (read: () => T | Promise, timeout = 5_000) => expect.poll(read, { timeout }); -type CallInit = { - readonly target?: string; - readonly method?: string; - readonly url?: string; - readonly headers?: Record; - readonly body?: ReadableStream; - readonly signal?: AbortSignal; - readonly tenant?: string; - readonly leaseId?: string; -}; - -const WORKER_ID = "wrk_a"; - -function buildCall(init: CallInit) { - const request = new Request(init.url ?? "http://gateway.test:4321/agent-device/rpc?x=1&y=2", { - method: init.method ?? "GET", - ...(init.headers === undefined ? {} : { headers: init.headers }), - ...(init.body === undefined ? {} : { body: init.body, duplex: "half" }), - ...(init.signal === undefined ? {} : { signal: init.signal }), - } as RequestInit); - return { - request, - target: init.target ?? "echo", - tenant: init.tenant ?? "gw.g1.alice", - ...(init.leaseId === undefined ? {} : { leaseId: init.leaseId }), - }; -} - -interface ClientSpy { - readonly events: string[]; -} - -function spyOn(client: GatewayWorkerClient): { client: GatewayWorkerClient; spy: ClientSpy } { - const spy: ClientSpy = { events: [] }; - const wrapHooks = (value: unknown): unknown => { - if (typeof value !== "object" || value === null) return value; - const hooks = { ...(value as Record) }; - const original = hooks.onRelayHead; - if (typeof original === "function") { - hooks.onRelayHead = (...args: unknown[]) => { - spy.events.push("push:onRelayHead"); - return (original as (...rest: unknown[]) => unknown)(...args); - }; - } - return hooks; - }; - const wrapped = new Proxy(client, { - get(target, property) { - const original: unknown = Reflect.get(target, property, target); - if (typeof original !== "function") return original; - const method = original as (...args: unknown[]) => Promise; - if (property === "relayRequest") { - return async (...args: unknown[]) => { - const value = await method.apply(target, args.map(wrapHooks)); - spy.events.push(`request-answered:${JSON.stringify(value)}`); - return value; - }; - } - if (property === "relayCancel") { - return async (...args: unknown[]) => { - spy.events.push("cancel-sent"); - const value = await method.apply(target, args); - spy.events.push("cancel-answered"); - return value; - }; - } - return method.bind(target); - }, - }); - return { client: wrapped, spy }; -} - describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { const daemons: DaemonServer[] = []; + const directories: string[] = []; + let behavior: RelayTarget = () => Promise.resolve(new Response("ok")); + const targetCalls: { url: string; method: string; @@ -126,6 +33,7 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { let twoWayBehavior: TwoWayRelayTarget = () => Promise.resolve(new Response(null, { status: 200 })); + const twoWayTarget: TwoWayRelayTarget = (request, tenant, leaseId, pipe) => twoWayBehavior(request, tenant, leaseId, pipe); @@ -147,42 +55,6 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { await cleanUp(daemons, directories); }); - async function startPair( - options: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, - ) { - const fixture: WorkerFixture = await startWorker( - directories, - daemons, - { "agent-device": target, echo: target }, - { relayTwoWayTargets: { echo: twoWayTarget } }, - ); - const pair = linkPair(); - const gatewayEnd = pair.gatewayEnd; - fixture.worker.acceptUplink(pair.worker); - const connected = (await connectSimlockClient( - { connection: pair.gateway, principal: "gw:test" }, - true, - )) as GatewayWorkerClient; - const { client, spy } = spyOn(connected); - const directory: WorkerDirectory = { - target: (workerId) => - workerId === WORKER_ID - ? { client: () => client, reachable: true, refresh: async () => undefined, workerId } - : undefined, - }; - const makeRelay = ( - relayOptions: { readonly deadlineMs?: number; readonly idleMs?: number } = {}, - ) => - new GatewayRelay({ - directory, - ...(relayOptions.deadlineMs === undefined ? {} : { deadlineMs: relayOptions.deadlineMs }), - ...(relayOptions.idleMs === undefined ? {} : { idleMs: relayOptions.idleMs }), - }); - const relay = makeRelay(options); - const opened = () => pair.gatewayTap.binaries(TYPE.OPEN).length; - return { fixture, gatewayEnd, makeRelay, opened, pair, relay, spy }; - } - const startFleet = ( execCommand?: FleetExec, extra: { @@ -196,14 +68,6 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { { ...(execCommand === undefined ? {} : { execCommand }), ...extra }, ); - const agent = (overrides: Partial = {}): DispatchSession => ({ - manageEventSubscription: () => undefined, - via: "socket", - principal: "fleet-agent-1", - role: "agent", - ...overrides, - }); - it("a GET relayed to the fake target arrives with its URL (origin, path and query), method, headers, tenant and lease id, and its status, headers and body come back unchanged", async () => { behavior = () => Promise.resolve( @@ -425,988 +289,4 @@ describe("a gateway relaying to a worker", { timeout: 15_000 }, () => { expect(await refusal.text()).toBe("not for you"); expect(missing).toMatchObject({ code: "RELAY_TARGET_MISSING", state: "rejected" }); }, 30_000); - - async function fileRig(limits?: LeaseFilesLimits, fileSilenceMs?: number) { - const fleet = await startFleet(undefined, { - ...(limits === undefined ? {} : { worker: { leaseFilesLimits: limits } }), - ...(fileSilenceMs === undefined ? {} : { gateway: { fileSilenceMs } }), - }); - const { secret, token } = await fleet.gateway.dispatch( - "token.create", - { label: "holder", role: "agent" }, - relayAdminSession, - ); - const holder = agent({ principal: token.id }); - const grant = await fleet.gateway.dispatch( - "lease.request", - { model: "Pixel-A", noWait: true, platform: "android" }, - holder, - ); - const callOn = - (leaseId: string, bearer: string) => - (path: string, init: RequestInit = {}) => - fetch(`http://127.0.0.1:${String(fleet.port)}/v1/leases/${leaseId}/files${path}`, { - ...init, - headers: { authorization: `Bearer ${bearer}`, ...init.headers }, - }); - const call = callOn(grant.lease.id, secret); - const base = `http://127.0.0.1:${String(fleet.port)}/v1/leases/${grant.lease.id}/files`; - // why: `fetch` refuses a `Content-Length` that does not match its body, and the 2 GiB refusal is decided on the header alone. - const declare = (path: string, bytes: number) => - new Promise<{ status: number; body: string }>((resolve, reject) => { - const request = httpRequest( - `${base}${path}`, - { - headers: { authorization: `Bearer ${secret}`, "content-length": String(bytes) }, - method: "PUT", - }, - (response) => { - const parts: Buffer[] = []; - response.on("data", (part: Buffer) => parts.push(part)); - response.on("end", () => - resolve({ - body: Buffer.concat(parts).toString(), - status: response.statusCode ?? 0, - }), - ); - }, - ); - request.on("error", reject); - // why: a gateway that waits for the body instead of refusing the header must fail the assertion, not the test's timeout. - request.setTimeout(5_000, () => { - request.destroy(); - resolve({ body: "no answer within 5 s", status: 0 }); - }); - request.flushHeaders(); - }); - const channelsOpened = () => fleet.taps.flatMap((tap) => tap.binaries(TYPE.OPEN)).length; - // why: CI saw NO_CAPACITY for a request right after the release, so the helper retries until the fleet can grant. - const newLease = async () => { - let granted: string | undefined; - await poll(async () => { - granted = await fleet.gateway - .dispatch( - "lease.request", - { model: "Pixel-A", noWait: true, platform: "android" }, - holder, - ) - .then( - (next) => next.lease.id, - () => undefined, - ); - return granted !== undefined; - }, 15_000).toBe(true); - const leaseId = granted as string; - return { call: callOn(leaseId, secret), leaseId }; - }; - const stranger = async () => { - const other = await fleet.gateway.dispatch( - "token.create", - { label: "stranger", role: "agent" }, - relayAdminSession, - ); - return callOn(grant.lease.id, other.secret); - }; - return { - ...fleet, - call, - channelsOpened, - declare, - holder, - leaseId: grant.lease.id, - newLease, - secret, - stranger, - }; - } - - // why: a loaded machine can hand a request to the worker late; holding a connection's bytes back makes the upload reach the worker late, so the test must wait for it before it probes the name. - async function lateProxy(port: number, holdMs: number) { - const sockets = new Set<{ destroy: () => void }>(); - const server = createServer((client) => { - sockets.add(client); - client.pause(); - setTimeout(() => { - const upstream = connect(port, "127.0.0.1"); - sockets.add(upstream); - client.pipe(upstream).pipe(client); - client.resume(); - client.on("error", () => upstream.destroy()); - upstream.on("error", () => client.destroy()); - }, holdMs); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - onTestFinished( - () => - new Promise((resolve) => { - server.close(() => resolve()); - for (const socket of sockets) socket.destroy(); - }), - ); - return (server.address() as { port: number }).port; - } - - // why: `feed.write` resolving only means fetch took the first byte; the worker's store holds the name once its `.upload-` file exists. - async function uploadsOnWorker(directory: string) { - const root = join(directory, "lease-files"); - const folders = await readdir(root).catch(() => []); - const entries = await Promise.all(folders.map((folder) => readdir(join(root, folder)))); - return entries.flat().filter((entry) => entry.startsWith(".upload-")).length; - } - - it("through a gateway, a lease holder's PUT, GET, list and DELETE reach the lease's worker and answer as on a worker", async () => { - const { call, channelsOpened, declare } = await fileRig({ - fileBytes: 32 * MIB, - leaseBytes: 512 * MIB, - silenceMs: 330_000, - }); - const bytes = patternBytes(20 * MIB, 5); - - const put = await call("/fixture.tar", { - body: patternStream(20 * MIB, 5), - duplex: "half", - method: "PUT", - } as RequestInit); - const get = await call("/fixture.tar"); - const ranged = await call("/fixture.tar", { headers: { range: "bytes=10-19" } }); - const listed = await call(""); - const deleted = await call("/fixture.tar", { method: "DELETE" }); - const after = await call(""); - const names = Array.from({ length: 10 }, (_, index) => `many-${String(index)}.bin`); - const many = await Promise.all( - names.map((name, index) => - call(`/${name}`, { - body: patternStream(20 * MIB, 20 + index), - duplex: "half", - method: "PUT", - } as RequestInit), - ), - ); - const stored = await (await call("")).json(); - const missing = await call("/missing.bin"); - const oversized = await call("/large.bin", { - body: patternStream(33 * MIB, 6), - duplex: "half", - method: "PUT", - } as RequestInit); - const opened = channelsOpened(); - const tooLarge = await declare("/huge.bin", 2 * 1024 * MIB + 1); - - expect(put.status).toBe(201); - expect(await put.json()).toEqual({ name: "fixture.tar", size: 20 * MIB }); - expect(get.status).toBe(200); - expect(digestOf([new Uint8Array(await get.arrayBuffer())])).toBe(digestOf([bytes])); - expect(get.headers.get("content-length")).toBe(String(20 * MIB)); - expect(ranged.status).toBe(206); - expect(ranged.headers.get("content-length")).toBe("10"); - expect(ranged.headers.get("content-range")).toBe(`bytes 10-19/${String(20 * MIB)}`); - expect(new Uint8Array(await ranged.arrayBuffer())).toEqual(bytes.subarray(10, 20)); - expect(await listed.json()).toMatchObject({ files: [{ name: "fixture.tar", size: 20 * MIB }] }); - expect(deleted.status).toBe(204); - expect(await after.json()).toEqual({ files: [] }); - expect(many.map((response) => response.status)).toEqual(names.map(() => 201)); - expect(stored).toMatchObject({ files: names.map((name) => ({ name, size: 20 * MIB })) }); - expect(missing.status).toBe(404); - expect(await missing.json()).toMatchObject({ error: { code: "UNKNOWN_FILE" } }); - expect(oversized.status).toBe(413); - expect(await oversized.json()).toMatchObject({ - error: { code: "FILE_TOO_LARGE", name: "large.bin" }, - }); - expect(tooLarge.status).toBe(413); - expect(JSON.parse(tooLarge.body)).toMatchObject({ - error: { code: "FILE_TOO_LARGE", name: "huge.bin" }, - }); - expect(channelsOpened()).toBe(opened); - }, 120_000); - - it("through a gateway, a PUT of a name whose upload is still arriving gets 409 FILE_BUSY, once that upload has reached the worker late and is still arriving", async () => { - const rig = await fileRig({ fileBytes: 32 * MIB, leaseBytes: 512 * MIB, silenceMs: 330_000 }); - const late = await lateProxy(rig.port, 500); - const busy = new TransformStream(); - const holding = fetch( - `http://127.0.0.1:${String(late)}/v1/leases/${rig.leaseId}/files/busy.bin`, - { - body: busy.readable, - duplex: "half", - headers: { authorization: `Bearer ${rig.secret}` }, - method: "PUT", - } as RequestInit, - ); - const feed = busy.writable.getWriter(); - await feed.write(new Uint8Array(1)); - await poll(() => uploadsOnWorker(rig.worker.directory)).toBe(1); - - const second = await rig.call("/busy.bin", { body: "x", method: "PUT" }); - await feed.close(); - const first = await holding; - - expect(second.status).toBe(409); - expect(await second.json()).toMatchObject({ error: { code: "FILE_BUSY", name: "busy.bin" } }); - expect(first.status).toBe(201); - expect(await first.json()).toEqual({ name: "busy.bin", size: 1 }); - }, 30_000); - - it("through a gateway, a caller that does not hold the lease gets 404 UNKNOWN_LEASE, and a lease whose worker is gone gets 503 WORKER_UNREACHABLE", async () => { - const rig = await fileRig({ fileBytes: 32 * MIB, leaseBytes: MIB, silenceMs: 330_000 }); - const asStranger = await rig.stranger(); - const routes = (call: typeof rig.call) => [ - () => call("/fixture.tar", { body: "x", method: "PUT" }), - () => call("/fixture.tar"), - () => call("/fixture.tar", { method: "DELETE" }), - () => call(""), - ]; - - const refused = await Promise.all(routes(asStranger).map((send) => send())); - const full = await rig.call("/big.bin", { body: new Uint8Array(MIB + 1), method: "PUT" }); - const feed = new TransformStream(); - const writer = feed.writable.getWriter(); - const running = rig.call("/running.bin", { - body: feed.readable, - duplex: "half", - method: "PUT", - } as RequestInit); - await writer.write(new Uint8Array(1)); - await poll(async () => (await rig.call("")).status).toBe(200); - await rig.gateway.dispatch("lease.release", { leaseId: rig.leaseId }, rig.holder); - await writer.write(new Uint8Array(1)); - await writer.close(); - const ended = await running; - const next = await rig.newLease(); - await rig.worker.worker.stop("test"); - await poll(async () => (await next.call("")).status).toBe(503); - - expect(refused.map((response) => response.status)).toEqual([404, 404, 404, 404]); - for (const response of refused) { - expect(await response.json()).toMatchObject({ - error: { code: "UNKNOWN_LEASE", leaseId: rig.leaseId }, - }); - } - expect(full.status).toBe(413); - const fullBody = (await full.json()) as { - error: { code: string; message: string; leaseId: string }; - }; - expect(fullBody.error).toMatchObject({ code: "LEASE_FILES_FULL", leaseId: rig.leaseId }); - expect(fullBody.error.message).toContain(`Lease ${rig.leaseId} holds`); - expect(ended.status).toBe(404); - const endedBody = (await ended.json()) as { - error: { code: string; message: string; leaseId: string }; - }; - expect(endedBody.error).toMatchObject({ code: "UNKNOWN_LEASE", leaseId: rig.leaseId }); - expect(endedBody.error.message).toBe(`Unknown lease: ${rig.leaseId}`); - expect(await (await next.call("")).json()).toMatchObject({ - error: { code: "WORKER_UNREACHABLE" }, - }); - }, 60_000); - - it("through a gateway, a file upload is cut only by 330 s of silence, not by 330 s in total", async () => { - const { call } = await fileRig(undefined, 1_000); - const pieces = 12; - let sent = 0; - const flowing = new ReadableStream({ - async pull(controller) { - await sleep(300); - controller.enqueue(new Uint8Array(KIB)); - sent += 1; - if (sent === pieces) controller.close(); - }, - }); - const silent = new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array(KIB)); - }, - }); - - const slow = await call("/slow.bin", { - body: flowing, - duplex: "half", - method: "PUT", - } as RequestInit); - const stalled = await settleWithin( - call("/stalled.bin", { body: silent, duplex: "half", method: "PUT" } as RequestInit), - 6_000, - ); - - expect(slow.status).toBe(201); - expect(await slow.json()).toEqual({ name: "slow.bin", size: pieces * KIB }); - expect(stalled).toMatchObject({ state: "fulfilled", value: { status: 408 } }); - const body = await (stalled as { value: Response }).value.json(); - expect(body).toEqual({ - error: { - code: "UPLOAD_STALLED", - message: "No bytes of stalled.bin arrived for 1 s", - name: "stalled.bin", - }, - }); - expect((await (await call("")).json()) as { files: { name: string }[] }).toEqual({ - files: [{ name: "slow.bin", size: pieces * KIB }], - }); - }, 30_000); - - it("a 20 MiB PUT body arrives at the target byte for byte, and a 20 MiB response body comes back byte for byte", async () => { - let received = ""; - behavior = async (request) => { - received = digestOf(await readParts(request.body)); - return new Response(patternStream(20 * MIB, 9)); - }; - const { relay, taps, workerId } = await startFleet(); - - const response = await relay.relay( - workerId, - buildCall({ body: patternStream(20 * MIB, 5), method: "PUT" }), - ); - const back = digestOf(await readParts(response.body)); - const dataBytes = taps - .flatMap((tap) => tap.binaries(TYPE.DATA)) - .reduce((total, envelope) => total + envelope.payload.length, 0); - - expect(received).toBe(patternDigest(20 * MIB, 5)); - expect(back).toBe(patternDigest(20 * MIB, 9)); - expect(dataBytes).toBeGreaterThanOrEqual(20 * MIB); - }, 120_000); - - it("an empty upload (0 bytes) reaches the target as an empty body, and a GET relays as before", async () => { - const lengths: number[] = []; - behavior = async (request) => { - lengths.push((await request.arrayBuffer()).byteLength); - return new Response("fine"); - }; - const { relay, workerId } = await startFleet(); - - const put = await relay.relay( - workerId, - buildCall({ body: patternStream(0, 1), method: "PUT" }), - ); - const get = await relay.relay(workerId, buildCall({})); - - expect(await put.text()).toBe("fine"); - expect(await get.text()).toBe("fine"); - expect(lengths).toEqual([0, 0]); - }, 30_000); - - it("while a 20 MiB relay runs, a lease.renew forwarded to the same worker answers", async () => { - const midway = gate(); - let finished = false; - behavior = async (request) => { - const reader = request.body?.getReader(); - let total = 0; - for (;;) { - const step = await reader?.read(); - if (step === undefined || step.done) break; - total += step.value.length; - if (total >= 5 * MIB) midway.open(); - } - finished = true; - return new Response("done"); - }; - const { gateway, relay, workerId } = await startFleet(); - const grant = await gateway.dispatch( - "lease.request", - { model: "Pixel-A", noWait: true, platform: "android" }, - agent(), - ); - - const pending = relay.relay( - workerId, - buildCall({ body: patternStream(20 * MIB, 5), method: "PUT" }), - ); - await midway.promise; - const renewed = await settleWithin( - gateway.dispatch("lease.renew", { leaseId: grant.lease.id }, agent()), - 10_000, - ); - const relayFinishedBeforeRenew = finished; - const response = await pending; - - expect(renewed).toMatchObject({ state: "fulfilled", value: { id: grant.lease.id } }); - expect(relayFinishedBeforeRenew).toBe(false); - expect(await response.text()).toBe("done"); - }, 120_000); - - it("a relay and a device.exec with output running at once on one worker each get only their own pushes", async () => { - const execStarted = gate(); - behavior = () => { - const encoder = new TextEncoder(); - return Promise.resolve( - new Response( - new ReadableStream({ - async pull(controller) { - controller.enqueue(encoder.encode("r1")); - await execStarted.promise; - controller.enqueue(encoder.encode("r2")); - controller.close(); - }, - }), - ), - ); - }; - const { gateway, relay, workerId } = await startFleet({ - args: ["-e", 'process.stdout.write("e1");setTimeout(()=>process.stdout.write("e2"),300);'], - command: process.execPath, - env: {}, - }); - const grant = await gateway.dispatch( - "lease.request", - { model: "Pixel-A", noWait: true, platform: "android" }, - agent(), - ); - const output: string[] = []; - - const exec = gateway.dispatch( - "device.exec", - { args: [], leaseId: grant.lease.id, tool: "adb" }, - agent({ - onOutput: (_stream, chunk) => { - output.push(chunk); - execStarted.open(); - }, - }), - ); - const response = await relay.relay(workerId, buildCall({})); - const body = await response.text(); - await exec; - - expect(body).toBe("r1r2"); - expect(output.join("")).toBe("e1e2"); - }, 30_000); - - it("a target that answers 401 without reading its 20 MiB upload: the caller gets the 401 and its body, and the gateway stops reading the caller's body", async () => { - behavior = async () => { - // why: the 401 comes after the upload has begun, so the gateway is reading the caller's body when the target answers - await sleep(300); - return new Response("nope", { status: 401 }); - }; - const { relay } = await startPair(); - let pulls = 0; - let cancelled = false; - const body = new ReadableStream({ - async pull(controller) { - pulls += 1; - await sleep(5); - controller.enqueue(patternBytes(256 * KIB, 4)); - if (pulls >= 100) controller.close(); - }, - cancel() { - cancelled = true; - }, - }); - - const response = await relay.relay(WORKER_ID, buildCall({ body, method: "PUT" })); - const text = await response.text(); - await sleep(300); - const pullsAfterAnswer = pulls; - await sleep(300); - - expect(response.status).toBe(401); - expect(text).toBe("nope"); - expect(pulls).toBe(pullsAfterAnswer); - expect(pulls).toBeLessThan(100); - expect(cancelled).toBe(true); - }, 30_000); - - it("a response the target streams in parts reaches the gateway caller part by part, before the target's body ends", async () => { - const second = gate(); - behavior = () => { - const encoder = new TextEncoder(); - let step = 0; - return Promise.resolve( - new Response( - new ReadableStream({ - async pull(controller) { - step += 1; - if (step === 1) controller.enqueue(encoder.encode("part-1")); - else { - await second.promise; - controller.enqueue(encoder.encode("part-2")); - controller.close(); - } - }, - }), - ), - ); - }; - const { relay } = await startPair(); - - const response = await relay.relay(WORKER_ID, buildCall({})); - const reader = response.body?.getReader(); - const first = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body")), 3_000); - second.open(); - const rest = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body")), 3_000); - - expect( - first.state === "fulfilled" ? new TextDecoder().decode(first.value.value) : first.state, - ).toBe("part-1"); - expect( - rest.state === "fulfilled" ? new TextDecoder().decode(rest.value.value) : rest.state, - ).toBe("part-2"); - }, 30_000); - - it.each([ - { method: "GET", status: 204 }, - { method: "GET", status: 205 }, - { method: "GET", status: 304 }, - { method: "HEAD", status: 200 }, - ])( - "a $status answer to a $method reaches the caller with a null body", - async ({ method, status }) => { - behavior = () => Promise.resolve(new Response(method === "HEAD" ? "x" : null, { status })); - const { relay } = await startPair(); - - const response = await relay.relay(WORKER_ID, buildCall({ method })); - - expect(response.status).toBe(status); - expect(response.body).toBeNull(); - }, - ); - - it("a caller that reads nothing slows only its own transfer: another relay to the same worker completes while the first's response is unread, and the target has not sent all 80 MiB", async () => { - let sent = 0; - behavior = (request) => { - if (new URL(request.url).searchParams.get("slow") === null) { - return Promise.resolve(new Response("quick")); - } - return Promise.resolve( - new Response( - new ReadableStream({ - async pull(controller) { - await sleep(5); - controller.enqueue(patternBytes(MIB, 6, sent)); - sent += MIB; - if (sent >= 80 * MIB) controller.close(); - }, - }), - ), - ); - }; - const { relay } = await startPair(); - - const slow = await relay.relay( - WORKER_ID, - buildCall({ url: "http://gateway.test:4321/agent-device/rpc?slow=1" }), - ); - await sleep(500); - const quick = await settleWithin( - relay.relay(WORKER_ID, buildCall({})).then((response) => response.text()), - 5_000, - ); - - expect(quick).toEqual({ state: "fulfilled", value: "quick" }); - expect(sent).toBeLessThan(80 * MIB); - expect(slow.status).toBe(200); - }, 60_000); - - it("aborting the caller's request signal before the head comes fails the call; after it, it errors the caller's body stream; either way the target's request signal aborts and relay.request answers { cancelled: true } with no push after the cancel", async () => { - let signal: AbortSignal | undefined; - behavior = async (request) => { - signal = request.signal; - await aborted(request.signal); - return new Response("late"); - }; - const before = await startPair(); - const controller = new AbortController(); - const pendingBefore = before.relay.relay(WORKER_ID, buildCall({ signal: controller.signal })); - const settledBefore = settleWithin(pendingBefore); - await poll(() => signal !== undefined).toBe(true); - controller.abort(new Error("caller left before the head")); - const failure = await settledBefore; - await poll(() => signal?.aborted).toBe(true); - await poll(() => - before.spy.events.filter((event) => event.startsWith("request-answered")), - ).toEqual(['request-answered:{"cancelled":true}']); - - signal = undefined; - behavior = (request) => { - signal = request.signal; - return Promise.resolve( - new Response( - new ReadableStream({ - async pull(streamController) { - await sleep(10); - streamController.enqueue(patternBytes(512, 1)); - }, - }), - ), - ); - }; - const after = await startPair(); - const secondController = new AbortController(); - const response = await after.relay.relay( - WORKER_ID, - buildCall({ signal: secondController.signal }), - ); - const reader = response.body?.getReader(); - await reader?.read(); - secondController.abort(new Error("caller left after the head")); - const bodyFailure = await settleWithin( - (async () => { - for (;;) { - const step = await reader?.read(); - if (step === undefined || step.done) return; - } - })(), - ); - await poll(() => signal?.aborted).toBe(true); - await poll(() => - after.spy.events.filter((event) => event.startsWith("request-answered")), - ).toEqual(['request-answered:{"cancelled":true}']); - await sleep(150); - const eventsAfterCancel = after.spy.events.slice( - after.spy.events.indexOf("cancel-answered") + 1, - ); - - expect(failure).toMatchObject({ message: "caller left before the head", state: "rejected" }); - expect(bodyFailure).toMatchObject({ message: "caller left after the head", state: "rejected" }); - expect(after.spy.events).toContain("cancel-answered"); - expect(eventsAfterCancel.filter((event) => event.startsWith("push:"))).toEqual([]); - }, 30_000); - - it("a caller's request body that errors mid-upload aborts the target's request signal", async () => { - let signal: AbortSignal | undefined; - behavior = async (request) => { - signal = request.signal; - await readParts(request.body).catch(() => undefined); - await aborted(request.signal); - return new Response("late"); - }; - const { relay } = await startPair(); - let step = 0; - const body = new ReadableStream({ - pull(controller) { - step += 1; - if (step === 1) controller.enqueue(patternBytes(1024, 8)); - else controller.error(new Error("upload broke")); - }, - }); - - const outcome = await settleWithin(relay.relay(WORKER_ID, buildCall({ body, method: "PUT" }))); - - expect(outcome).toMatchObject({ message: "upload broke", state: "rejected" }); - await poll(() => signal?.aborted).toBe(true); - }, 30_000); - - it("a target that never answers is cancelled after a 200 ms deadline, and the caller gets WORKER_UNREACHABLE with message `worker did not finish the relayed request within 200 ms`; one that stalls after its head, under a 10 s deadline and a 200 ms idle bound, gets its body stream errored with WORKER_UNREACHABLE `worker sent nothing on the relayed response for 200 ms`", async () => { - let signal: AbortSignal | undefined; - behavior = async (request) => { - signal = request.signal; - await aborted(request.signal); - return new Response("late"); - }; - const never = await startPair({ deadlineMs: 200 }); - const noHead = await settleWithin(never.relay.relay(WORKER_ID, buildCall({}))); - await poll(() => signal?.aborted).toBe(true); - - behavior = () => - Promise.resolve( - new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("x")); - }, - }), - ), - ); - const stalling = await startPair({ deadlineMs: 10_000, idleMs: 200 }); - const response = await stalling.relay.relay(WORKER_ID, buildCall({})); - const reader = response.body?.getReader(); - await reader?.read(); - const stalled = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body"))); - - const message = `worker ${WORKER_ID} did not finish the relayed request within 200 ms`; - expect(noHead).toMatchObject({ code: "WORKER_UNREACHABLE", message, state: "rejected" }); - expect(stalled).toMatchObject({ - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} sent nothing on the relayed response for 200 ms`, - state: "rejected", - }); - }, 30_000); - - it("closing the uplink mid-relay fails the caller with WORKER_UNREACHABLE, or errors its body stream after the head, and aborts the target's request signal", async () => { - let signal: AbortSignal | undefined; - behavior = async (request) => { - signal = request.signal; - await aborted(request.signal); - return new Response("late"); - }; - const before = await startPair(); - const noHead = settleWithin(before.relay.relay(WORKER_ID, buildCall({}))); - await poll(() => signal !== undefined).toBe(true); - await before.pair.gatewayEnd.close(); - const noHeadOutcome = await noHead; - await poll(() => signal?.aborted).toBe(true); - - signal = undefined; - behavior = (request) => { - signal = request.signal; - return Promise.resolve( - new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode("x")); - }, - }), - ), - ); - }; - const after = await startPair(); - const response = await after.relay.relay(WORKER_ID, buildCall({})); - const reader = response.body?.getReader(); - await reader?.read(); - await after.pair.gatewayEnd.close(); - const bodyOutcome = await settleWithin(reader?.read() ?? Promise.reject(new Error("no body"))); - - expect(noHeadOutcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - expect(bodyOutcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - await poll(() => signal?.aborted).toBe(true); - }, 30_000); - - it("closing the uplink ends every two-way relay: the gateway side gets WORKER_UNREACHABLE and the target's signal aborts", async () => { - const signals: AbortSignal[] = []; - twoWayBehavior = (_request, _tenant, _leaseId, pipe) => { - signals.push(pipe.signal); - return Promise.resolve(new Response(null, { status: 200 })); - }; - const { gatewayEnd, relay } = await startPair(); - const open = async () => { - const opened = (await relay.relayTwoWay(WORKER_ID, buildCall({}))) as TwoWayRelay; - return { reader: opened.readable.getReader(), writer: opened.writable.getWriter() }; - }; - const first = await open(); - const second = await open(); - await poll(() => signals.map((signal) => signal.aborted)).toEqual([false, false]); - - await gatewayEnd.close(); - const outcomes = await Promise.all([ - settleWithin(first.reader.read()), - settleWithin(first.writer.write(new Uint8Array(1))), - settleWithin(second.reader.read()), - settleWithin(second.writer.write(new Uint8Array(1))), - ]); - - for (const settled of outcomes) { - expect(settled).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - } - await poll(() => signals.map((signal) => signal.aborted)).toEqual([true, true]); - }, 30_000); - - it("a relay to a worker with no uplink fails with WORKER_UNREACHABLE and the target sees nothing", async () => { - const { relay } = await startPair(); - - const outcome = await settleWithin(relay.relay("wrk_nobody", buildCall({}))); - await sleep(100); - - expect(outcome).toMatchObject({ code: "WORKER_UNREACHABLE", state: "rejected" }); - expect(targetCalls).toEqual([]); - }); - - it("a relay naming a target nobody registered fails with RELAY_TARGET_MISSING", async () => { - const { relay } = await startPair(); - - const outcome = await settleWithin(relay.relay(WORKER_ID, buildCall({ target: "nobody" }))); - - expect(outcome).toMatchObject({ code: "RELAY_TARGET_MISSING", state: "rejected" }); - expect(targetCalls).toEqual([]); - }); - - it("the caller's authorization, x-agent-device-token, connection and transfer-encoding headers never reach the target, and a target's content-length and connection headers never reach the caller", async () => { - behavior = () => - Promise.resolve( - new Response("hello", { - headers: { connection: "close", "content-length": "5", "x-kept": "yes" }, - }), - ); - const { relay } = await startPair(); - - const response = await relay.relay( - WORKER_ID, - buildCall({ - headers: { - authorization: "Bearer gateway-secret", - connection: "keep-alive", - "transfer-encoding": "chunked", - "x-agent-device-token": "device-secret", - "x-keep-me": "yes", - }, - }), - ); - await response.text(); - - const seen = targetCalls[0]?.headers; - expect(seen?.get("x-keep-me")).toBe("yes"); - expect(seen?.get("authorization")).toBeNull(); - expect(seen?.get("x-agent-device-token")).toBeNull(); - expect(seen?.get("connection")).toBeNull(); - expect(seen?.get("transfer-encoding")).toBeNull(); - expect(response.headers.get("x-kept")).toBe("yes"); - expect(response.headers.get("content-length")).toBeNull(); - expect(response.headers.get("connection")).toBeNull(); - }); - - it("a 65th relay to one worker starts once one of 64 ends, and not before", async () => { - const releases: (() => void)[] = []; - behavior = async () => { - const held = gate(); - releases.push(held.open); - await held.promise; - return new Response("ok"); - }; - const { opened, relay } = await startPair(); - - const calls = Array.from({ length: 65 }, () => - settleWithin(relay.relay(WORKER_ID, buildCall({})), 20_000), - ); - await poll(() => targetCalls.length, 10_000).toBe(64); - await sleep(200); - const startedBeforeRelease = targetCalls.length; - releases[0]?.(); - await poll(() => targetCalls.length, 10_000).toBe(65); - for (const release of releases) release(); - const outcomes = await Promise.all(calls); - - expect(startedBeforeRelease).toBe(64); - expect(opened()).toBe(65); - expect(outcomes.map((outcome) => outcome.state)).toEqual( - Array.from({ length: 65 }, () => "fulfilled"), - ); - }, 60_000); - - it("a 65th relay whose 200 ms deadline passes first fails WORKER_UNREACHABLE with the deadline message, and its target is never called", async () => { - const release = gate(); - behavior = async () => { - await release.promise; - return new Response("ok"); - }; - const { makeRelay } = await startPair(); - const long = makeRelay(); - const short = makeRelay({ deadlineMs: 200 }); - const holders = Array.from({ length: 64 }, () => - settleWithin(long.relay(WORKER_ID, buildCall({})), 20_000), - ); - await poll(() => targetCalls.length, 10_000).toBe(64); - - const outcome = await settleWithin(short.relay(WORKER_ID, buildCall({}))); - release.open(); - await Promise.all(holders); - - expect(outcome).toEqual({ - code: "WORKER_UNREACHABLE", - message: `worker ${WORKER_ID} did not finish the relayed request within 200 ms`, - state: "rejected", - }); - expect(targetCalls).toHaveLength(64); - }, 60_000); - - it("a 65th relay whose caller aborts while it waits for a channel fails at once with the abort reason", async () => { - const release = gate(); - behavior = async () => { - await release.promise; - return new Response("ok"); - }; - const { relay } = await startPair(); - const holders = Array.from({ length: 64 }, () => - settleWithin(relay.relay(WORKER_ID, buildCall({})), 20_000), - ); - await poll(() => targetCalls.length, 10_000).toBe(64); - const controller = new AbortController(); - const waiting = settleWithin( - relay.relay(WORKER_ID, buildCall({ signal: controller.signal })), - 3_000, - ); - await sleep(100); - - controller.abort(new Error("caller left while waiting")); - const outcome = await waiting; - release.open(); - await Promise.all(holders); - - expect(outcome).toMatchObject({ message: "caller left while waiting", state: "rejected" }); - expect(targetCalls).toHaveLength(64); - }, 60_000); - - const leaving: readonly { - readonly name: string; - readonly run: ( - relay: GatewayRelay, - makeRelay: (o?: { deadlineMs?: number }) => GatewayRelay, - ) => Promise; - }[] = [ - { - name: "were cancelled", - run: async (relay) => { - const controller = new AbortController(); - const pending = settleWithin( - relay.relay(WORKER_ID, buildCall({ signal: controller.signal })), - ); - await sleep(20); - controller.abort(new Error("cancelled")); - return pending; - }, - }, - { - name: "timed out", - run: (_relay, makeRelay) => - settleWithin(makeRelay({ deadlineMs: 100 }).relay(WORKER_ID, buildCall({}))), - }, - { - name: "failed with a missing target", - run: (relay) => settleWithin(relay.relay(WORKER_ID, buildCall({ target: "nobody" }))), - }, - { - name: "had a target that answered without reading its upload", - run: async (relay) => { - const response = await relay.relay( - WORKER_ID, - buildCall({ body: patternStream(MIB, 3), method: "PUT" }), - ); - return settleWithin(response.text()); - }, - }, - ]; - - it.each(leaving)( - "after 64 relays to one worker that $name, a 65th starts at once", - async ({ run }) => { - const held = gate(); - behavior = async (request) => { - if (request.headers.get("x-marker") === "last") { - await held.promise; - return new Response("last"); - } - if (request.method === "PUT") return new Response("not reading"); - await aborted(request.signal); - return new Response("late"); - }; - const { makeRelay, opened, relay } = await startPair(); - - for (let batch = 0; batch < 4; batch += 1) { - await Promise.all(Array.from({ length: 16 }, () => run(relay, makeRelay))); - } - const last = settleWithin( - relay.relay(WORKER_ID, buildCall({ headers: { "x-marker": "last" } })), - 10_000, - ); - await poll(() => targetCalls.some((call) => call.headers.get("x-marker") === "last")).toBe( - true, - ); - const channels = opened(); - held.open(); - await last; - - expect(channels).toBe(65); - }, - 60_000, - ); - - it("with the worker's transport held so the {} answer overtakes a 20 MiB response, the caller still gets every byte, and its body ends only at the channel's end", async () => { - behavior = () => Promise.resolve(new Response(patternStream(20 * MIB, 9))); - const { pair, relay, spy } = await startPair(); - pair.workerEnd.hold(); - - const pending = relay.relay(WORKER_ID, buildCall({})); - await sleep(300); - pair.workerEnd.release(); - const response = await pending; - const digest = digestOf(await readParts(response.body)); - - expect(digest).toBe(patternDigest(20 * MIB, 9)); - expect(spy.events).toContain("request-answered:{}"); - }, 60_000); }); diff --git a/src/daemon/test-relay-gateway.ts b/src/daemon/test-relay-gateway.ts new file mode 100644 index 000000000..33e7713aa --- /dev/null +++ b/src/daemon/test-relay-gateway.ts @@ -0,0 +1,76 @@ +import { type GatewayWorkerClient } from "../gateway/index.js"; + +export type CallInit = { + readonly target?: string; + readonly method?: string; + readonly url?: string; + readonly headers?: Record; + readonly body?: ReadableStream; + readonly signal?: AbortSignal; + readonly tenant?: string; + readonly leaseId?: string; +}; + +export const WORKER_ID = "wrk_a"; + +export function buildCall(init: CallInit) { + const request = new Request(init.url ?? "http://gateway.test:4321/agent-device/rpc?x=1&y=2", { + method: init.method ?? "GET", + ...(init.headers === undefined ? {} : { headers: init.headers }), + ...(init.body === undefined ? {} : { body: init.body, duplex: "half" }), + ...(init.signal === undefined ? {} : { signal: init.signal }), + } as RequestInit); + return { + request, + target: init.target ?? "echo", + tenant: init.tenant ?? "gw.g1.alice", + ...(init.leaseId === undefined ? {} : { leaseId: init.leaseId }), + }; +} + +export interface ClientSpy { + readonly events: string[]; +} + +export function spyOn(client: GatewayWorkerClient): { + client: GatewayWorkerClient; + spy: ClientSpy; +} { + const spy: ClientSpy = { events: [] }; + const wrapHooks = (value: unknown): unknown => { + if (typeof value !== "object" || value === null) return value; + const hooks = { ...(value as Record) }; + const original = hooks.onRelayHead; + if (typeof original === "function") { + hooks.onRelayHead = (...args: unknown[]) => { + spy.events.push("push:onRelayHead"); + return (original as (...rest: unknown[]) => unknown)(...args); + }; + } + return hooks; + }; + const wrapped = new Proxy(client, { + get(target, property) { + const original: unknown = Reflect.get(target, property, target); + if (typeof original !== "function") return original; + const method = original as (...args: unknown[]) => Promise; + if (property === "relayRequest") { + return async (...args: unknown[]) => { + const value = await method.apply(target, args.map(wrapHooks)); + spy.events.push(`request-answered:${JSON.stringify(value)}`); + return value; + }; + } + if (property === "relayCancel") { + return async (...args: unknown[]) => { + spy.events.push("cancel-sent"); + const value = await method.apply(target, args); + spy.events.push("cancel-answered"); + return value; + }; + } + return method.bind(target); + }, + }); + return { client: wrapped, spy }; +} From 8971d22eab2b62fb027cf4cced2b085ba34ac20b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:23:57 +0100 Subject: [PATCH 38/83] test(leasing): split lease-acquisition-coordinator.test.ts by topic --- ...cquisition-coordinator-device-mode.test.ts | 198 ++++ ...quisition-coordinator-failed-boots.test.ts | 315 +++++ ...sition-coordinator-reported-events.test.ts | 205 ++++ ...sition-coordinator-stored-requests.test.ts | 174 +++ ...-coordinator-waiting-and-baselines.test.ts | 140 +++ .../lease-acquisition-coordinator.test.ts | 1041 +---------------- src/leasing/test-lease-acquisition-commits.ts | 34 + 7 files changed, 1069 insertions(+), 1038 deletions(-) create mode 100644 src/leasing/lease-acquisition-coordinator-device-mode.test.ts create mode 100644 src/leasing/lease-acquisition-coordinator-failed-boots.test.ts create mode 100644 src/leasing/lease-acquisition-coordinator-reported-events.test.ts create mode 100644 src/leasing/lease-acquisition-coordinator-stored-requests.test.ts create mode 100644 src/leasing/lease-acquisition-coordinator-waiting-and-baselines.test.ts create mode 100644 src/leasing/test-lease-acquisition-commits.ts diff --git a/src/leasing/lease-acquisition-coordinator-device-mode.test.ts b/src/leasing/lease-acquisition-coordinator-device-mode.test.ts new file mode 100644 index 000000000..f3e387e03 --- /dev/null +++ b/src/leasing/lease-acquisition-coordinator-device-mode.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { type DeviceMode, type VirtualSpec, specMode } from "../core/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { request, createHarness, seedReady } from "./test-lease-acquisition.js"; + +describe("LeaseAcquisitionCoordinator", () => { + describe("device mode", () => { + /** A driver that slims 26.5 and cannot slim 17.5, like iOS on either side of 18.5. */ + function slimmingDriver(options: { readonly mode?: DeviceMode } = {}): FakeDriver { + return new FakeDriver({ + availableOsVersions: ["17.5", "26.5"], + clock: new FakeClock(1_000), + platform: "ios", + slimmableOsVersions: ["26.5"], + ...options, + }); + } + + const owner = (id: string) => ({ ownerId: id, requesterId: id }); + + it.each([ + ["full", "no mode", undefined, "full"], + ["slim", "no mode", undefined, "slim"], + ["full", "mode slim", "slim", "slim"], + ["slim", "mode full", "full", "full"], + ] as const)( + "on a worker whose default is %s, a request with %s gets that device mode", + async (defaultMode, _label, mode, planned) => { + const harness = await createHarness({ + defaultModes: { ios: defaultMode }, + drivers: [slimmingDriver()], + }); + + const granted = await harness.coordinator.request( + { ...request, ...(mode === undefined ? {} : { mode }) }, + owner("agent"), + ); + + expect(specMode(granted.device.spec)).toBe(planned); + expect(granted.device.mode).toBe(planned); + }, + ); + + it.each([ + ["slim", "slim", true], + ["slim", "full", false], + ["full", "full", true], + ["full", "slim", false], + ] as const)( + "on a worker whose default is %s, a device whose pool mode is %s reports servesDefaultMode %s", + async (defaultMode, poolMode, served) => { + const harness = await createHarness({ + defaultModes: { ios: defaultMode }, + drivers: [slimmingDriver()], + }); + const spec = { + physical: false as const, + model: "iPhone 17", + osVersion: "26.5", + platform: "ios" as const, + ...(poolMode === "slim" ? { mode: "slim" as const } : {}), + }; + + expect(harness.coordinator.servesDefaultMode(spec)).toBe(served); + }, + ); + + it("compares a device's pool mode with its own platform's default, full when that platform has none", async () => { + const harness = await createHarness({ + defaultModes: { ios: "slim" }, + drivers: [slimmingDriver()], + }); + const android = { + physical: false as const, + model: "Pixel 9", + osVersion: "35", + platform: "android" as const, + }; + + expect(harness.coordinator.servesDefaultMode(android)).toBe(true); + expect(harness.coordinator.servesDefaultMode({ ...android, mode: "slim" })).toBe(false); + }); + + it.each(["ios", "android"] as const)( + "reports servesDefaultMode false for a physical spec on %s, though it names no mode and the platform's default is full", + async (platform) => { + const harness = await createHarness({ defaultModes: { [platform]: "full" } }); + + expect( + harness.coordinator.servesDefaultMode({ + class: "phone", + model: "Some Phone", + osVersion: "1", + physical: true, + platform, + }), + ).toBe(false); + }, + ); + + it("passes the resolved mode to resolveSpec, the default filled in for a request that named none", async () => { + const driver = slimmingDriver(); + const harness = await createHarness({ defaultModes: { ios: "slim" }, drivers: [driver] }); + + await harness.coordinator.request(request, owner("agent")); + + const resolved = driver.calls.find((call) => call.operation === "resolveSpec"); + expect(resolved?.arguments[0]).toMatchObject({ mode: "slim" }); + }); + + it("never plans a full request onto a slim spec, even when the driver's resolveSpec returns one", async () => { + class AlwaysSlimDriver extends FakeDriver { + override async resolveSpec( + ...args: Parameters + ): Promise { + return { ...(await super.resolveSpec(...args)), mode: "slim" }; + } + } + const driver = new AlwaysSlimDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }); + const harness = await createHarness({ drivers: [driver] }); + + const granted = await harness.coordinator.request( + { ...request, mode: "full" }, + owner("agent"), + ); + + expect(granted.device.spec).not.toHaveProperty("mode"); + const provisioned = driver.calls.find((call) => call.operation === "provision"); + expect(provisioned?.arguments[0]).not.toHaveProperty("mode"); + }); + + it.each([ + { name: "full request an idle slim", seeded: "slim", requested: "full" }, + { name: "slim request an idle full", seeded: "full", requested: "slim" }, + ] as const)( + "does not give a $name device of the same model and runtime", + async ({ seeded, requested }) => { + const harness = await createHarness({ drivers: [slimmingDriver()], maxDevices: 2 }); + const idle = await seedReady( + harness, + seeded === "slim" ? { ...request, mode: "slim" } : request, + ); + + const granted = await harness.coordinator.request( + { ...request, mode: requested }, + owner("agent"), + ); + + expect(granted.device.id).not.toBe(idle.id); + expect(granted.device.mode).toBe(requested); + }, + ); + + it("resolves a slim request on a runtime the driver cannot slim to the full spec, reusing an idle full device", async () => { + const oldRuntime = { ...request, osVersion: "17.5" }; + const harness = await createHarness({ drivers: [slimmingDriver()] }); + const full = await seedReady(harness, oldRuntime); + + const granted = await harness.coordinator.request( + { ...oldRuntime, mode: "slim" }, + owner("agent"), + ); + + expect(granted.device.id).toBe(full.id); + expect(granted.device.spec).toEqual(oldRuntime); + expect(granted.device.mode).toBe("full"); + }); + + it("grants a slim request against a driver that knows nothing of modes a full device", async () => { + const harness = await createHarness(); + + const granted = await harness.coordinator.request( + { ...request, mode: "slim" }, + owner("agent"), + ); + + expect(granted.device.spec).toEqual(request); + expect(granted.device.mode).toBe("full"); + }); + + it("grants a slim-spec device whose slim pass failed as full, and keeps it in the slim pool", async () => { + const harness = await createHarness({ drivers: [slimmingDriver({ mode: "full" })] }); + + const granted = await harness.coordinator.request( + { ...request, mode: "slim" }, + owner("agent"), + ); + + expect(granted.device.mode).toBe("full"); + expect(granted.device.spec).toEqual({ ...request, mode: "slim" }); + }); + }); +}); diff --git a/src/leasing/lease-acquisition-coordinator-failed-boots.test.ts b/src/leasing/lease-acquisition-coordinator-failed-boots.test.ts new file mode 100644 index 000000000..b9a428d6e --- /dev/null +++ b/src/leasing/lease-acquisition-coordinator-failed-boots.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it, vi } from "vitest"; +import { FakeClock, JsonLinesLogger, type Logger, MemoryLogSink } from "../ports/index.js"; +import { capacityDevice, capacityDevices, DriverCrashError } from "../core/index.js"; +import { + request, + createHarness, + seedReady, + settle, + seedShutdown, +} from "./test-lease-acquisition.js"; + +function capturingLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { + const sink = new MemoryLogSink(); + return { logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), sink }; +} + +describe("LeaseAcquisitionCoordinator", () => { + it("A failed eviction shutdown logs the device id and the error, and the waiter is deferred as before.", async () => { + const { logger, sink } = capturingLogger(); + // Two devices allowed but only one running: a running device of another spec is + // evicted by shutdown (not deleted) to make room for the request. + const harness = await createHarness({ logger, maxDevices: 2 }); + const victim = await seedReady(harness, { ...request, model: "iPhone SE" }); + harness.driver.failOn("shutdown", 1, new DriverCrashError("shutdown wedged")); + + let settled = false; + const acquisition = harness.coordinator.request(request, { + ownerId: "evictor", + requesterId: "evictor", + }); + void acquisition.then( + () => (settled = true), + () => (settled = true), + ); + await settle(); + + expect(sink.records).toEqual([ + expect.objectContaining({ + level: "warn", + module: "daemon.lease-acquisition-coordinator", + fields: { + deviceId: victim.id, + error: "DriverCrashError: shutdown wedged", + requesterId: "evictor", + step: "shutdown", + }, + }), + ]); + // Deferred: back in the queue, still pending, and the victim untouched. + expect(settled).toBe(false); + expect(harness.coordinator.queueDepth).toBe(1); + expect(harness.registry.snapshot.devices.find((device) => device.id === victim.id)?.state).toBe( + "ready", + ); + await harness.coordinator.cancelPending("evictor"); + await expect(acquisition).rejects.toMatchObject({ name: "RequestCancelledError" }); + }); + + it("A failed eviction delete logs the device id and the error.", async () => { + const { logger, sink } = capturingLogger(); + // One device allowed: a managed device of another spec is deleted to make room. + const harness = await createHarness({ logger, maxDevices: 1 }); + const victim = await seedShutdown(harness, { ...request, model: "iPhone SE" }); + harness.driver.failOn("destroy", 1, new DriverCrashError("simctl delete refused")); + + const acquisition = harness.coordinator.request(request, { + ownerId: "evictor", + requesterId: "evictor", + }); + await settle(); + + expect(sink.records).toEqual([ + expect.objectContaining({ + level: "warn", + fields: { + deviceId: victim.id, + error: "DriverCrashError: simctl delete refused", + requesterId: "evictor", + step: "delete", + }, + }), + ]); + await harness.coordinator.cancelPending("evictor"); + await expect(acquisition).rejects.toMatchObject({ name: "RequestCancelledError" }); + }); + + it("A failed destroy after a shut-down device fails to boot for a waiter logs the device id and the error.", async () => { + const { logger, sink } = capturingLogger(); + const harness = await createHarness({ logger }); + const shutdown = await seedShutdown(harness); + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + harness.driver.failOn("destroy", 1, new DriverCrashError("simulator would not die")); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ + name: "BootTimeoutError", + message: expect.stringContaining("simulator never booted"), + }); + + expect(sink.records.filter((record) => record.fields?.["step"] === "destroy")).toEqual([ + expect.objectContaining({ + level: "warn", + fields: { + deviceId: shutdown.id, + error: "DriverCrashError: simulator would not die", + requesterId: "booter", + step: "destroy", + }, + }), + ]); + }); + + it("A shut-down device that fails to boot for a waiter and then fails to be destroyed is quarantined and counts once against running capacity.", async () => { + // why: two running slots — the failed device's and the one a second request needs. + const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); + const shutdown = await seedShutdown(harness); + // why: call 1 is seedShutdown's own boot; call 2 is the waiter's. + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + harness.driver.failOn("destroy", 1, new DriverCrashError("simulator would not die")); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ name: "BootTimeoutError" }); + await settle(); + + expect( + harness.registry.snapshot.devices.find((device) => device.id === shutdown.id)?.state, + ).toBe("quarantined"); + // why: `quarantined` already counts as running — a reservation still held would count + // this device twice and refuse the second slot. + await expect( + harness.coordinator.request( + { ...request, model: "iPhone SE" }, + { noWait: true, ownerId: "second", requesterId: "second" }, + ), + ).resolves.toMatchObject({ device: { spec: { model: "iPhone SE" }, state: "leased" } }); + }); + + it("grants a request that arrives while a failed boot's device is being destroyed a device of its own, without waiting for the destroy.", async () => { + const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); + const shutdownId = (await seedShutdown(harness)).id; + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + let failDestroy: (error: Error) => void = () => undefined; + const realDestroy = harness.driver.destroy.bind(harness.driver); + let destroys = 0; + vi.spyOn(harness.driver, "destroy").mockImplementation(async (device) => { + destroys += 1; + if (destroys > 1) return realDestroy(device); + return new Promise((_resolve, reject) => { + failDestroy = reject; + }); + }); + + const booter = harness.coordinator.request(request, { + ownerId: "booter", + requesterId: "booter", + }); + const booterOutcome = booter.catch((error: unknown) => error); + await settle(); + const second = harness.coordinator.request(request, { ownerId: "b", requesterId: "b" }); + let secondGranted = false; + void second.then(() => { + secondGranted = true; + }); + await settle(); + + expect(secondGranted).toBe(true); + expect((await second).device.id).not.toBe(shutdownId); + + failDestroy(new DriverCrashError("simulator would not die")); + await booterOutcome; + }); + + it("A device whose boot for a waiter failed is deleted by the lease engine, and its slot is freed.", async () => { + const harness = await createHarness(); + const shutdown = await seedShutdown(harness); + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ name: "BootTimeoutError" }); + + expect(harness.bus.replay().filter((event) => event.event === "device.deleted")).toEqual([ + expect.objectContaining({ + payload: expect.objectContaining({ deviceId: shutdown.id, initiator: "lease-engine" }), + }), + ]); + await expect( + harness.coordinator.request(request, { + noWait: true, + ownerId: "second", + requesterId: "second", + }), + ).resolves.toMatchObject({ lease: { requesterId: "second" } }); + }); + + it("A failed boot whose device the destroy can no longer claim frees the waiter's running slot and leaves no fence.", async () => { + const harness = await createHarness(); + const shutdown = await seedShutdown(harness); + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + const realMakeReady = harness.driver.makeReady.bind(harness.driver); + let failedBoots = 0; + vi.spyOn(harness.driver, "makeReady").mockImplementation(async (...args) => { + try { + return await realMakeReady(...args); + } finally { + failedBoots += 1; + // The record leaves `shutdown` while the boot fails, so the destroy finds nothing to claim. + if (failedBoots === 1) { + await harness.registry.transitionDevice(shutdown.id, "deleted", { + event: "device.deleted", + payload: { deviceId: shutdown.id, initiator: "test" }, + }); + } + } + }); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ name: "BootTimeoutError" }); + + await expect( + harness.coordinator.request(request, { + noWait: true, + ownerId: "second", + requesterId: "second", + }), + ).resolves.toMatchObject({ lease: { requesterId: "second" } }); + }); + + it.each([ + ["answers that another operation owns the device", async (): Promise => false], + ["throws", async (): Promise => Promise.reject(new Error("state write failed"))], + ] as const)( + "A failed boot whose destroy throws frees the waiter's running slot and leaves no fence, even when quarantine %s.", + async (_label, enterFromStalledTransition) => { + const harness = await createHarness({ + lifecycle: (lifecycle) => ({ + bootForLease: (device, claim) => lifecycle.bootForLease(device, claim), + rebootForLease: (...args) => lifecycle.rebootForLease(...args), + dispose: (...args) => lifecycle.dispose(...args), + shutdown: (...args) => lifecycle.shutdown(...args), + destroy: () => Promise.reject(new DriverCrashError("simulator would not delete")), + }), + quarantine: { enterFromStalledTransition }, + }); + const shutdown = await seedShutdown(harness); + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ name: "BootTimeoutError" }); + await settle(); + + await expect( + harness.coordinator.request(request, { ownerId: "second", requesterId: "second" }), + ).resolves.toMatchObject({ device: { id: shutdown.id, state: "leased" } }); + }, + ); + + it("releases the running slot reserved to boot a shut-down device for a waiter when the boot fails and another boot claims the device before it is destroyed", async () => { + // The warm pool's boot reserves its own slot and claims the device in one decision. Here it + // lands after the failed boot has released its claim and before the destroy claims the device. + let raced = false; + let warmBoot: { release(): void } | undefined; + const harness = await createHarness({ + maxDevices: 2, + maxRunning: 2, + lifecycle: (lifecycle, { capacity, claims, registry }) => ({ + bootForLease: (device, claim) => lifecycle.bootForLease(device, claim), + rebootForLease: (...args) => lifecycle.rebootForLease(...args), + dispose: (...args) => lifecycle.dispose(...args), + shutdown: (...args) => lifecycle.shutdown(...args), + destroy: (device, initiator, operation, claim) => { + const reservation = capacity.tryReserveBoot( + capacityDevice(device), + capacityDevices(registry.snapshot.virtualDevices), + ); + const warmClaim = claims.tryClaim(device.id, "boot"); + raced = reservation.ok && warmClaim !== undefined; + warmBoot = { + release: () => { + warmClaim?.release(); + if (reservation.ok) reservation.reservation.release(); + }, + }; + return lifecycle.destroy(device, initiator, operation, claim); + }, + }), + }); + await seedShutdown(harness); + // Call 1 is seedShutdown's own boot; call 2 is the waiter's. + harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); + + await expect( + harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), + ).rejects.toMatchObject({ name: "BootTimeoutError" }); + expect(raced).toBe(true); + + // The warm pool's boot ends and gives back its own claim and slot. + warmBoot?.release(); + await settle(); + + for (const requester of ["first", "second"]) { + await expect( + harness.coordinator.request(request, { + noWait: true, + ownerId: requester, + requesterId: requester, + }), + ).resolves.toMatchObject({ lease: { requesterId: requester } }); + } + }); +}); diff --git a/src/leasing/lease-acquisition-coordinator-reported-events.test.ts b/src/leasing/lease-acquisition-coordinator-reported-events.test.ts new file mode 100644 index 000000000..54f244878 --- /dev/null +++ b/src/leasing/lease-acquisition-coordinator-reported-events.test.ts @@ -0,0 +1,205 @@ +import { describe, expect, it } from "vitest"; +import { + request, + createHarness, + seedReady, + flush, + settle, + seedShutdown, +} from "./test-lease-acquisition.js"; + +describe("LeaseAcquisitionCoordinator: what lease.granted reports", () => { + const owner = (id: string) => ({ ownerId: id, requesterId: id }); + + function granted(harness: Awaited>) { + const events: Array<{ requestId: string; source: string; leaseId: string }> = []; + harness.bus.subscribe("lease.granted", (envelope) => events.push(envelope.payload)); + const requested: string[] = []; + harness.bus.subscribe("lease.requested", (envelope) => + requested.push(envelope.payload.requestId), + ); + return { events, requested }; + } + + it.each([ + { + name: "warm for a lease granted from a ready device", + source: "warm", + seed: async (harness) => { + await seedReady(harness); + }, + }, + { + name: "booted for a lease granted after booting a shut-down device", + source: "booted", + seed: async (harness) => { + await seedShutdown(harness); + }, + }, + { + name: "provisioned for a lease granted on a device provisioned for it", + source: "provisioned", + seed: async () => undefined, + }, + { + name: "provisioned when an eviction deleted a device and a new one was created", + source: "provisioned", + seed: async (harness) => { + await seedReady(harness, { ...request, model: "iPhone SE" }); + }, + }, + { + // why: with one running slot taken by a ready device of another model, the shut-down device + // of the requested model can boot only after an eviction gives the slot back. + name: "booted when an eviction shut a device down and a shut-down device was then booted", + source: "booted", + limits: { maxDevices: 2, maxRunning: 1 }, + seed: async (harness) => { + await seedShutdown(harness); + return seedReady(harness, { ...request, model: "iPhone SE" }); + }, + evictedLeavesReady: true, + }, + ] satisfies ReadonlyArray<{ + name: string; + source: "warm" | "booted" | "provisioned"; + limits?: { maxDevices: number; maxRunning: number }; + seed: (harness: Awaited>) => Promise<{ id: string } | void>; + evictedLeavesReady?: boolean; + }>)( + "emits lease.granted with exactly the grant's device, lease, requester, request and platform, and source $name", + async ({ source, limits, seed, evictedLeavesReady }) => { + const harness = await createHarness(limits); + const evicted = await seed(harness); + const { events, requested } = granted(harness); + + const grant = await harness.coordinator.request(request, owner("agent")); + + expect(requested[0]).toMatch(/^req_/); + expect(events).toEqual([ + { + deviceId: grant.device.id, + leaseId: grant.lease.id, + platform: grant.device.spec.platform, + requester: "agent", + requestId: requested[0], + source, + }, + ]); + if (evictedLeavesReady === true) { + expect( + harness.registry.snapshot.devices.find((device) => device.id === evicted?.id)?.state, + ).not.toBe("ready"); + } + }, + ); + + it("gives a request refused as already-leased the id its admission would have minted, and emits no lease.requested for it", async () => { + const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); + const first = await harness.coordinator.request(request, owner("agent")); + const requested: string[] = []; + harness.bus.subscribe("lease.requested", (envelope) => + requested.push(envelope.payload.requestId), + ); + const rejected: Array<{ requestId: string; requester: string; reason: string }> = []; + harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + const mintedBefore = harness.requestIds.length; + + await expect(harness.coordinator.request(request, owner("agent"))).rejects.toMatchObject({ + existingLeaseId: first.lease.id, + }); + + expect(requested).toEqual([]); + expect(harness.requestIds).toHaveLength(mintedBefore + 1); + expect(rejected).toEqual([ + expect.objectContaining({ + reason: "already-leased", + requestId: `req_${harness.requestIds[mintedBefore]}`, + requester: "agent", + }), + ]); + + // The same position in the sequence, on a request that passes admission: that id is the one + // the stored request gets, and nothing mints a second. + const mintedBeforeAdmitted = harness.requestIds.length; + await harness.coordinator.request(request, owner("other")); + expect(harness.requestIds).toHaveLength(mintedBeforeAdmitted + 1); + expect(requested).toEqual([`req_${harness.requestIds[mintedBeforeAdmitted]}`]); + }); + + it("gives a request refused as killed the id it would have been stored under", async () => { + const harness = await createHarness(); + await harness.coordinator.beginMaintenance(); + const rejected: Array<{ requestId: string; requester: string; reason: string }> = []; + harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + await expect(harness.coordinator.request(request, owner("agent"))).rejects.toMatchObject({ + name: "NukeCancelledError", + }); + + expect(rejected).toEqual([ + expect.objectContaining({ + reason: "killed", + requestId: `req_${harness.requestIds[0]}`, + requester: "agent", + }), + ]); + }); +}); + +describe("LeaseAcquisitionCoordinator probes (ADR 0021)", () => { + const probe = (requesterId: string) => ({ + fleetRequestId: "req_gw1", + noWait: true, + ownerId: requesterId, + requesterId, + }); + + it("emits lease.declined, and no lease.rejected, for a probe refused killed at admission", async () => { + const harness = await createHarness(); + await harness.coordinator.beginMaintenance(); + const declined: unknown[] = []; + const rejected: unknown[] = []; + harness.bus.subscribe("lease.declined", (envelope) => declined.push(envelope.payload)); + harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + + await expect(harness.coordinator.request(request, probe("gw:one"))).rejects.toMatchObject({ + name: "NukeCancelledError", + }); + + expect(declined).toEqual([ + { + fleetRequestId: "req_gw1", + reason: "killed", + requestId: `req_${harness.requestIds[0]}`, + requestSpec: request, + requester: "gw:one", + }, + ]); + expect(rejected).toEqual([]); + }); + + it("emits lease.declined, and no lease.rejected, for a probe in progress when beginMaintenance runs", async () => { + const harness = await createHarness(); + harness.driver.hangMakeReady(); + const declined: unknown[] = []; + const rejected: unknown[] = []; + harness.bus.subscribe("lease.declined", (envelope) => declined.push(envelope.payload)); + harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + const acquisition = harness.coordinator.request(request, probe("gw:one")); + void acquisition.catch(() => undefined); + await settle(); + + const maintenance = harness.coordinator.beginMaintenance(); + await flush(); + harness.driver.releaseMakeReady(); + await maintenance; + + await expect(acquisition).rejects.toMatchObject({ name: "NukeCancelledError" }); + expect(declined).toEqual([ + expect.objectContaining({ fleetRequestId: "req_gw1", reason: "killed", requester: "gw:one" }), + ]); + expect(rejected).toEqual([]); + await harness.coordinator.endMaintenance(); + }); +}); diff --git a/src/leasing/lease-acquisition-coordinator-stored-requests.test.ts b/src/leasing/lease-acquisition-coordinator-stored-requests.test.ts new file mode 100644 index 000000000..3f4acd5fa --- /dev/null +++ b/src/leasing/lease-acquisition-coordinator-stored-requests.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { NoCapacityError } from "./lease-acquisition-coordinator.js"; +import { IdempotencyConflictError, LeaseRequestForbiddenError } from "./lease-request-book.js"; +import { request, createHarness, flush } from "./test-lease-acquisition.js"; +import { inspectCommits } from "./test-lease-acquisition-commits.js"; + +// why: drives the coordinator on its own, so a test can interleave its lifecycle, queue and clock in orders the leasing surface cannot force. +describe("LeaseAcquisitionCoordinator stored requests", () => { + const keyed = { idempotencyKey: "key-1", ownerId: "agent", requesterId: "agent" } as const; + + /** Grants the one device capacity allows to `holder`, then frees it again on demand. */ + async function holdTheOnlyDevice(harness: Awaited>) { + const held = await harness.coordinator.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + return async () => { + await harness.registry.beginRelease(held.lease.id); + await harness.registry.transitionDevice(held.device.id, "ready", { + event: "device.reclaimed", + payload: { deviceId: held.device.id, duration: 0, strategy: "wipe" }, + }); + harness.coordinator.kick(); + await flush(); + }; + } + + it("returns the first result to a repeated request under the same key and grants no second lease", async () => { + const harness = await createHarness(); + const first = await harness.coordinator.request(request, keyed); + + const second = await harness.coordinator.request(request, keyed); + + expect(second.lease.id).toBe(first.lease.id); + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); + + it("attaches a repeat of an open request to the existing wait instead of starting a second one", async () => { + const harness = await createHarness(); + const free = await holdTheOnlyDevice(harness); + const first = harness.coordinator.request(request, keyed); + await flush(); + expect(harness.coordinator.queueDepth).toBe(1); + + const repeat = harness.coordinator.request(request, keyed); + await flush(); + expect(harness.coordinator.queueDepth).toBe(1); + expect( + harness.registry.leaseRequests().filter((record) => record.requesterId === "agent"), + ).toHaveLength(1); + + await free(); + const [granted, repeated] = await Promise.all([first, repeat]); + expect(repeated.lease.id).toBe(granted.lease.id); + }); + + it("tells a repeat joining an open wait where that wait stands now", async () => { + const harness = await createHarness(); + await holdTheOnlyDevice(harness); + void harness.coordinator.request(request, keyed).catch(() => undefined); + await flush(); + + const heard: unknown[] = []; + void harness.coordinator + .request(request, { ...keyed, onProgress: (progress) => heard.push(progress) }) + .catch(() => undefined); + await flush(); + + expect(heard).toEqual([{ queuePosition: 1, stage: "queued" }]); + }); + + it("stores a request before the wait queue sees it", async () => { + const harness = await createHarness(); + const order: string[] = []; + inspectCommits(harness, ({ requestRequesters }) => { + if (requestRequesters.includes("agent") && !order.includes("stored")) order.push("stored"); + }); + const create = harness.queue.create.bind(harness.queue); + harness.queue.create = (...args) => { + order.push("queued"); + return create(...args); + }; + + await harness.coordinator.request(request, keyed); + + expect(order.slice(0, 2)).toEqual(["stored", "queued"]); + }); + + it("keeps a capacity failure terminal under the same key after capacity frees up", async () => { + const harness = await createHarness(); + const free = await holdTheOnlyDevice(harness); + await expect( + harness.coordinator.request(request, { ...keyed, noWait: true }), + ).rejects.toBeInstanceOf(NoCapacityError); + await free(); + + await expect( + harness.coordinator.request(request, { ...keyed, noWait: true }), + ).rejects.toMatchObject({ code: "NO_CAPACITY", name: "ReplayedLeaseRequestError" }); + expect(harness.registry.snapshot.leases).toHaveLength(0); + }); + + it("starts a fresh request for a new key once the prior one is terminal", async () => { + const harness = await createHarness(); + const free = await holdTheOnlyDevice(harness); + await expect( + harness.coordinator.request(request, { ...keyed, noWait: true }), + ).rejects.toBeInstanceOf(NoCapacityError); + await free(); + + const granted = await harness.coordinator.request(request, { + ...keyed, + idempotencyKey: "key-2", + }); + + expect(granted.lease.requesterId).toBe("agent"); + // The grant and its stored result land in one commit; the flush lets the wait settle. + await flush(); + expect(harness.registry.leaseRequests().map((record) => record.state)).toEqual([ + "granted", + "failed", + "granted", + ]); + }); + + it("refuses the same key naming a different device as an idempotency conflict", async () => { + const harness = await createHarness(); + await harness.coordinator.request(request, keyed); + + await expect( + harness.coordinator.request({ ...request, model: "iPhone 17" }, keyed), + ).rejects.toBeInstanceOf(IdempotencyConflictError); + }); + + it("refuses a repeat sent by a different principal", async () => { + const harness = await createHarness(); + await harness.coordinator.request(request, keyed); + + await expect( + harness.coordinator.request(request, { ...keyed, ownerId: "someone-else" }), + ).rejects.toBeInstanceOf(LeaseRequestForbiddenError); + }); + + it("stores one record for two concurrent requests under one key", async () => { + const harness = await createHarness(); + + const [first, second] = await Promise.all([ + harness.coordinator.request(request, keyed), + harness.coordinator.request(request, keyed), + ]); + + expect(harness.registry.leaseRequests()).toHaveLength(1); + expect(second.lease.id).toBe(first.lease.id); + }); + + it("stores a request that carries no idempotency key, and names it on lease.requested", async () => { + const harness = await createHarness(); + + await harness.coordinator.request(request, { ownerId: "agent", requesterId: "agent" }); + await flush(); + + expect(harness.registry.leaseRequests()).toMatchObject([ + { requesterId: "agent", state: "granted" }, + ]); + const [stored] = harness.registry.leaseRequests(); + expect( + harness.bus + .replay() + .filter((event) => event.event === "lease.requested") + .map((event) => event.payload), + ).toMatchObject([{ requestId: stored?.id, requester: "agent" }]); + expect(harness.registry.leaseRequests()[0]).not.toHaveProperty("idempotencyKey"); + }); +}); diff --git a/src/leasing/lease-acquisition-coordinator-waiting-and-baselines.test.ts b/src/leasing/lease-acquisition-coordinator-waiting-and-baselines.test.ts new file mode 100644 index 000000000..1ae97f6c4 --- /dev/null +++ b/src/leasing/lease-acquisition-coordinator-waiting-and-baselines.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { FakeDriver } from "../core/testing.js"; +import { request, createHarness, seedReady, flush } from "./test-lease-acquisition.js"; + +describe("LeaseAcquisitionCoordinator waiting demand", () => { + it("lists a request waiting for a device with what a device must satisfy, and drops it once it is granted", async () => { + const harness = await createHarness(); + const held = await harness.coordinator.request(request, { + ownerId: "holder", + requesterId: "holder", + }); + expect(harness.coordinator.waitingDemand()).toEqual([]); + + const waiting = harness.coordinator.request(request, { + ownerId: "waiter", + requesterId: "waiter", + }); + await flush(); + + expect(harness.coordinator.waitingDemand()).toEqual([ + { + classOf: expect.any(Function), + inFlight: false, + mode: "full", + platform: "ios", + requirement: { + imageTag: undefined, + osVersion: { kind: "exact", version: "26.5" }, + platform: "ios", + target: { kind: "model", model: "iPhone 16" }, + }, + spec: request, + }, + ]); + + await harness.registry.beginRelease(held.lease.id); + await harness.registry.transitionDevice(held.device.id, "ready", { + event: "device.reclaimed", + payload: { deviceId: held.device.id, duration: 0, strategy: "wipe" }, + }); + harness.coordinator.kick(); + await waiting; + expect(harness.coordinator.waitingDemand()).toEqual([]); + }); + + it("reports the mode a request resolved to, and marks a request whose device work has begun as in flight", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { provision: 50 }, + platform: "ios", + slimmableOsVersions: ["26.5"], + }); + const harness = await createHarness({ drivers: [driver] }); + const pending = harness.coordinator.request( + { ...request, mode: "slim" }, + { ownerId: "slim", requesterId: "slim" }, + ); + await flush(); + + expect(harness.coordinator.waitingDemand()).toMatchObject([{ inFlight: true, mode: "slim" }]); + + clock.advance(50); + await pending; + expect(harness.coordinator.waitingDemand()).toEqual([]); + }); + + it("reports maintenance active between beginMaintenance and endMaintenance", async () => { + const harness = await createHarness(); + expect(harness.coordinator.maintenanceActive).toBe(false); + + await harness.coordinator.beginMaintenance(); + expect(harness.coordinator.maintenanceActive).toBe(true); + + await harness.coordinator.endMaintenance(); + expect(harness.coordinator.maintenanceActive).toBe(false); + }); +}); + +describe("LeaseAcquisitionCoordinator: a request naming a baseline", () => { + it("plans against the version current at its decision, so a save between the resolve and the plan grants an idle device of the new version", async () => { + const current = (version: number) => ({ + mode: "full" as const, + model: request.model, + name: "checkout", + osVersion: request.osVersion, + platform: request.platform, + sourceId: `source-${version}`, + version, + }); + const reads = [current(1)]; + const harness = await createHarness({ + baselines: { current: () => reads.shift() ?? current(2) }, + maxDevices: 2, + }); + const idleV2 = await seedReady(harness, { + ...request, + baseline: { name: "checkout", version: 2 }, + }); + + const granted = await harness.coordinator.request( + { baseline: "checkout", platform: "ios" }, + { ownerId: "agent", requesterId: "agent" }, + ); + + expect(granted.device.id).toBe(idleV2.id); + expect(granted.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); + }); + + it("ends a request whose baseline was removed while it waited as UnknownBaselineError at once, without a retry", async () => { + const harness = await createHarness({ + baselines: { + current: () => ({ + mode: "full" as const, + model: request.model, + name: "checkout", + osVersion: request.osVersion, + platform: request.platform, + sourceId: "source-1", + version: 1, + }), + }, + maxDevices: 1, + }); + + const outcome = await harness.coordinator + .request( + { baseline: "checkout", platform: "ios" }, + { ownerId: "agent", requesterId: "agent" }, + ) + .then( + () => "granted", + (error: unknown) => (error as Error).name, + ); + + expect(outcome).toBe("UnknownBaselineError"); + }); +}); diff --git a/src/leasing/lease-acquisition-coordinator.test.ts b/src/leasing/lease-acquisition-coordinator.test.ts index 13e36205e..4ce607354 100644 --- a/src/leasing/lease-acquisition-coordinator.test.ts +++ b/src/leasing/lease-acquisition-coordinator.test.ts @@ -1,20 +1,9 @@ import { describe, expect, it, vi } from "vitest"; - -import { FakeClock, JsonLinesLogger, type Logger, MemoryLogSink } from "../ports/index.js"; -import type { StateDatabase } from "../state/index.js"; +import { FakeClock } from "../ports/index.js"; import { promiseState } from "../test-support/promise-state.js"; -import { - capacityDevice, - capacityDevices, - DriverCrashError, - type DeviceMode, - type VirtualSpec, - specMode, - type ManagedDeviceLifecycle, -} from "../core/index.js"; +import { DriverCrashError, type ManagedDeviceLifecycle } from "../core/index.js"; import { FakeDriver } from "../core/testing.js"; import { NoCapacityError } from "./lease-acquisition-coordinator.js"; -import { IdempotencyConflictError, LeaseRequestForbiddenError } from "./lease-request-book.js"; import { RequesterAlreadyLeasedError } from "./wait-queue.js"; import { request, @@ -24,42 +13,7 @@ import { settle, seedShutdown, } from "./test-lease-acquisition.js"; - -interface StoredByCommit { - readonly leaseRequesters: readonly string[]; - readonly requestRequesters: readonly string[]; -} - -/** - * why: runs `inspect` inside every registry commit, once its rows are written and before it commits: it - * sees what the commit stores, and throwing from it rolls that commit back. - */ -function inspectCommits( - harness: { readonly state: StateDatabase }, - inspect: (stored: StoredByCommit) => void, -): void { - const original = harness.state.transaction.bind(harness.state); - vi.spyOn(harness.state, "transaction").mockImplementation((work) => - original(() => { - const result = work(); - const requesters = (table: string) => - harness.state.database - .prepare(`SELECT requester_id FROM ${table}`) - .all() - .map((row) => String(row["requester_id"])); - inspect({ - leaseRequesters: requesters("leases"), - requestRequesters: requesters("lease_requests"), - }); - return result; - }), - ); -} - -function capturingLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { - const sink = new MemoryLogSink(); - return { logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), sink }; -} +import { inspectCommits } from "./test-lease-acquisition-commits.js"; // why: drives the coordinator on its own, so a test can interleave its lifecycle, queue and clock in orders the leasing surface cannot force. describe("LeaseAcquisitionCoordinator", () => { @@ -880,993 +834,4 @@ describe("LeaseAcquisitionCoordinator", () => { }), ).resolves.toMatchObject({ lease: { ownerId: "reopened", requesterId: "reopened" } }); }); - - describe("device mode", () => { - /** A driver that slims 26.5 and cannot slim 17.5, like iOS on either side of 18.5. */ - function slimmingDriver(options: { readonly mode?: DeviceMode } = {}): FakeDriver { - return new FakeDriver({ - availableOsVersions: ["17.5", "26.5"], - clock: new FakeClock(1_000), - platform: "ios", - slimmableOsVersions: ["26.5"], - ...options, - }); - } - - const owner = (id: string) => ({ ownerId: id, requesterId: id }); - - it.each([ - ["full", "no mode", undefined, "full"], - ["slim", "no mode", undefined, "slim"], - ["full", "mode slim", "slim", "slim"], - ["slim", "mode full", "full", "full"], - ] as const)( - "on a worker whose default is %s, a request with %s gets that device mode", - async (defaultMode, _label, mode, planned) => { - const harness = await createHarness({ - defaultModes: { ios: defaultMode }, - drivers: [slimmingDriver()], - }); - - const granted = await harness.coordinator.request( - { ...request, ...(mode === undefined ? {} : { mode }) }, - owner("agent"), - ); - - expect(specMode(granted.device.spec)).toBe(planned); - expect(granted.device.mode).toBe(planned); - }, - ); - - it.each([ - ["slim", "slim", true], - ["slim", "full", false], - ["full", "full", true], - ["full", "slim", false], - ] as const)( - "on a worker whose default is %s, a device whose pool mode is %s reports servesDefaultMode %s", - async (defaultMode, poolMode, served) => { - const harness = await createHarness({ - defaultModes: { ios: defaultMode }, - drivers: [slimmingDriver()], - }); - const spec = { - physical: false as const, - model: "iPhone 17", - osVersion: "26.5", - platform: "ios" as const, - ...(poolMode === "slim" ? { mode: "slim" as const } : {}), - }; - - expect(harness.coordinator.servesDefaultMode(spec)).toBe(served); - }, - ); - - it("compares a device's pool mode with its own platform's default, full when that platform has none", async () => { - const harness = await createHarness({ - defaultModes: { ios: "slim" }, - drivers: [slimmingDriver()], - }); - const android = { - physical: false as const, - model: "Pixel 9", - osVersion: "35", - platform: "android" as const, - }; - - expect(harness.coordinator.servesDefaultMode(android)).toBe(true); - expect(harness.coordinator.servesDefaultMode({ ...android, mode: "slim" })).toBe(false); - }); - - it.each(["ios", "android"] as const)( - "reports servesDefaultMode false for a physical spec on %s, though it names no mode and the platform's default is full", - async (platform) => { - const harness = await createHarness({ defaultModes: { [platform]: "full" } }); - - expect( - harness.coordinator.servesDefaultMode({ - class: "phone", - model: "Some Phone", - osVersion: "1", - physical: true, - platform, - }), - ).toBe(false); - }, - ); - - it("passes the resolved mode to resolveSpec, the default filled in for a request that named none", async () => { - const driver = slimmingDriver(); - const harness = await createHarness({ defaultModes: { ios: "slim" }, drivers: [driver] }); - - await harness.coordinator.request(request, owner("agent")); - - const resolved = driver.calls.find((call) => call.operation === "resolveSpec"); - expect(resolved?.arguments[0]).toMatchObject({ mode: "slim" }); - }); - - it("never plans a full request onto a slim spec, even when the driver's resolveSpec returns one", async () => { - class AlwaysSlimDriver extends FakeDriver { - override async resolveSpec( - ...args: Parameters - ): Promise { - return { ...(await super.resolveSpec(...args)), mode: "slim" }; - } - } - const driver = new AlwaysSlimDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }); - const harness = await createHarness({ drivers: [driver] }); - - const granted = await harness.coordinator.request( - { ...request, mode: "full" }, - owner("agent"), - ); - - expect(granted.device.spec).not.toHaveProperty("mode"); - const provisioned = driver.calls.find((call) => call.operation === "provision"); - expect(provisioned?.arguments[0]).not.toHaveProperty("mode"); - }); - - it.each([ - { name: "full request an idle slim", seeded: "slim", requested: "full" }, - { name: "slim request an idle full", seeded: "full", requested: "slim" }, - ] as const)( - "does not give a $name device of the same model and runtime", - async ({ seeded, requested }) => { - const harness = await createHarness({ drivers: [slimmingDriver()], maxDevices: 2 }); - const idle = await seedReady( - harness, - seeded === "slim" ? { ...request, mode: "slim" } : request, - ); - - const granted = await harness.coordinator.request( - { ...request, mode: requested }, - owner("agent"), - ); - - expect(granted.device.id).not.toBe(idle.id); - expect(granted.device.mode).toBe(requested); - }, - ); - - it("resolves a slim request on a runtime the driver cannot slim to the full spec, reusing an idle full device", async () => { - const oldRuntime = { ...request, osVersion: "17.5" }; - const harness = await createHarness({ drivers: [slimmingDriver()] }); - const full = await seedReady(harness, oldRuntime); - - const granted = await harness.coordinator.request( - { ...oldRuntime, mode: "slim" }, - owner("agent"), - ); - - expect(granted.device.id).toBe(full.id); - expect(granted.device.spec).toEqual(oldRuntime); - expect(granted.device.mode).toBe("full"); - }); - - it("grants a slim request against a driver that knows nothing of modes a full device", async () => { - const harness = await createHarness(); - - const granted = await harness.coordinator.request( - { ...request, mode: "slim" }, - owner("agent"), - ); - - expect(granted.device.spec).toEqual(request); - expect(granted.device.mode).toBe("full"); - }); - - it("grants a slim-spec device whose slim pass failed as full, and keeps it in the slim pool", async () => { - const harness = await createHarness({ drivers: [slimmingDriver({ mode: "full" })] }); - - const granted = await harness.coordinator.request( - { ...request, mode: "slim" }, - owner("agent"), - ); - - expect(granted.device.mode).toBe("full"); - expect(granted.device.spec).toEqual({ ...request, mode: "slim" }); - }); - }); - - it("A failed eviction shutdown logs the device id and the error, and the waiter is deferred as before.", async () => { - const { logger, sink } = capturingLogger(); - // Two devices allowed but only one running: a running device of another spec is - // evicted by shutdown (not deleted) to make room for the request. - const harness = await createHarness({ logger, maxDevices: 2 }); - const victim = await seedReady(harness, { ...request, model: "iPhone SE" }); - harness.driver.failOn("shutdown", 1, new DriverCrashError("shutdown wedged")); - - let settled = false; - const acquisition = harness.coordinator.request(request, { - ownerId: "evictor", - requesterId: "evictor", - }); - void acquisition.then( - () => (settled = true), - () => (settled = true), - ); - await settle(); - - expect(sink.records).toEqual([ - expect.objectContaining({ - level: "warn", - module: "daemon.lease-acquisition-coordinator", - fields: { - deviceId: victim.id, - error: "DriverCrashError: shutdown wedged", - requesterId: "evictor", - step: "shutdown", - }, - }), - ]); - // Deferred: back in the queue, still pending, and the victim untouched. - expect(settled).toBe(false); - expect(harness.coordinator.queueDepth).toBe(1); - expect(harness.registry.snapshot.devices.find((device) => device.id === victim.id)?.state).toBe( - "ready", - ); - await harness.coordinator.cancelPending("evictor"); - await expect(acquisition).rejects.toMatchObject({ name: "RequestCancelledError" }); - }); - - it("A failed eviction delete logs the device id and the error.", async () => { - const { logger, sink } = capturingLogger(); - // One device allowed: a managed device of another spec is deleted to make room. - const harness = await createHarness({ logger, maxDevices: 1 }); - const victim = await seedShutdown(harness, { ...request, model: "iPhone SE" }); - harness.driver.failOn("destroy", 1, new DriverCrashError("simctl delete refused")); - - const acquisition = harness.coordinator.request(request, { - ownerId: "evictor", - requesterId: "evictor", - }); - await settle(); - - expect(sink.records).toEqual([ - expect.objectContaining({ - level: "warn", - fields: { - deviceId: victim.id, - error: "DriverCrashError: simctl delete refused", - requesterId: "evictor", - step: "delete", - }, - }), - ]); - await harness.coordinator.cancelPending("evictor"); - await expect(acquisition).rejects.toMatchObject({ name: "RequestCancelledError" }); - }); - - it("A failed destroy after a shut-down device fails to boot for a waiter logs the device id and the error.", async () => { - const { logger, sink } = capturingLogger(); - const harness = await createHarness({ logger }); - const shutdown = await seedShutdown(harness); - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - harness.driver.failOn("destroy", 1, new DriverCrashError("simulator would not die")); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ - name: "BootTimeoutError", - message: expect.stringContaining("simulator never booted"), - }); - - expect(sink.records.filter((record) => record.fields?.["step"] === "destroy")).toEqual([ - expect.objectContaining({ - level: "warn", - fields: { - deviceId: shutdown.id, - error: "DriverCrashError: simulator would not die", - requesterId: "booter", - step: "destroy", - }, - }), - ]); - }); - - it("A shut-down device that fails to boot for a waiter and then fails to be destroyed is quarantined and counts once against running capacity.", async () => { - // why: two running slots — the failed device's and the one a second request needs. - const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); - const shutdown = await seedShutdown(harness); - // why: call 1 is seedShutdown's own boot; call 2 is the waiter's. - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - harness.driver.failOn("destroy", 1, new DriverCrashError("simulator would not die")); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ name: "BootTimeoutError" }); - await settle(); - - expect( - harness.registry.snapshot.devices.find((device) => device.id === shutdown.id)?.state, - ).toBe("quarantined"); - // why: `quarantined` already counts as running — a reservation still held would count - // this device twice and refuse the second slot. - await expect( - harness.coordinator.request( - { ...request, model: "iPhone SE" }, - { noWait: true, ownerId: "second", requesterId: "second" }, - ), - ).resolves.toMatchObject({ device: { spec: { model: "iPhone SE" }, state: "leased" } }); - }); - - it("grants a request that arrives while a failed boot's device is being destroyed a device of its own, without waiting for the destroy.", async () => { - const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); - const shutdownId = (await seedShutdown(harness)).id; - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - let failDestroy: (error: Error) => void = () => undefined; - const realDestroy = harness.driver.destroy.bind(harness.driver); - let destroys = 0; - vi.spyOn(harness.driver, "destroy").mockImplementation(async (device) => { - destroys += 1; - if (destroys > 1) return realDestroy(device); - return new Promise((_resolve, reject) => { - failDestroy = reject; - }); - }); - - const booter = harness.coordinator.request(request, { - ownerId: "booter", - requesterId: "booter", - }); - const booterOutcome = booter.catch((error: unknown) => error); - await settle(); - const second = harness.coordinator.request(request, { ownerId: "b", requesterId: "b" }); - let secondGranted = false; - void second.then(() => { - secondGranted = true; - }); - await settle(); - - expect(secondGranted).toBe(true); - expect((await second).device.id).not.toBe(shutdownId); - - failDestroy(new DriverCrashError("simulator would not die")); - await booterOutcome; - }); - - it("A device whose boot for a waiter failed is deleted by the lease engine, and its slot is freed.", async () => { - const harness = await createHarness(); - const shutdown = await seedShutdown(harness); - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ name: "BootTimeoutError" }); - - expect(harness.bus.replay().filter((event) => event.event === "device.deleted")).toEqual([ - expect.objectContaining({ - payload: expect.objectContaining({ deviceId: shutdown.id, initiator: "lease-engine" }), - }), - ]); - await expect( - harness.coordinator.request(request, { - noWait: true, - ownerId: "second", - requesterId: "second", - }), - ).resolves.toMatchObject({ lease: { requesterId: "second" } }); - }); - - it("A failed boot whose device the destroy can no longer claim frees the waiter's running slot and leaves no fence.", async () => { - const harness = await createHarness(); - const shutdown = await seedShutdown(harness); - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - const realMakeReady = harness.driver.makeReady.bind(harness.driver); - let failedBoots = 0; - vi.spyOn(harness.driver, "makeReady").mockImplementation(async (...args) => { - try { - return await realMakeReady(...args); - } finally { - failedBoots += 1; - // The record leaves `shutdown` while the boot fails, so the destroy finds nothing to claim. - if (failedBoots === 1) { - await harness.registry.transitionDevice(shutdown.id, "deleted", { - event: "device.deleted", - payload: { deviceId: shutdown.id, initiator: "test" }, - }); - } - } - }); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ name: "BootTimeoutError" }); - - await expect( - harness.coordinator.request(request, { - noWait: true, - ownerId: "second", - requesterId: "second", - }), - ).resolves.toMatchObject({ lease: { requesterId: "second" } }); - }); - - it.each([ - ["answers that another operation owns the device", async (): Promise => false], - ["throws", async (): Promise => Promise.reject(new Error("state write failed"))], - ] as const)( - "A failed boot whose destroy throws frees the waiter's running slot and leaves no fence, even when quarantine %s.", - async (_label, enterFromStalledTransition) => { - const harness = await createHarness({ - lifecycle: (lifecycle) => ({ - bootForLease: (device, claim) => lifecycle.bootForLease(device, claim), - rebootForLease: (...args) => lifecycle.rebootForLease(...args), - dispose: (...args) => lifecycle.dispose(...args), - shutdown: (...args) => lifecycle.shutdown(...args), - destroy: () => Promise.reject(new DriverCrashError("simulator would not delete")), - }), - quarantine: { enterFromStalledTransition }, - }); - const shutdown = await seedShutdown(harness); - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ name: "BootTimeoutError" }); - await settle(); - - await expect( - harness.coordinator.request(request, { ownerId: "second", requesterId: "second" }), - ).resolves.toMatchObject({ device: { id: shutdown.id, state: "leased" } }); - }, - ); - - it("releases the running slot reserved to boot a shut-down device for a waiter when the boot fails and another boot claims the device before it is destroyed", async () => { - // The warm pool's boot reserves its own slot and claims the device in one decision. Here it - // lands after the failed boot has released its claim and before the destroy claims the device. - let raced = false; - let warmBoot: { release(): void } | undefined; - const harness = await createHarness({ - maxDevices: 2, - maxRunning: 2, - lifecycle: (lifecycle, { capacity, claims, registry }) => ({ - bootForLease: (device, claim) => lifecycle.bootForLease(device, claim), - rebootForLease: (...args) => lifecycle.rebootForLease(...args), - dispose: (...args) => lifecycle.dispose(...args), - shutdown: (...args) => lifecycle.shutdown(...args), - destroy: (device, initiator, operation, claim) => { - const reservation = capacity.tryReserveBoot( - capacityDevice(device), - capacityDevices(registry.snapshot.virtualDevices), - ); - const warmClaim = claims.tryClaim(device.id, "boot"); - raced = reservation.ok && warmClaim !== undefined; - warmBoot = { - release: () => { - warmClaim?.release(); - if (reservation.ok) reservation.reservation.release(); - }, - }; - return lifecycle.destroy(device, initiator, operation, claim); - }, - }), - }); - await seedShutdown(harness); - // Call 1 is seedShutdown's own boot; call 2 is the waiter's. - harness.driver.failOn("makeReady", 2, new DriverCrashError("simulator never booted")); - - await expect( - harness.coordinator.request(request, { ownerId: "booter", requesterId: "booter" }), - ).rejects.toMatchObject({ name: "BootTimeoutError" }); - expect(raced).toBe(true); - - // The warm pool's boot ends and gives back its own claim and slot. - warmBoot?.release(); - await settle(); - - for (const requester of ["first", "second"]) { - await expect( - harness.coordinator.request(request, { - noWait: true, - ownerId: requester, - requesterId: requester, - }), - ).resolves.toMatchObject({ lease: { requesterId: requester } }); - } - }); -}); - -describe("LeaseAcquisitionCoordinator stored requests", () => { - const keyed = { idempotencyKey: "key-1", ownerId: "agent", requesterId: "agent" } as const; - - /** Grants the one device capacity allows to `holder`, then frees it again on demand. */ - async function holdTheOnlyDevice(harness: Awaited>) { - const held = await harness.coordinator.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - return async () => { - await harness.registry.beginRelease(held.lease.id); - await harness.registry.transitionDevice(held.device.id, "ready", { - event: "device.reclaimed", - payload: { deviceId: held.device.id, duration: 0, strategy: "wipe" }, - }); - harness.coordinator.kick(); - await flush(); - }; - } - - it("returns the first result to a repeated request under the same key and grants no second lease", async () => { - const harness = await createHarness(); - const first = await harness.coordinator.request(request, keyed); - - const second = await harness.coordinator.request(request, keyed); - - expect(second.lease.id).toBe(first.lease.id); - expect(harness.registry.snapshot.leases).toHaveLength(1); - }); - - it("attaches a repeat of an open request to the existing wait instead of starting a second one", async () => { - const harness = await createHarness(); - const free = await holdTheOnlyDevice(harness); - const first = harness.coordinator.request(request, keyed); - await flush(); - expect(harness.coordinator.queueDepth).toBe(1); - - const repeat = harness.coordinator.request(request, keyed); - await flush(); - expect(harness.coordinator.queueDepth).toBe(1); - expect( - harness.registry.leaseRequests().filter((record) => record.requesterId === "agent"), - ).toHaveLength(1); - - await free(); - const [granted, repeated] = await Promise.all([first, repeat]); - expect(repeated.lease.id).toBe(granted.lease.id); - }); - - it("tells a repeat joining an open wait where that wait stands now", async () => { - const harness = await createHarness(); - await holdTheOnlyDevice(harness); - void harness.coordinator.request(request, keyed).catch(() => undefined); - await flush(); - - const heard: unknown[] = []; - void harness.coordinator - .request(request, { ...keyed, onProgress: (progress) => heard.push(progress) }) - .catch(() => undefined); - await flush(); - - expect(heard).toEqual([{ queuePosition: 1, stage: "queued" }]); - }); - - it("stores a request before the wait queue sees it", async () => { - const harness = await createHarness(); - const order: string[] = []; - inspectCommits(harness, ({ requestRequesters }) => { - if (requestRequesters.includes("agent") && !order.includes("stored")) order.push("stored"); - }); - const create = harness.queue.create.bind(harness.queue); - harness.queue.create = (...args) => { - order.push("queued"); - return create(...args); - }; - - await harness.coordinator.request(request, keyed); - - expect(order.slice(0, 2)).toEqual(["stored", "queued"]); - }); - - it("keeps a capacity failure terminal under the same key after capacity frees up", async () => { - const harness = await createHarness(); - const free = await holdTheOnlyDevice(harness); - await expect( - harness.coordinator.request(request, { ...keyed, noWait: true }), - ).rejects.toBeInstanceOf(NoCapacityError); - await free(); - - await expect( - harness.coordinator.request(request, { ...keyed, noWait: true }), - ).rejects.toMatchObject({ code: "NO_CAPACITY", name: "ReplayedLeaseRequestError" }); - expect(harness.registry.snapshot.leases).toHaveLength(0); - }); - - it("starts a fresh request for a new key once the prior one is terminal", async () => { - const harness = await createHarness(); - const free = await holdTheOnlyDevice(harness); - await expect( - harness.coordinator.request(request, { ...keyed, noWait: true }), - ).rejects.toBeInstanceOf(NoCapacityError); - await free(); - - const granted = await harness.coordinator.request(request, { - ...keyed, - idempotencyKey: "key-2", - }); - - expect(granted.lease.requesterId).toBe("agent"); - // The grant and its stored result land in one commit; the flush lets the wait settle. - await flush(); - expect(harness.registry.leaseRequests().map((record) => record.state)).toEqual([ - "granted", - "failed", - "granted", - ]); - }); - - it("refuses the same key naming a different device as an idempotency conflict", async () => { - const harness = await createHarness(); - await harness.coordinator.request(request, keyed); - - await expect( - harness.coordinator.request({ ...request, model: "iPhone 17" }, keyed), - ).rejects.toBeInstanceOf(IdempotencyConflictError); - }); - - it("refuses a repeat sent by a different principal", async () => { - const harness = await createHarness(); - await harness.coordinator.request(request, keyed); - - await expect( - harness.coordinator.request(request, { ...keyed, ownerId: "someone-else" }), - ).rejects.toBeInstanceOf(LeaseRequestForbiddenError); - }); - - it("stores one record for two concurrent requests under one key", async () => { - const harness = await createHarness(); - - const [first, second] = await Promise.all([ - harness.coordinator.request(request, keyed), - harness.coordinator.request(request, keyed), - ]); - - expect(harness.registry.leaseRequests()).toHaveLength(1); - expect(second.lease.id).toBe(first.lease.id); - }); - - it("stores a request that carries no idempotency key, and names it on lease.requested", async () => { - const harness = await createHarness(); - - await harness.coordinator.request(request, { ownerId: "agent", requesterId: "agent" }); - await flush(); - - expect(harness.registry.leaseRequests()).toMatchObject([ - { requesterId: "agent", state: "granted" }, - ]); - const [stored] = harness.registry.leaseRequests(); - expect( - harness.bus - .replay() - .filter((event) => event.event === "lease.requested") - .map((event) => event.payload), - ).toMatchObject([{ requestId: stored?.id, requester: "agent" }]); - expect(harness.registry.leaseRequests()[0]).not.toHaveProperty("idempotencyKey"); - }); -}); - -describe("LeaseAcquisitionCoordinator: what lease.granted reports", () => { - const owner = (id: string) => ({ ownerId: id, requesterId: id }); - - function granted(harness: Awaited>) { - const events: Array<{ requestId: string; source: string; leaseId: string }> = []; - harness.bus.subscribe("lease.granted", (envelope) => events.push(envelope.payload)); - const requested: string[] = []; - harness.bus.subscribe("lease.requested", (envelope) => - requested.push(envelope.payload.requestId), - ); - return { events, requested }; - } - - it.each([ - { - name: "warm for a lease granted from a ready device", - source: "warm", - seed: async (harness) => { - await seedReady(harness); - }, - }, - { - name: "booted for a lease granted after booting a shut-down device", - source: "booted", - seed: async (harness) => { - await seedShutdown(harness); - }, - }, - { - name: "provisioned for a lease granted on a device provisioned for it", - source: "provisioned", - seed: async () => undefined, - }, - { - name: "provisioned when an eviction deleted a device and a new one was created", - source: "provisioned", - seed: async (harness) => { - await seedReady(harness, { ...request, model: "iPhone SE" }); - }, - }, - { - // why: with one running slot taken by a ready device of another model, the shut-down device - // of the requested model can boot only after an eviction gives the slot back. - name: "booted when an eviction shut a device down and a shut-down device was then booted", - source: "booted", - limits: { maxDevices: 2, maxRunning: 1 }, - seed: async (harness) => { - await seedShutdown(harness); - return seedReady(harness, { ...request, model: "iPhone SE" }); - }, - evictedLeavesReady: true, - }, - ] satisfies ReadonlyArray<{ - name: string; - source: "warm" | "booted" | "provisioned"; - limits?: { maxDevices: number; maxRunning: number }; - seed: (harness: Awaited>) => Promise<{ id: string } | void>; - evictedLeavesReady?: boolean; - }>)( - "emits lease.granted with exactly the grant's device, lease, requester, request and platform, and source $name", - async ({ source, limits, seed, evictedLeavesReady }) => { - const harness = await createHarness(limits); - const evicted = await seed(harness); - const { events, requested } = granted(harness); - - const grant = await harness.coordinator.request(request, owner("agent")); - - expect(requested[0]).toMatch(/^req_/); - expect(events).toEqual([ - { - deviceId: grant.device.id, - leaseId: grant.lease.id, - platform: grant.device.spec.platform, - requester: "agent", - requestId: requested[0], - source, - }, - ]); - if (evictedLeavesReady === true) { - expect( - harness.registry.snapshot.devices.find((device) => device.id === evicted?.id)?.state, - ).not.toBe("ready"); - } - }, - ); - - it("gives a request refused as already-leased the id its admission would have minted, and emits no lease.requested for it", async () => { - const harness = await createHarness({ maxDevices: 2, maxRunning: 2 }); - const first = await harness.coordinator.request(request, owner("agent")); - const requested: string[] = []; - harness.bus.subscribe("lease.requested", (envelope) => - requested.push(envelope.payload.requestId), - ); - const rejected: Array<{ requestId: string; requester: string; reason: string }> = []; - harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - const mintedBefore = harness.requestIds.length; - - await expect(harness.coordinator.request(request, owner("agent"))).rejects.toMatchObject({ - existingLeaseId: first.lease.id, - }); - - expect(requested).toEqual([]); - expect(harness.requestIds).toHaveLength(mintedBefore + 1); - expect(rejected).toEqual([ - expect.objectContaining({ - reason: "already-leased", - requestId: `req_${harness.requestIds[mintedBefore]}`, - requester: "agent", - }), - ]); - - // The same position in the sequence, on a request that passes admission: that id is the one - // the stored request gets, and nothing mints a second. - const mintedBeforeAdmitted = harness.requestIds.length; - await harness.coordinator.request(request, owner("other")); - expect(harness.requestIds).toHaveLength(mintedBeforeAdmitted + 1); - expect(requested).toEqual([`req_${harness.requestIds[mintedBeforeAdmitted]}`]); - }); - - it("gives a request refused as killed the id it would have been stored under", async () => { - const harness = await createHarness(); - await harness.coordinator.beginMaintenance(); - const rejected: Array<{ requestId: string; requester: string; reason: string }> = []; - harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - - await expect(harness.coordinator.request(request, owner("agent"))).rejects.toMatchObject({ - name: "NukeCancelledError", - }); - - expect(rejected).toEqual([ - expect.objectContaining({ - reason: "killed", - requestId: `req_${harness.requestIds[0]}`, - requester: "agent", - }), - ]); - }); -}); - -describe("LeaseAcquisitionCoordinator probes (ADR 0021)", () => { - const probe = (requesterId: string) => ({ - fleetRequestId: "req_gw1", - noWait: true, - ownerId: requesterId, - requesterId, - }); - - it("emits lease.declined, and no lease.rejected, for a probe refused killed at admission", async () => { - const harness = await createHarness(); - await harness.coordinator.beginMaintenance(); - const declined: unknown[] = []; - const rejected: unknown[] = []; - harness.bus.subscribe("lease.declined", (envelope) => declined.push(envelope.payload)); - harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - - await expect(harness.coordinator.request(request, probe("gw:one"))).rejects.toMatchObject({ - name: "NukeCancelledError", - }); - - expect(declined).toEqual([ - { - fleetRequestId: "req_gw1", - reason: "killed", - requestId: `req_${harness.requestIds[0]}`, - requestSpec: request, - requester: "gw:one", - }, - ]); - expect(rejected).toEqual([]); - }); - - it("emits lease.declined, and no lease.rejected, for a probe in progress when beginMaintenance runs", async () => { - const harness = await createHarness(); - harness.driver.hangMakeReady(); - const declined: unknown[] = []; - const rejected: unknown[] = []; - harness.bus.subscribe("lease.declined", (envelope) => declined.push(envelope.payload)); - harness.bus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - const acquisition = harness.coordinator.request(request, probe("gw:one")); - void acquisition.catch(() => undefined); - await settle(); - - const maintenance = harness.coordinator.beginMaintenance(); - await flush(); - harness.driver.releaseMakeReady(); - await maintenance; - - await expect(acquisition).rejects.toMatchObject({ name: "NukeCancelledError" }); - expect(declined).toEqual([ - expect.objectContaining({ fleetRequestId: "req_gw1", reason: "killed", requester: "gw:one" }), - ]); - expect(rejected).toEqual([]); - await harness.coordinator.endMaintenance(); - }); -}); - -describe("LeaseAcquisitionCoordinator waiting demand", () => { - it("lists a request waiting for a device with what a device must satisfy, and drops it once it is granted", async () => { - const harness = await createHarness(); - const held = await harness.coordinator.request(request, { - ownerId: "holder", - requesterId: "holder", - }); - expect(harness.coordinator.waitingDemand()).toEqual([]); - - const waiting = harness.coordinator.request(request, { - ownerId: "waiter", - requesterId: "waiter", - }); - await flush(); - - expect(harness.coordinator.waitingDemand()).toEqual([ - { - classOf: expect.any(Function), - inFlight: false, - mode: "full", - platform: "ios", - requirement: { - imageTag: undefined, - osVersion: { kind: "exact", version: "26.5" }, - platform: "ios", - target: { kind: "model", model: "iPhone 16" }, - }, - spec: request, - }, - ]); - - await harness.registry.beginRelease(held.lease.id); - await harness.registry.transitionDevice(held.device.id, "ready", { - event: "device.reclaimed", - payload: { deviceId: held.device.id, duration: 0, strategy: "wipe" }, - }); - harness.coordinator.kick(); - await waiting; - expect(harness.coordinator.waitingDemand()).toEqual([]); - }); - - it("reports the mode a request resolved to, and marks a request whose device work has begun as in flight", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { provision: 50 }, - platform: "ios", - slimmableOsVersions: ["26.5"], - }); - const harness = await createHarness({ drivers: [driver] }); - const pending = harness.coordinator.request( - { ...request, mode: "slim" }, - { ownerId: "slim", requesterId: "slim" }, - ); - await flush(); - - expect(harness.coordinator.waitingDemand()).toMatchObject([{ inFlight: true, mode: "slim" }]); - - clock.advance(50); - await pending; - expect(harness.coordinator.waitingDemand()).toEqual([]); - }); - - it("reports maintenance active between beginMaintenance and endMaintenance", async () => { - const harness = await createHarness(); - expect(harness.coordinator.maintenanceActive).toBe(false); - - await harness.coordinator.beginMaintenance(); - expect(harness.coordinator.maintenanceActive).toBe(true); - - await harness.coordinator.endMaintenance(); - expect(harness.coordinator.maintenanceActive).toBe(false); - }); -}); - -describe("LeaseAcquisitionCoordinator: a request naming a baseline", () => { - it("plans against the version current at its decision, so a save between the resolve and the plan grants an idle device of the new version", async () => { - const current = (version: number) => ({ - mode: "full" as const, - model: request.model, - name: "checkout", - osVersion: request.osVersion, - platform: request.platform, - sourceId: `source-${version}`, - version, - }); - const reads = [current(1)]; - const harness = await createHarness({ - baselines: { current: () => reads.shift() ?? current(2) }, - maxDevices: 2, - }); - const idleV2 = await seedReady(harness, { - ...request, - baseline: { name: "checkout", version: 2 }, - }); - - const granted = await harness.coordinator.request( - { baseline: "checkout", platform: "ios" }, - { ownerId: "agent", requesterId: "agent" }, - ); - - expect(granted.device.id).toBe(idleV2.id); - expect(granted.device.spec.baseline).toEqual({ name: "checkout", version: 2 }); - }); - - it("ends a request whose baseline was removed while it waited as UnknownBaselineError at once, without a retry", async () => { - const harness = await createHarness({ - baselines: { - current: () => ({ - mode: "full" as const, - model: request.model, - name: "checkout", - osVersion: request.osVersion, - platform: request.platform, - sourceId: "source-1", - version: 1, - }), - }, - maxDevices: 1, - }); - - const outcome = await harness.coordinator - .request( - { baseline: "checkout", platform: "ios" }, - { ownerId: "agent", requesterId: "agent" }, - ) - .then( - () => "granted", - (error: unknown) => (error as Error).name, - ); - - expect(outcome).toBe("UnknownBaselineError"); - }); }); diff --git a/src/leasing/test-lease-acquisition-commits.ts b/src/leasing/test-lease-acquisition-commits.ts new file mode 100644 index 000000000..d0e987ad4 --- /dev/null +++ b/src/leasing/test-lease-acquisition-commits.ts @@ -0,0 +1,34 @@ +import { vi } from "vitest"; + +import type { StateDatabase } from "../state/index.js"; + +interface StoredByCommit { + readonly leaseRequesters: readonly string[]; + readonly requestRequesters: readonly string[]; +} + +/** + * why: runs `inspect` inside every registry commit, once its rows are written and before it commits: it + * sees what the commit stores, and throwing from it rolls that commit back. + */ +export function inspectCommits( + harness: { readonly state: StateDatabase }, + inspect: (stored: StoredByCommit) => void, +): void { + const original = harness.state.transaction.bind(harness.state); + vi.spyOn(harness.state, "transaction").mockImplementation((work) => + original(() => { + const result = work(); + const requesters = (table: string) => + harness.state.database + .prepare(`SELECT requester_id FROM ${table}`) + .all() + .map((row) => String(row["requester_id"])); + inspect({ + leaseRequesters: requesters("leases"), + requestRequesters: requesters("lease_requests"), + }); + return result; + }), + ); +} From 6ba191a333176436257d8ad17284e0c500c791dc Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:26:27 +0100 Subject: [PATCH 39/83] test(leasing): split physical-device-watcher.test.ts by topic --- .../physical-device-watcher-acting.test.ts | 509 ++++++++ ...l-device-watcher-restarts-and-nuke.test.ts | 313 +++++ src/leasing/physical-device-watcher.test.ts | 1114 +---------------- src/leasing/test-physical-device-watcher.ts | 327 +++++ 4 files changed, 1176 insertions(+), 1087 deletions(-) create mode 100644 src/leasing/physical-device-watcher-acting.test.ts create mode 100644 src/leasing/physical-device-watcher-restarts-and-nuke.test.ts create mode 100644 src/leasing/test-physical-device-watcher.ts diff --git a/src/leasing/physical-device-watcher-acting.test.ts b/src/leasing/physical-device-watcher-acting.test.ts new file mode 100644 index 000000000..84d55ddfb --- /dev/null +++ b/src/leasing/physical-device-watcher-acting.test.ts @@ -0,0 +1,509 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as coreModule from "../core/index.js"; +import { type LeaseGrant } from "../core/index.js"; +import { FakeClock, JsonLinesLogger, type Logger, MemoryLogSink } from "../ports/index.js"; +import { + PROBE_MS, + absentHarness, + absentReasonOf, + answer, + asking, + back, + createHarness, + deferred, + enrollIphone, + eventsOf, + flush, + gone, + harnesses, + holdNextReclaim, + isSettled, + markAbsent, + physicalRequest, + readsOf, + reclaimCalls, + recordStateWrites, + reopen, + startWatcher, + stateOf, + tick, + watcherOf, +} from "./test-physical-device-watcher.js"; + +const latestPresence = (driverDeviceId: string): unknown => + (coreModule as unknown as Record unknown>)["latestPresence"]?.( + driverDeviceId, + ); + +async function outcome(promise: Promise): Promise { + return Promise.race([ + promise.catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))), + flush().then(() => "pending" as const), + ]); +} + +function capturingLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { + const sink = new MemoryLogSink(); + return { logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), sink }; +} + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + watcherOf(harness)?.dispose(); + harness.engine.dispose(); + } +}); + +describe("PhysicalDeviceWatcher acting on a device", () => { + it("an absent device read not present with another reason gets its absentReason rewritten in one registry write, with no event; a read with the same reason writes nothing", async () => { + const { device, harness } = await absentHarness("locked"); + gone(harness, "UDID-A", "locked"); + const writes = recordStateWrites(harness); + startWatcher(harness); + + await tick(harness); + expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); + expect(writes).toHaveLength(0); + expect(absentReasonOf(harness, device)).toBe("locked"); + + gone(harness, "UDID-A", "not-present"); + const eventsBefore = harness.bus.replay().length; + await tick(harness); + expect(absentReasonOf(harness, device)).toBe("not-present"); + expect(writes).toHaveLength(1); + expect(harness.bus.replay()).toHaveLength(eventsBefore); + + await tick(harness); + expect(writes).toHaveLength(1); + expect(absentReasonOf(harness, device)).toBe("not-present"); + }); + + it("a device under an operation claim read not present past the grace period keeps its state; the first tick after the claim is released moves it", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + gone(harness, "UDID-A"); + const claim = harness.engine.core.claims.tryClaim(device.id, "enrollment"); + expect(claim).toBeDefined(); + startWatcher(harness); + + await tick(harness, 3); + expect(readsOf(harness.driver)).toHaveLength(3); + expect(stateOf(harness, device)).toBe("ready"); + + claim?.release(); + await tick(harness); + expect(stateOf(harness, device)).toBe("absent"); + }); + + it("a grant that commits between the watcher's read and its write leaves the new lease in place", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + startWatcher(harness); + gone(harness, "UDID-A"); + await tick(harness); + expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); + + back(harness, "UDID-A"); + const gate = deferred(); + vi.spyOn(harness.driver, "listPresentPhysical").mockImplementationOnce(async (ids) => { + await gate.promise; + return answer( + ids.map((driverDeviceId) => ({ driverDeviceId, present: false, reason: "not-present" })), + ); + }); + harness.clock.advance(PROBE_MS); + await flush(); + const granted = await outcome(harness.engine.request(physicalRequest, asking("agent"))); + expect(granted).toMatchObject({ device: { id: device.id } }); + gate.resolve(); + await flush(); + + expect(stateOf(harness, device)).toBe("leased"); + expect(harness.registry.snapshot.leases).toMatchObject([{ deviceId: device.id }]); + expect(eventsOf(harness, "lease.released")).toEqual([]); + }); + + it("the return reclaim holds a reclaim claim from before the absent → reclaiming write until it settles, and releases it when the reclaim throws", async () => { + const held = await absentHarness("not-present"); + back(held.harness, "UDID-A"); + const writes = recordStateWrites(held.harness); + const gate = holdNextReclaim(held.harness); + startWatcher(held.harness); + + await tick(held.harness); + expect(reclaimCalls(held.harness.driver)).toHaveLength(1); + expect(held.harness.engine.core.claims.claim(held.device.id)?.kind).toBe("reclaim"); + expect(stateOf(held.harness, held.device)).toBe("reclaiming"); + expect(writes.find((write) => write.states[held.device.id] === "reclaiming")).toMatchObject({ + claims: { [held.device.id]: "reclaim" }, + }); + gate.resolve(); + await flush(); + expect(held.harness.engine.core.claims.isClaimed(held.device.id)).toBe(false); + expect(stateOf(held.harness, held.device)).toBe("ready"); + + const thrown = await absentHarness("not-present"); + back(thrown.harness, "UDID-A"); + thrown.harness.driver.failOn("reclaim", 1, new Error("usb dropped")); + startWatcher(thrown.harness); + + await tick(thrown.harness); + expect(reclaimCalls(thrown.harness.driver)).toHaveLength(1); + expect(thrown.harness.engine.core.claims.isClaimed(thrown.device.id)).toBe(false); + }); + + it("an absent device read present while device add holds its claim gets no write and no reclaim that tick", async () => { + const { device, harness } = await absentHarness("not-present"); + back(harness, "UDID-A"); + const claim = harness.engine.core.claims.tryClaim(device.id, "enrollment"); + expect(claim).toBeDefined(); + const writes = recordStateWrites(harness); + startWatcher(harness); + + await tick(harness); + expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); + expect(writes).toHaveLength(0); + expect(reclaimCalls(harness.driver)).toHaveLength(0); + expect(stateOf(harness, device)).toBe("absent"); + + claim?.release(); + await tick(harness); + expect(reclaimCalls(harness.driver)).toHaveLength(1); + }); + + it("an absent device read present is returned and reclaimed on that tick, at most one health.probeIntervalMs after it was scripted present", async () => { + const { device, harness } = await absentHarness("not-present"); + startWatcher(harness); + harness.clock.advance(10_000); + await flush(); + + back(harness, "UDID-A"); + harness.clock.advance(PROBE_MS); + await flush(); + + expect(eventsOf(harness, "device.returned")).toHaveLength(1); + expect(reclaimCalls(harness.driver)).toHaveLength(1); + expect(stateOf(harness, device)).toBe("ready"); + }); + + it("a device the fake driver reports present again has a latest read of present within one health.probeIntervalMs", async () => { + const harness = await createHarness(); + await enrollIphone(harness, "UDID-LATEST-WATCH"); + startWatcher(harness); + gone(harness, "UDID-LATEST-WATCH", "locked"); + await tick(harness); + expect(latestPresence("UDID-LATEST-WATCH")).toStrictEqual({ present: false, reason: "locked" }); + + back(harness, "UDID-LATEST-WATCH"); + harness.clock.advance(PROBE_MS); + await flush(); + + expect(latestPresence("UDID-LATEST-WATCH")).toStrictEqual({ present: true }); + }); + + it("two ready devices read not present for the grace period both become absent, each judged on its own state", async () => { + const harness = await createHarness(); + const first = await enrollIphone(harness, "UDID-A"); + const second = await enrollIphone(harness, "UDID-B"); + gone(harness, "UDID-A", "locked"); + gone(harness, "UDID-B", "untrusted"); + startWatcher(harness); + + await tick(harness, 2); + + expect([stateOf(harness, first), stateOf(harness, second)]).toEqual(["absent", "absent"]); + expect([absentReasonOf(harness, first), absentReasonOf(harness, second)]).toEqual([ + "locked", + "untrusted", + ]); + }); + + it("a tick where one device's action throws (its registry write fails) logs it, still acts on the other devices, and arms the next tick; that device's count is not reset, so the next not-present read acts on it again", async () => { + const { logger, sink } = capturingLogger(); + const harness = await createHarness({ logger }); + const first = await enrollIphone(harness, "UDID-A"); + const second = await enrollIphone(harness, "UDID-B"); + gone(harness, "UDID-A"); + gone(harness, "UDID-B"); + startWatcher(harness); + await tick(harness); + vi.spyOn(harness.state, "transaction").mockImplementationOnce(() => { + throw new Error("disk full"); + }); + const problems = () => + sink.records.filter((record) => record.level === "warn" || record.level === "error"); + const problemsBefore = problems().length; + + await tick(harness); + const states = [stateOf(harness, first), stateOf(harness, second)]; + expect(states.filter((state) => state === "absent")).toHaveLength(1); + expect(states.filter((state) => state === "ready")).toHaveLength(1); + expect(problems().length).toBeGreaterThan(problemsBefore); + const failed = [first, second].find((device) => stateOf(harness, device) === "ready"); + expect( + problems().filter((record) => record.message === "physical device action failed"), + ).toMatchObject([ + { fields: { deviceId: failed?.id, error: expect.anything() }, level: "warn" }, + ]); + + await tick(harness); + expect(stateOf(harness, first)).toBe("absent"); + expect(stateOf(harness, second)).toBe("absent"); + }); + + it("a ready device read not present once and then present wakes the queue: a physical request waiting for it is granted on that tick", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + gone(harness, "UDID-A"); + const waiting = harness.engine.request(physicalRequest, asking("agent")); + expect(await outcome(waiting)).toBe("pending"); + startWatcher(harness); + + await tick(harness); + expect(readsOf(harness.driver)).toContainEqual(["UDID-A"]); + back(harness, "UDID-A"); + expect(await outcome(waiting)).toBe("pending"); + + await tick(harness); + expect(await outcome(waiting)).toMatchObject({ device: { id: device.id } }); + }); + + it("a tick whose read resolves present after dispose does not wake the queue: a physical request waiting for the device stays pending", async () => { + const harness = await createHarness(); + await enrollIphone(harness, "UDID-A"); + gone(harness, "UDID-A"); + const waiting = harness.engine.request(physicalRequest, asking("agent")); + expect(await outcome(waiting)).toBe("pending"); + startWatcher(harness); + await tick(harness); + back(harness, "UDID-A"); + harness.driver.holdPresenceReads("UDID-A"); + harness.clock.advance(PROBE_MS); + await flush(); + expect(readsOf(harness.driver)).toHaveLength(3); + + watcherOf(harness)?.dispose(); + harness.driver.releasePresenceReads(); + + expect(await outcome(waiting)).toBe("pending"); + }); + + it("a tick whose read resolves after dispose writes nothing and starts no reclaim, and leasing.settle() resolves only after that tick", async () => { + const ready = await createHarness(); + const readyDevice = await enrollIphone(ready, "UDID-A"); + gone(ready, "UDID-A"); + startWatcher(ready); + await tick(ready); + const readyWrites = recordStateWrites(ready); + ready.driver.holdPresenceReads("UDID-A"); + ready.clock.advance(PROBE_MS); + await flush(); + expect(readsOf(ready.driver)).toHaveLength(2); + + watcherOf(ready)?.dispose(); + const settling = ready.engine.leasing.settle(); + expect(await isSettled(settling)).toBe(false); + ready.driver.releasePresenceReads(); + expect(await isSettled(settling)).toBe(true); + expect(stateOf(ready, readyDevice)).toBe("ready"); + expect(readyWrites).toHaveLength(0); + + const { device, harness } = await absentHarness("not-present"); + back(harness, "UDID-A"); + const writes = recordStateWrites(harness); + harness.driver.holdPresenceReads("UDID-A"); + startWatcher(harness); + harness.clock.advance(PROBE_MS); + await flush(); + expect(readsOf(harness.driver)).toHaveLength(1); + + watcherOf(harness)?.dispose(); + harness.driver.releasePresenceReads(); + await flush(); + expect(stateOf(harness, device)).toBe("absent"); + expect(writes).toHaveLength(0); + expect(reclaimCalls(harness.driver)).toHaveLength(0); + }); + + it("a platform with no driver changes no count and no state, and the other platform is still read", async () => { + const harness = await createHarness(); + const iphone = await enrollIphone(harness, "UDID-A"); + const pixel = await harness.registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: "EMU-1", + model: "Pixel 8", + osVersion: "14", + platform: "android", + }); + gone(harness, "UDID-A"); + startWatcher(harness); + + await tick(harness, 10); + + expect(stateOf(harness, iphone)).toBe("absent"); + expect(stateOf(harness, pixel)).toBe("ready"); + expect(readsOf(harness.driver)[0]).toEqual(["UDID-A"]); + }); + + it("a listing entry whose ID has no record causes no write and no driver call but the listing", async () => { + const harness = await createHarness(); + await enrollIphone(harness, "UDID-A"); + const listing = vi.spyOn(harness.driver, "listPresentPhysical").mockResolvedValue( + answer([ + { driverDeviceId: "UDID-A", osVersion: "18.4", present: true }, + { driverDeviceId: "UDID-GHOST-1", osVersion: "18.4", present: true }, + { driverDeviceId: "UDID-GHOST-2", present: false, reason: "locked" }, + ]), + ); + const writes = recordStateWrites(harness); + startWatcher(harness); + + await tick(harness, 3); + + expect(listing.mock.calls.length).toBeGreaterThan(0); + expect(harness.driver.calls.map((call) => call.operation)).toEqual([]); + expect(writes).toHaveLength(0); + }); + + it("a ready device that stays not present for health.stableObservations reads emits one device.absence-detected with previousState ready and the last read's reason", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + startWatcher(harness); + gone(harness, "UDID-A", "locked"); + + await tick(harness, 2); + + expect(eventsOf(harness, "device.absence-detected")).toMatchObject([ + { payload: { absentReason: "locked", deviceId: device.id, previousState: "ready" } }, + ]); + }); + + it("a ready device read present without a miss does not wake the queue", async () => { + const harness = await createHarness(); + await enrollIphone(harness, "UDID-A"); + const kick = vi.spyOn( + harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, + "kick", + ); + startWatcher(harness); + + await tick(harness, 2); + + expect(readsOf(harness.driver)).toHaveLength(2); + expect(kick).not.toHaveBeenCalled(); + }); + + it("a ready device read not present once and then present wakes the queue once, not again on the next present read", async () => { + const harness = await createHarness(); + await enrollIphone(harness, "UDID-A"); + const kick = vi.spyOn( + harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, + "kick", + ); + startWatcher(harness); + gone(harness, "UDID-A"); + await tick(harness); + expect(kick).not.toHaveBeenCalled(); + + back(harness, "UDID-A"); + await tick(harness); + expect(kick).toHaveBeenCalledTimes(1); + + await tick(harness); + expect(kick).toHaveBeenCalledTimes(1); + }); + + it("a leased device read not present once and then present does not wake the queue", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + startWatcher(harness); + await harness.engine.request(physicalRequest, asking("agent")); + expect(stateOf(harness, device)).toBe("leased"); + const kick = vi.spyOn( + harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, + "kick", + ); + gone(harness, "UDID-A"); + await tick(harness); + back(harness, "UDID-A"); + + await tick(harness); + + expect(stateOf(harness, device)).toBe("leased"); + expect(kick).not.toHaveBeenCalled(); + }); + + it("an absent device read present returns with the osVersion of the read, or with the one on its record when the read has none", async () => { + const withVersion = await absentHarness("not-present"); + vi.spyOn(withVersion.harness.driver, "listPresentPhysical").mockResolvedValue( + answer([{ driverDeviceId: "UDID-A", osVersion: "18.5", present: true }]), + ); + startWatcher(withVersion.harness); + await tick(withVersion.harness); + expect(eventsOf(withVersion.harness, "device.returned")).toMatchObject([ + { payload: { deviceId: withVersion.device.id, osVersion: "18.5" } }, + ]); + + const withoutVersion = await absentHarness("not-present"); + vi.spyOn(withoutVersion.harness.driver, "listPresentPhysical").mockResolvedValue( + answer([{ driverDeviceId: "UDID-A", present: true }]), + ); + startWatcher(withoutVersion.harness); + await tick(withoutVersion.harness); + expect(eventsOf(withoutVersion.harness, "device.returned")).toMatchObject([ + { payload: { deviceId: withoutVersion.device.id, osVersion: "18.4" } }, + ]); + }); + + it("a tick whose watcher is disposed while it acts on one device acts on no later device of that tick", async () => { + const options = { health: { stableObservations: 1 } }; + const first = await createHarness(options); + const missing = await enrollIphone(first, "UDID-A"); + const returned = await enrollIphone(first, "UDID-B"); + const harness = await reopen( + first, + (database) => markAbsent(database, "UDID-B", "not-present"), + options, + ); + gone(harness, "UDID-A"); + back(harness, "UDID-B"); + const original = harness.state.transaction.bind(harness.state); + vi.spyOn(harness.state, "transaction").mockImplementationOnce((work) => { + watcherOf(harness)?.dispose(); + return original(work); + }); + startWatcher(harness); + + await tick(harness); + + expect(stateOf(harness, missing)).toBe("absent"); + expect(stateOf(harness, returned)).toBe("absent"); + expect(reclaimCalls(harness.driver)).toEqual([]); + expect(eventsOf(harness, "device.returned")).toEqual([]); + }); + + it("a tick whose watcher is disposed while it acts on one device does not wake the queue for a later device of that tick", async () => { + const first = await createHarness(); + const missing = await enrollIphone(first, "UDID-A"); + const flickering = await enrollIphone(first, "UDID-B"); + gone(first, "UDID-A"); + gone(first, "UDID-B"); + const waiting = first.engine.request(physicalRequest, asking("agent")); + expect(await outcome(waiting)).toBe("pending"); + startWatcher(first); + await tick(first); + back(first, "UDID-B"); + const original = first.state.transaction.bind(first.state); + vi.spyOn(first.state, "transaction").mockImplementationOnce((work) => { + watcherOf(first)?.dispose(); + return original(work); + }); + + await tick(first); + + expect(stateOf(first, missing)).toBe("absent"); + expect(stateOf(first, flickering)).toBe("ready"); + expect(await outcome(waiting)).toBe("pending"); + }); +}); diff --git a/src/leasing/physical-device-watcher-restarts-and-nuke.test.ts b/src/leasing/physical-device-watcher-restarts-and-nuke.test.ts new file mode 100644 index 000000000..a55639735 --- /dev/null +++ b/src/leasing/physical-device-watcher-restarts-and-nuke.test.ts @@ -0,0 +1,313 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { type PhysicalDevice } from "../core/index.js"; +import { + type Harness, + absentHarness, + absentReasonOf, + asking, + back, + createHarness, + enrollIphone, + eventsOf, + flush, + gone, + harnesses, + holdNextReclaim, + isSettled, + markAbsent, + physicalRequest, + readsOf, + reclaimCalls, + recordStateWrites, + reopen, + startWatcher, + stateOf, + tick, + watcherOf, +} from "./test-physical-device-watcher.js"; + +const DAY_MS = 24 * 60 * 60_000; + +afterEach(() => { + for (const harness of harnesses.splice(0)) { + watcherOf(harness)?.dispose(); + harness.engine.dispose(); + } +}); + +describe("PhysicalDeviceWatcher around restarts and doctor", () => { + it("a device left reclaiming by a killed daemon, read present, whose re-run reclaim fails: still present, it is quarantined with device.quarantined only; read not present after the failure, it is absent with device.absence-detected previousState reclaiming", async () => { + const restarted = async () => { + const first = await createHarness(); + const device = await enrollIphone(first, "UDID-A"); + await first.registry.transitionDevice(device.id, "leased"); + await first.registry.transitionDevice(device.id, "reclaiming"); + const harness = await reopen(first, () => undefined); + back(harness, "UDID-A"); + await harness.engine.convergeRunningCapacity(); + return { device, harness }; + }; + + const present = await restarted(); + vi.spyOn(present.harness.driver, "reclaim").mockRejectedValueOnce( + new Error("uninstall failed"), + ); + startWatcher(present.harness); + await tick(present.harness); + expect(stateOf(present.harness, present.device)).toBe("quarantined"); + expect(eventsOf(present.harness, "device.quarantined")).toHaveLength(1); + expect(eventsOf(present.harness, "device.purge-failed")).toEqual([]); + expect(eventsOf(present.harness, "device.absence-detected")).toEqual([]); + + const unplugged = await restarted(); + vi.spyOn(unplugged.harness.driver, "reclaim").mockImplementationOnce(async () => { + gone(unplugged.harness, "UDID-A"); + throw new Error("usb lost"); + }); + startWatcher(unplugged.harness); + await tick(unplugged.harness); + expect(stateOf(unplugged.harness, unplugged.device)).toBe("absent"); + expect(eventsOf(unplugged.harness, "device.absence-detected")).toMatchObject([ + { + payload: { + absentReason: "not-present", + deviceId: unplugged.device.id, + previousState: "reclaiming", + }, + }, + ]); + expect(eventsOf(unplugged.harness, "device.quarantined")).toEqual([]); + expect(eventsOf(unplugged.harness, "device.purge-failed")).toEqual([]); + }); + + it("a physical device that entered reclaiming in this process and holds no claim is never reclaimed by the watcher", async () => { + const first = await createHarness(); + const restartedDevice = await enrollIphone(first, "UDID-A"); + const processDevice = await enrollIphone(first, "UDID-B"); + await first.registry.transitionDevice(restartedDevice.id, "leased"); + await first.registry.transitionDevice(restartedDevice.id, "reclaiming"); + const harness = await reopen(first, () => undefined); + back(harness, "UDID-A"); + back(harness, "UDID-B"); + await harness.engine.convergeRunningCapacity(); + await harness.registry.transitionDevice(processDevice.id, "leased"); + await harness.registry.transitionDevice(processDevice.id, "reclaiming"); + startWatcher(harness); + + await tick(harness, 3); + + expect(stateOf(harness, restartedDevice)).toBe("ready"); + expect(stateOf(harness, processDevice)).toBe("reclaiming"); + expect( + reclaimCalls(harness.driver).filter((call) => + JSON.stringify(call.arguments).includes("UDID-B"), + ), + ).toEqual([]); + }); + + it("a device left reclaiming by a killed daemon that left reclaiming before the first tick, and entered it again in this process, is never reclaimed by the watcher", async () => { + const first = await createHarness(); + const device = await enrollIphone(first, "UDID-A"); + await first.registry.transitionDevice(device.id, "leased"); + await first.registry.transitionDevice(device.id, "reclaiming"); + const harness = await reopen(first, () => undefined); + back(harness, "UDID-A"); + await harness.engine.convergeRunningCapacity(); + await harness.registry.transitionDevice(device.id, "ready", { + event: "device.reclaimed", + payload: { deviceId: device.id, duration: 0, strategy: "uninstall" }, + }); + startWatcher(harness); + await tick(harness); + expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); + expect(reclaimCalls(harness.driver)).toEqual([]); + + await harness.registry.transitionDevice(device.id, "leased"); + await harness.registry.transitionDevice(device.id, "reclaiming"); + await tick(harness, 2); + + expect(readsOf(harness.driver)).toHaveLength(3); + expect(stateOf(harness, device)).toBe("reclaiming"); + expect(reclaimCalls(harness.driver)).toEqual([]); + }); + + it("a device left reclaiming by a killed daemon that doctor --fix quarantines before the watcher's tick stays quarantined, and the watcher runs no reclaim on it", async () => { + const options = { quarantineBackoffMs: 10 * DAY_MS }; + const first = await createHarness(options); + const device = await enrollIphone(first, "UDID-A"); + await first.registry.transitionDevice(device.id, "leased"); + await first.registry.transitionDevice(device.id, "reclaiming"); + const harness = await reopen( + first, + (database) => { + database.exec("UPDATE devices SET last_lease_ended_at = 1"); + }, + options, + ); + await harness.engine.convergeRunningCapacity(); + harness.clock.advance(DAY_MS); + await harness.engine.core.doctor.reconcile({ fix: true }); + expect(stateOf(harness, device)).toBe("quarantined"); + back(harness, "UDID-A"); + startWatcher(harness); + + await tick(harness, 3); + + expect(readsOf(harness.driver)).toContainEqual(["UDID-A"]); + expect(stateOf(harness, device)).toBe("quarantined"); + expect(reclaimCalls(harness.driver)).toEqual([]); + }); +}); + +describe("PhysicalDeviceWatcher and nuke maintenance", () => { + it("the device-lost ending waits while nuke maintenance is active, and runs when it ends", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + startWatcher(harness); + await harness.engine.request(physicalRequest, asking("agent")); + gone(harness, "UDID-A"); + const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; + await maintenance.beginMaintenance(); + + await tick(harness, 3); + expect(readsOf(harness.driver).length).toBeGreaterThan(0); + expect(stateOf(harness, device)).toBe("leased"); + expect(harness.registry.snapshot.leases).toHaveLength(1); + + await maintenance.endMaintenance(); + await flush(); + await tick(harness); + expect(stateOf(harness, device)).toBe("absent"); + expect(harness.registry.snapshot.leases).toEqual([]); + expect(eventsOf(harness, "lease.released")).toMatchObject([ + { payload: { reason: "device-lost" } }, + ]); + }); + + it("while nuke maintenance is active, a ready device past the grace period stays ready, an absent device read with another reason keeps its old one, and an absent device read present gets no write and no reclaim; all three are acted on once maintenance ends", async () => { + const options = { health: { stableObservations: 3 } }; + const first = await createHarness(options); + const ready = await enrollIphone(first, "UDID-R"); + const reasoned = await enrollIphone(first, "UDID-X"); + const returned = await enrollIphone(first, "UDID-Y"); + const harness = await reopen( + first, + (database) => { + markAbsent(database, "UDID-X", "locked"); + markAbsent(database, "UDID-Y", "locked"); + }, + options, + ); + gone(harness, "UDID-R"); + gone(harness, "UDID-X", "locked"); + gone(harness, "UDID-Y", "locked"); + startWatcher(harness); + await tick(harness, 2); + const writes = recordStateWrites(harness); + gone(harness, "UDID-X", "not-present"); + back(harness, "UDID-Y"); + const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; + await maintenance.beginMaintenance(); + + await tick(harness); + expect(readsOf(harness.driver)).toHaveLength(3); + expect(stateOf(harness, ready)).toBe("ready"); + expect(absentReasonOf(harness, reasoned)).toBe("locked"); + expect(stateOf(harness, returned)).toBe("absent"); + expect(writes).toHaveLength(0); + expect(reclaimCalls(harness.driver)).toHaveLength(0); + + await maintenance.endMaintenance(); + await flush(); + await tick(harness); + expect(stateOf(harness, ready)).toBe("absent"); + expect(absentReasonOf(harness, reasoned)).toBe("not-present"); + expect(reclaimCalls(harness.driver)).toHaveLength(1); + expect(["reclaiming", "ready"]).toContain(stateOf(harness, returned)); + }); + + it.each([ + { + name: "nuke's lease maintenance, begun while it runs,", + start: (harness: Harness) => + harness.engine.leasing.corePorts.leaseMaintenance.leases.beginMaintenance(), + finish: async (harness: Harness) => { + await harness.engine.leasing.corePorts.leaseMaintenance.leases.endMaintenance(); + }, + }, + { + name: "leasing.settle()", + start: (harness: Harness) => harness.engine.leasing.settle(), + finish: async (harness: Harness, device: PhysicalDevice) => { + expect(stateOf(harness, device)).toBe("ready"); + }, + }, + ])("$name resolves only after a return reclaim the watcher started has settled", async (row) => { + const { device, harness } = await absentHarness("not-present"); + back(harness, "UDID-A"); + const gate = holdNextReclaim(harness); + startWatcher(harness); + await tick(harness); + expect(reclaimCalls(harness.driver)).toHaveLength(1); + + const waiting = row.start(harness); + expect(await isSettled(waiting)).toBe(false); + + gate.resolve(); + expect(await isSettled(waiting)).toBe(true); + await row.finish(harness, device); + }); + + it("an absentReason rewrite held back by nuke maintenance is dropped when the watcher is disposed before maintenance ends", async () => { + const { device, harness } = await absentHarness("locked"); + gone(harness, "UDID-A", "not-present"); + startWatcher(harness); + const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; + await maintenance.beginMaintenance(); + await tick(harness); + expect(readsOf(harness.driver)).toHaveLength(1); + + watcherOf(harness)?.dispose(); + await maintenance.endMaintenance(); + await flush(); + + expect(absentReasonOf(harness, device)).toBe("locked"); + }); + + it("a return reclaim waiting on nuke maintenance when the watcher is disposed never starts: the device stays absent", async () => { + const { device, harness } = await absentHarness("not-present"); + back(harness, "UDID-A"); + startWatcher(harness); + const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; + await maintenance.beginMaintenance(); + await tick(harness); + expect(readsOf(harness.driver)).toHaveLength(1); + + watcherOf(harness)?.dispose(); + await maintenance.endMaintenance(); + await flush(); + + expect(stateOf(harness, device)).toBe("absent"); + expect(reclaimCalls(harness.driver)).toHaveLength(0); + }); + + it("a device-lost ending waiting on nuke maintenance when the watcher is disposed leaves the lease in place", async () => { + const harness = await createHarness(); + const device = await enrollIphone(harness, "UDID-A"); + startWatcher(harness); + await harness.engine.request(physicalRequest, asking("agent")); + gone(harness, "UDID-A"); + await tick(harness); + const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; + await maintenance.beginMaintenance(); + await tick(harness); + expect(readsOf(harness.driver).length).toBeGreaterThan(1); + + watcherOf(harness)?.dispose(); + await maintenance.endMaintenance(); + await flush(); + + expect(stateOf(harness, device)).toBe("leased"); + expect(harness.registry.snapshot.leases).toHaveLength(1); + }); +}); diff --git a/src/leasing/physical-device-watcher.test.ts b/src/leasing/physical-device-watcher.test.ts index f5a56fb86..095002f9e 100644 --- a/src/leasing/physical-device-watcher.test.ts +++ b/src/leasing/physical-device-watcher.test.ts @@ -1,361 +1,25 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import * as coreModule from "../core/index.js"; +import { Registry } from "../core/index.js"; import { - type Config, - type DeviceRequest, - type LeaseGrant, - type PhysicalDevice, - Registry, -} from "../core/index.js"; -import { - FakeDriver, - loadTestRegistry, - openTestState, - testComponentWiring, - testStateOf, -} from "../core/testing.js"; -import type { StateDatabase } from "../state/index.js"; -import { EventBus } from "../bus/index.js"; -import { - FakeClock, - FakeSystemStats, - JsonLinesLogger, - type Logger, - MemoryLogSink, - type SqliteDatabase, -} from "../ports/index.js"; -import { createTestEngine, type TestEngine, type TestEngineConfig } from "./testing.js"; - -const latestPresence = (driverDeviceId: string): unknown => - (coreModule as unknown as Record unknown>)["latestPresence"]?.( - driverDeviceId, - ); - -const gibibyte = 1024 ** 3; -const PROBE_MS = 30_000; -const DAY_MS = 24 * 60 * 60_000; - -const physicalRequest = { - model: "iPhone 15 Pro", - physical: true, - platform: "ios", -} as const satisfies DeviceRequest; - -interface HarnessOptions { - readonly android?: boolean; - readonly health?: Partial; - readonly logger?: Logger; - readonly quarantineBackoffMs?: number; -} - -function config(options: HarnessOptions): TestEngineConfig { - const quarantineBackoffMs = options.quarantineBackoffMs ?? 30_000; - return { - devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, - health: { - enabled: true, - maxConcurrentRecoveries: 1, - maxRecoveryAttempts: 3, - probeIntervalMs: PROBE_MS, - recoveryBackoffMs: 5_000, - stableObservations: 2, - ...options.health, - }, - stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, - idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, - lease: { - defaultTtlMs: 3_600_000, - maxTtlMs: 14_400_000, - identity: { ios: "reusable", android: "reusable" }, - requestRetentionMs: 600_000, - endedIdRetentionMs: 600_000, - maxRequestRecords: 10_000, - maxPerRequester: 1, - endWaitMs: 600_000, - }, - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - }, - warmPool: { - enabled: false, - maxConcurrentBoots: 1, - reserveRunning: { android: 0, ios: 0 }, - targets: [], - quarantine: { - maxRetries: 3, - maxRetryBackoffMs: Math.max(300_000, quarantineBackoffMs), - retryBackoffMs: quarantineBackoffMs, - retryBackoffMultiplier: 2, - }, - }, - }; -} - -interface Harness { - readonly androidDriver: FakeDriver; - readonly bus: EventBus; - readonly clock: FakeClock; - readonly driver: FakeDriver; - readonly engine: TestEngine; - readonly registry: Registry; - readonly state: StateDatabase; -} - -const harnesses: Harness[] = []; - -async function createHarness( - options: HarnessOptions = {}, - state?: StateDatabase, -): Promise { - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const androidDriver = new FakeDriver({ - availableOsVersions: ["14"], - clock, - platform: "android", - }); - const drivers = options.android === true ? [driver, androidDriver] : [driver]; - let nextId = 1; - const stateDatabase = state ?? (await openTestState()); - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => `${nextId++}` }, - state: stateDatabase, - }); - const wiring = testComponentWiring({ clock, drivers, eventBus: bus, registry }); - const engine = createTestEngine({ - ...wiring, - clock, - config: config(options), - drivers, - eventBus: bus, - idGenerator: { generate: () => `request-${nextId++}` }, - ...(options.logger === undefined ? {} : { logger: options.logger }), - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - }); - const harness = { - androidDriver, - bus, - clock, - driver, - engine, - registry, - state: stateDatabase, - }; - harnesses.push(harness); - return harness; -} - -afterEach(() => { - for (const harness of harnesses.splice(0)) { - watcherOf(harness)?.dispose(); - harness.engine.dispose(); - } -}); - -interface WatcherHandle { - start(): void; - dispose(): void; -} - -function watcherOf(harness: Harness): WatcherHandle | undefined { - return (harness.engine.leasing as unknown as { physicalDeviceWatcher?: WatcherHandle }) - .physicalDeviceWatcher; -} - -function startWatcher(harness: Harness): void { - watcherOf(harness)?.start(); -} - -async function reopen( - first: Harness, - patch: (database: SqliteDatabase) => void, - options: HarnessOptions = {}, -): Promise { - patch(first.state.database); - first.engine.dispose(); - return createHarness(options, first.state); -} - -function markAbsent(database: SqliteDatabase, driverDeviceId: string, absentReason: string): void { - const { changes } = database - .prepare("UPDATE devices SET state = 'absent', absent_reason = ? WHERE driver_device_id = ?") - .run(absentReason, driverDeviceId); - if (changes === 0) throw new Error(`no record for ${driverDeviceId}`); -} - -async function enrollIphone(harness: Harness, driverDeviceId: string): Promise { - const device = await harness.registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId, - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - harness.driver.setPhysicalPresence(driverDeviceId, true); - return device; -} - -async function absentHarness( - reason: string, - options: HarnessOptions = {}, -): Promise<{ readonly device: PhysicalDevice; readonly harness: Harness }> { - const first = await createHarness(options); - const device = await enrollIphone(first, "UDID-A"); - const harness = await reopen( - first, - (database) => markAbsent(database, "UDID-A", reason), - options, - ); - return { device, harness }; -} - -function asking(requesterId: string) { - return { ownerId: requesterId, requesterId }; -} - -async function flush(): Promise { - for (let count = 0; count < 500; count += 1) { - await Promise.resolve(); - } -} - -async function tick(harness: Harness, times = 1): Promise { - for (let count = 0; count < times; count += 1) { - harness.clock.advance(PROBE_MS); - await flush(); - } -} - -async function outcome(promise: Promise): Promise { - return Promise.race([ - promise.catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))), - flush().then(() => "pending" as const), - ]); -} - -async function isSettled(promise: Promise): Promise { - return Promise.race([promise.then(() => true), flush().then(() => false)]); -} - -function deferred(): { readonly promise: Promise; readonly resolve: () => void } { - let resolve!: () => void; - const promise = new Promise((done) => { - resolve = done; - }); - return { promise, resolve }; -} - -function gone( - harness: Harness, - driverDeviceId: string, - reason?: "locked" | "untrusted" | "not-present", -): void { - harness.driver.setPhysicalPresence(driverDeviceId, false, reason === undefined ? {} : { reason }); -} - -function back(harness: Harness, driverDeviceId: string): void { - harness.driver.setPhysicalPresence(driverDeviceId, true); -} - -function stateOf(harness: Harness, device: PhysicalDevice): string | undefined { - return harness.registry.snapshot.devices.find((candidate) => candidate.id === device.id)?.state; -} - -function absentReasonOf(harness: Harness, device: PhysicalDevice): unknown { - const record = harness.registry.snapshot.devices.find((candidate) => candidate.id === device.id); - return (record as unknown as Record | undefined)?.absentReason; -} - -function eventsOf(harness: Harness, name: string) { - return harness.bus.replay().filter((event) => String(event.event) === name); -} - -function readsOf(driver: FakeDriver): unknown[] { - return driver.calls - .filter((call) => call.operation === "listPresentPhysical") - .map((call) => call.arguments[0]); -} - -const heldReclaims = new WeakMap(); - -function reclaimCalls(driver: FakeDriver) { - const held = Array.from({ length: heldReclaims.get(driver) ?? 0 }, () => ({ - arguments: [] as unknown[], - operation: "reclaim" as const, - })); - return [...driver.calls.filter((call) => call.operation === "reclaim"), ...held]; -} - -type PresenceAnswer = Awaited>; - -function answer( - entries: readonly { - readonly driverDeviceId: string; - readonly present: boolean; - readonly osVersion?: string; - readonly reason?: string; - }[], -): PresenceAnswer { - return entries as unknown as PresenceAnswer; -} - -interface RecordedWrite { - readonly claims: Record; - readonly states: Record; -} - -function recordStateWrites(harness: Harness): RecordedWrite[] { - const writes: RecordedWrite[] = []; - const original = harness.state.transaction.bind(harness.state); - vi.spyOn(harness.state, "transaction").mockImplementation((work) => - original(() => { - const result = work(); - const records = harness.state.database.prepare("SELECT id, state FROM devices").all() as { - id: string; - state: string; - }[]; - writes.push({ - claims: Object.fromEntries( - records.map((record) => [record.id, harness.engine.core.claims.claim(record.id)?.kind]), - ), - states: Object.fromEntries(records.map((record) => [record.id, record.state])), - }); - return result; - }), - ); - return writes; -} - -function holdNextReclaim(harness: Harness): { readonly resolve: () => void } { - const gate = deferred(); - const original = harness.driver.reclaim.bind(harness.driver); - vi.spyOn(harness.driver, "reclaim").mockImplementationOnce(async (device, options) => { - heldReclaims.set(harness.driver, (heldReclaims.get(harness.driver) ?? 0) + 1); - await gate.promise; - heldReclaims.set(harness.driver, (heldReclaims.get(harness.driver) ?? 1) - 1); - return original(device, options); - }); - return gate; -} - -function capturingLogger(): { readonly logger: Logger; readonly sink: MemoryLogSink } { - const sink = new MemoryLogSink(); - return { logger: new JsonLinesLogger({ clock: new FakeClock(1_000), sink }), sink }; -} + PROBE_MS, + absentReasonOf, + answer, + asking, + back, + createHarness, + enrollIphone, + flush, + gone, + harnesses, + holdNextReclaim, + physicalRequest, + readsOf, + recordStateWrites, + startWatcher, + stateOf, + tick, + watcherOf, +} from "./test-physical-device-watcher.js"; async function oneNotPresentReadThenReady( options: { readonly failFirstReclaim: boolean } = { failFirstReclaim: false }, @@ -374,6 +38,13 @@ async function oneNotPresentReadThenReady( return { device, harness }; } +afterEach(() => { + for (const harness of harnesses.splice(0)) { + watcherOf(harness)?.dispose(); + harness.engine.dispose(); + } +}); + describe("PhysicalDeviceWatcher counting", () => { it("a present read between not-present reads sets the count back to 0", async () => { const harness = await createHarness(); @@ -631,734 +302,3 @@ describe("PhysicalDeviceWatcher counting", () => { expect(stateOf(harness, device)).toBe("absent"); }); }); - -describe("PhysicalDeviceWatcher acting on a device", () => { - it("an absent device read not present with another reason gets its absentReason rewritten in one registry write, with no event; a read with the same reason writes nothing", async () => { - const { device, harness } = await absentHarness("locked"); - gone(harness, "UDID-A", "locked"); - const writes = recordStateWrites(harness); - startWatcher(harness); - - await tick(harness); - expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); - expect(writes).toHaveLength(0); - expect(absentReasonOf(harness, device)).toBe("locked"); - - gone(harness, "UDID-A", "not-present"); - const eventsBefore = harness.bus.replay().length; - await tick(harness); - expect(absentReasonOf(harness, device)).toBe("not-present"); - expect(writes).toHaveLength(1); - expect(harness.bus.replay()).toHaveLength(eventsBefore); - - await tick(harness); - expect(writes).toHaveLength(1); - expect(absentReasonOf(harness, device)).toBe("not-present"); - }); - - it("a device under an operation claim read not present past the grace period keeps its state; the first tick after the claim is released moves it", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - gone(harness, "UDID-A"); - const claim = harness.engine.core.claims.tryClaim(device.id, "enrollment"); - expect(claim).toBeDefined(); - startWatcher(harness); - - await tick(harness, 3); - expect(readsOf(harness.driver)).toHaveLength(3); - expect(stateOf(harness, device)).toBe("ready"); - - claim?.release(); - await tick(harness); - expect(stateOf(harness, device)).toBe("absent"); - }); - - it("a grant that commits between the watcher's read and its write leaves the new lease in place", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - startWatcher(harness); - gone(harness, "UDID-A"); - await tick(harness); - expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); - - back(harness, "UDID-A"); - const gate = deferred(); - vi.spyOn(harness.driver, "listPresentPhysical").mockImplementationOnce(async (ids) => { - await gate.promise; - return answer( - ids.map((driverDeviceId) => ({ driverDeviceId, present: false, reason: "not-present" })), - ); - }); - harness.clock.advance(PROBE_MS); - await flush(); - const granted = await outcome(harness.engine.request(physicalRequest, asking("agent"))); - expect(granted).toMatchObject({ device: { id: device.id } }); - gate.resolve(); - await flush(); - - expect(stateOf(harness, device)).toBe("leased"); - expect(harness.registry.snapshot.leases).toMatchObject([{ deviceId: device.id }]); - expect(eventsOf(harness, "lease.released")).toEqual([]); - }); - - it("the return reclaim holds a reclaim claim from before the absent → reclaiming write until it settles, and releases it when the reclaim throws", async () => { - const held = await absentHarness("not-present"); - back(held.harness, "UDID-A"); - const writes = recordStateWrites(held.harness); - const gate = holdNextReclaim(held.harness); - startWatcher(held.harness); - - await tick(held.harness); - expect(reclaimCalls(held.harness.driver)).toHaveLength(1); - expect(held.harness.engine.core.claims.claim(held.device.id)?.kind).toBe("reclaim"); - expect(stateOf(held.harness, held.device)).toBe("reclaiming"); - expect(writes.find((write) => write.states[held.device.id] === "reclaiming")).toMatchObject({ - claims: { [held.device.id]: "reclaim" }, - }); - gate.resolve(); - await flush(); - expect(held.harness.engine.core.claims.isClaimed(held.device.id)).toBe(false); - expect(stateOf(held.harness, held.device)).toBe("ready"); - - const thrown = await absentHarness("not-present"); - back(thrown.harness, "UDID-A"); - thrown.harness.driver.failOn("reclaim", 1, new Error("usb dropped")); - startWatcher(thrown.harness); - - await tick(thrown.harness); - expect(reclaimCalls(thrown.harness.driver)).toHaveLength(1); - expect(thrown.harness.engine.core.claims.isClaimed(thrown.device.id)).toBe(false); - }); - - it("an absent device read present while device add holds its claim gets no write and no reclaim that tick", async () => { - const { device, harness } = await absentHarness("not-present"); - back(harness, "UDID-A"); - const claim = harness.engine.core.claims.tryClaim(device.id, "enrollment"); - expect(claim).toBeDefined(); - const writes = recordStateWrites(harness); - startWatcher(harness); - - await tick(harness); - expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); - expect(writes).toHaveLength(0); - expect(reclaimCalls(harness.driver)).toHaveLength(0); - expect(stateOf(harness, device)).toBe("absent"); - - claim?.release(); - await tick(harness); - expect(reclaimCalls(harness.driver)).toHaveLength(1); - }); - - it("an absent device read present is returned and reclaimed on that tick, at most one health.probeIntervalMs after it was scripted present", async () => { - const { device, harness } = await absentHarness("not-present"); - startWatcher(harness); - harness.clock.advance(10_000); - await flush(); - - back(harness, "UDID-A"); - harness.clock.advance(PROBE_MS); - await flush(); - - expect(eventsOf(harness, "device.returned")).toHaveLength(1); - expect(reclaimCalls(harness.driver)).toHaveLength(1); - expect(stateOf(harness, device)).toBe("ready"); - }); - - it("a device the fake driver reports present again has a latest read of present within one health.probeIntervalMs", async () => { - const harness = await createHarness(); - await enrollIphone(harness, "UDID-LATEST-WATCH"); - startWatcher(harness); - gone(harness, "UDID-LATEST-WATCH", "locked"); - await tick(harness); - expect(latestPresence("UDID-LATEST-WATCH")).toStrictEqual({ present: false, reason: "locked" }); - - back(harness, "UDID-LATEST-WATCH"); - harness.clock.advance(PROBE_MS); - await flush(); - - expect(latestPresence("UDID-LATEST-WATCH")).toStrictEqual({ present: true }); - }); - - it("two ready devices read not present for the grace period both become absent, each judged on its own state", async () => { - const harness = await createHarness(); - const first = await enrollIphone(harness, "UDID-A"); - const second = await enrollIphone(harness, "UDID-B"); - gone(harness, "UDID-A", "locked"); - gone(harness, "UDID-B", "untrusted"); - startWatcher(harness); - - await tick(harness, 2); - - expect([stateOf(harness, first), stateOf(harness, second)]).toEqual(["absent", "absent"]); - expect([absentReasonOf(harness, first), absentReasonOf(harness, second)]).toEqual([ - "locked", - "untrusted", - ]); - }); - - it("a tick where one device's action throws (its registry write fails) logs it, still acts on the other devices, and arms the next tick; that device's count is not reset, so the next not-present read acts on it again", async () => { - const { logger, sink } = capturingLogger(); - const harness = await createHarness({ logger }); - const first = await enrollIphone(harness, "UDID-A"); - const second = await enrollIphone(harness, "UDID-B"); - gone(harness, "UDID-A"); - gone(harness, "UDID-B"); - startWatcher(harness); - await tick(harness); - vi.spyOn(harness.state, "transaction").mockImplementationOnce(() => { - throw new Error("disk full"); - }); - const problems = () => - sink.records.filter((record) => record.level === "warn" || record.level === "error"); - const problemsBefore = problems().length; - - await tick(harness); - const states = [stateOf(harness, first), stateOf(harness, second)]; - expect(states.filter((state) => state === "absent")).toHaveLength(1); - expect(states.filter((state) => state === "ready")).toHaveLength(1); - expect(problems().length).toBeGreaterThan(problemsBefore); - const failed = [first, second].find((device) => stateOf(harness, device) === "ready"); - expect( - problems().filter((record) => record.message === "physical device action failed"), - ).toMatchObject([ - { fields: { deviceId: failed?.id, error: expect.anything() }, level: "warn" }, - ]); - - await tick(harness); - expect(stateOf(harness, first)).toBe("absent"); - expect(stateOf(harness, second)).toBe("absent"); - }); - - it("a ready device read not present once and then present wakes the queue: a physical request waiting for it is granted on that tick", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - gone(harness, "UDID-A"); - const waiting = harness.engine.request(physicalRequest, asking("agent")); - expect(await outcome(waiting)).toBe("pending"); - startWatcher(harness); - - await tick(harness); - expect(readsOf(harness.driver)).toContainEqual(["UDID-A"]); - back(harness, "UDID-A"); - expect(await outcome(waiting)).toBe("pending"); - - await tick(harness); - expect(await outcome(waiting)).toMatchObject({ device: { id: device.id } }); - }); - - it("a tick whose read resolves present after dispose does not wake the queue: a physical request waiting for the device stays pending", async () => { - const harness = await createHarness(); - await enrollIphone(harness, "UDID-A"); - gone(harness, "UDID-A"); - const waiting = harness.engine.request(physicalRequest, asking("agent")); - expect(await outcome(waiting)).toBe("pending"); - startWatcher(harness); - await tick(harness); - back(harness, "UDID-A"); - harness.driver.holdPresenceReads("UDID-A"); - harness.clock.advance(PROBE_MS); - await flush(); - expect(readsOf(harness.driver)).toHaveLength(3); - - watcherOf(harness)?.dispose(); - harness.driver.releasePresenceReads(); - - expect(await outcome(waiting)).toBe("pending"); - }); - - it("a tick whose read resolves after dispose writes nothing and starts no reclaim, and leasing.settle() resolves only after that tick", async () => { - const ready = await createHarness(); - const readyDevice = await enrollIphone(ready, "UDID-A"); - gone(ready, "UDID-A"); - startWatcher(ready); - await tick(ready); - const readyWrites = recordStateWrites(ready); - ready.driver.holdPresenceReads("UDID-A"); - ready.clock.advance(PROBE_MS); - await flush(); - expect(readsOf(ready.driver)).toHaveLength(2); - - watcherOf(ready)?.dispose(); - const settling = ready.engine.leasing.settle(); - expect(await isSettled(settling)).toBe(false); - ready.driver.releasePresenceReads(); - expect(await isSettled(settling)).toBe(true); - expect(stateOf(ready, readyDevice)).toBe("ready"); - expect(readyWrites).toHaveLength(0); - - const { device, harness } = await absentHarness("not-present"); - back(harness, "UDID-A"); - const writes = recordStateWrites(harness); - harness.driver.holdPresenceReads("UDID-A"); - startWatcher(harness); - harness.clock.advance(PROBE_MS); - await flush(); - expect(readsOf(harness.driver)).toHaveLength(1); - - watcherOf(harness)?.dispose(); - harness.driver.releasePresenceReads(); - await flush(); - expect(stateOf(harness, device)).toBe("absent"); - expect(writes).toHaveLength(0); - expect(reclaimCalls(harness.driver)).toHaveLength(0); - }); - - it("a platform with no driver changes no count and no state, and the other platform is still read", async () => { - const harness = await createHarness(); - const iphone = await enrollIphone(harness, "UDID-A"); - const pixel = await harness.registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: "EMU-1", - model: "Pixel 8", - osVersion: "14", - platform: "android", - }); - gone(harness, "UDID-A"); - startWatcher(harness); - - await tick(harness, 10); - - expect(stateOf(harness, iphone)).toBe("absent"); - expect(stateOf(harness, pixel)).toBe("ready"); - expect(readsOf(harness.driver)[0]).toEqual(["UDID-A"]); - }); - - it("a listing entry whose ID has no record causes no write and no driver call but the listing", async () => { - const harness = await createHarness(); - await enrollIphone(harness, "UDID-A"); - const listing = vi.spyOn(harness.driver, "listPresentPhysical").mockResolvedValue( - answer([ - { driverDeviceId: "UDID-A", osVersion: "18.4", present: true }, - { driverDeviceId: "UDID-GHOST-1", osVersion: "18.4", present: true }, - { driverDeviceId: "UDID-GHOST-2", present: false, reason: "locked" }, - ]), - ); - const writes = recordStateWrites(harness); - startWatcher(harness); - - await tick(harness, 3); - - expect(listing.mock.calls.length).toBeGreaterThan(0); - expect(harness.driver.calls.map((call) => call.operation)).toEqual([]); - expect(writes).toHaveLength(0); - }); - - it("a ready device that stays not present for health.stableObservations reads emits one device.absence-detected with previousState ready and the last read's reason", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - startWatcher(harness); - gone(harness, "UDID-A", "locked"); - - await tick(harness, 2); - - expect(eventsOf(harness, "device.absence-detected")).toMatchObject([ - { payload: { absentReason: "locked", deviceId: device.id, previousState: "ready" } }, - ]); - }); - - it("a ready device read present without a miss does not wake the queue", async () => { - const harness = await createHarness(); - await enrollIphone(harness, "UDID-A"); - const kick = vi.spyOn( - harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, - "kick", - ); - startWatcher(harness); - - await tick(harness, 2); - - expect(readsOf(harness.driver)).toHaveLength(2); - expect(kick).not.toHaveBeenCalled(); - }); - - it("a ready device read not present once and then present wakes the queue once, not again on the next present read", async () => { - const harness = await createHarness(); - await enrollIphone(harness, "UDID-A"); - const kick = vi.spyOn( - harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, - "kick", - ); - startWatcher(harness); - gone(harness, "UDID-A"); - await tick(harness); - expect(kick).not.toHaveBeenCalled(); - - back(harness, "UDID-A"); - await tick(harness); - expect(kick).toHaveBeenCalledTimes(1); - - await tick(harness); - expect(kick).toHaveBeenCalledTimes(1); - }); - - it("a leased device read not present once and then present does not wake the queue", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - startWatcher(harness); - await harness.engine.request(physicalRequest, asking("agent")); - expect(stateOf(harness, device)).toBe("leased"); - const kick = vi.spyOn( - harness.engine.leasing.corePorts.leaseMaintenance.acquisition as unknown as { kick(): void }, - "kick", - ); - gone(harness, "UDID-A"); - await tick(harness); - back(harness, "UDID-A"); - - await tick(harness); - - expect(stateOf(harness, device)).toBe("leased"); - expect(kick).not.toHaveBeenCalled(); - }); - - it("an absent device read present returns with the osVersion of the read, or with the one on its record when the read has none", async () => { - const withVersion = await absentHarness("not-present"); - vi.spyOn(withVersion.harness.driver, "listPresentPhysical").mockResolvedValue( - answer([{ driverDeviceId: "UDID-A", osVersion: "18.5", present: true }]), - ); - startWatcher(withVersion.harness); - await tick(withVersion.harness); - expect(eventsOf(withVersion.harness, "device.returned")).toMatchObject([ - { payload: { deviceId: withVersion.device.id, osVersion: "18.5" } }, - ]); - - const withoutVersion = await absentHarness("not-present"); - vi.spyOn(withoutVersion.harness.driver, "listPresentPhysical").mockResolvedValue( - answer([{ driverDeviceId: "UDID-A", present: true }]), - ); - startWatcher(withoutVersion.harness); - await tick(withoutVersion.harness); - expect(eventsOf(withoutVersion.harness, "device.returned")).toMatchObject([ - { payload: { deviceId: withoutVersion.device.id, osVersion: "18.4" } }, - ]); - }); - - it("a tick whose watcher is disposed while it acts on one device acts on no later device of that tick", async () => { - const options = { health: { stableObservations: 1 } }; - const first = await createHarness(options); - const missing = await enrollIphone(first, "UDID-A"); - const returned = await enrollIphone(first, "UDID-B"); - const harness = await reopen( - first, - (database) => markAbsent(database, "UDID-B", "not-present"), - options, - ); - gone(harness, "UDID-A"); - back(harness, "UDID-B"); - const original = harness.state.transaction.bind(harness.state); - vi.spyOn(harness.state, "transaction").mockImplementationOnce((work) => { - watcherOf(harness)?.dispose(); - return original(work); - }); - startWatcher(harness); - - await tick(harness); - - expect(stateOf(harness, missing)).toBe("absent"); - expect(stateOf(harness, returned)).toBe("absent"); - expect(reclaimCalls(harness.driver)).toEqual([]); - expect(eventsOf(harness, "device.returned")).toEqual([]); - }); - - it("a tick whose watcher is disposed while it acts on one device does not wake the queue for a later device of that tick", async () => { - const first = await createHarness(); - const missing = await enrollIphone(first, "UDID-A"); - const flickering = await enrollIphone(first, "UDID-B"); - gone(first, "UDID-A"); - gone(first, "UDID-B"); - const waiting = first.engine.request(physicalRequest, asking("agent")); - expect(await outcome(waiting)).toBe("pending"); - startWatcher(first); - await tick(first); - back(first, "UDID-B"); - const original = first.state.transaction.bind(first.state); - vi.spyOn(first.state, "transaction").mockImplementationOnce((work) => { - watcherOf(first)?.dispose(); - return original(work); - }); - - await tick(first); - - expect(stateOf(first, missing)).toBe("absent"); - expect(stateOf(first, flickering)).toBe("ready"); - expect(await outcome(waiting)).toBe("pending"); - }); -}); - -describe("PhysicalDeviceWatcher around restarts and doctor", () => { - it("a device left reclaiming by a killed daemon, read present, whose re-run reclaim fails: still present, it is quarantined with device.quarantined only; read not present after the failure, it is absent with device.absence-detected previousState reclaiming", async () => { - const restarted = async () => { - const first = await createHarness(); - const device = await enrollIphone(first, "UDID-A"); - await first.registry.transitionDevice(device.id, "leased"); - await first.registry.transitionDevice(device.id, "reclaiming"); - const harness = await reopen(first, () => undefined); - back(harness, "UDID-A"); - await harness.engine.convergeRunningCapacity(); - return { device, harness }; - }; - - const present = await restarted(); - vi.spyOn(present.harness.driver, "reclaim").mockRejectedValueOnce( - new Error("uninstall failed"), - ); - startWatcher(present.harness); - await tick(present.harness); - expect(stateOf(present.harness, present.device)).toBe("quarantined"); - expect(eventsOf(present.harness, "device.quarantined")).toHaveLength(1); - expect(eventsOf(present.harness, "device.purge-failed")).toEqual([]); - expect(eventsOf(present.harness, "device.absence-detected")).toEqual([]); - - const unplugged = await restarted(); - vi.spyOn(unplugged.harness.driver, "reclaim").mockImplementationOnce(async () => { - gone(unplugged.harness, "UDID-A"); - throw new Error("usb lost"); - }); - startWatcher(unplugged.harness); - await tick(unplugged.harness); - expect(stateOf(unplugged.harness, unplugged.device)).toBe("absent"); - expect(eventsOf(unplugged.harness, "device.absence-detected")).toMatchObject([ - { - payload: { - absentReason: "not-present", - deviceId: unplugged.device.id, - previousState: "reclaiming", - }, - }, - ]); - expect(eventsOf(unplugged.harness, "device.quarantined")).toEqual([]); - expect(eventsOf(unplugged.harness, "device.purge-failed")).toEqual([]); - }); - - it("a physical device that entered reclaiming in this process and holds no claim is never reclaimed by the watcher", async () => { - const first = await createHarness(); - const restartedDevice = await enrollIphone(first, "UDID-A"); - const processDevice = await enrollIphone(first, "UDID-B"); - await first.registry.transitionDevice(restartedDevice.id, "leased"); - await first.registry.transitionDevice(restartedDevice.id, "reclaiming"); - const harness = await reopen(first, () => undefined); - back(harness, "UDID-A"); - back(harness, "UDID-B"); - await harness.engine.convergeRunningCapacity(); - await harness.registry.transitionDevice(processDevice.id, "leased"); - await harness.registry.transitionDevice(processDevice.id, "reclaiming"); - startWatcher(harness); - - await tick(harness, 3); - - expect(stateOf(harness, restartedDevice)).toBe("ready"); - expect(stateOf(harness, processDevice)).toBe("reclaiming"); - expect( - reclaimCalls(harness.driver).filter((call) => - JSON.stringify(call.arguments).includes("UDID-B"), - ), - ).toEqual([]); - }); - - it("a device left reclaiming by a killed daemon that left reclaiming before the first tick, and entered it again in this process, is never reclaimed by the watcher", async () => { - const first = await createHarness(); - const device = await enrollIphone(first, "UDID-A"); - await first.registry.transitionDevice(device.id, "leased"); - await first.registry.transitionDevice(device.id, "reclaiming"); - const harness = await reopen(first, () => undefined); - back(harness, "UDID-A"); - await harness.engine.convergeRunningCapacity(); - await harness.registry.transitionDevice(device.id, "ready", { - event: "device.reclaimed", - payload: { deviceId: device.id, duration: 0, strategy: "uninstall" }, - }); - startWatcher(harness); - await tick(harness); - expect(readsOf(harness.driver)).toEqual([["UDID-A"]]); - expect(reclaimCalls(harness.driver)).toEqual([]); - - await harness.registry.transitionDevice(device.id, "leased"); - await harness.registry.transitionDevice(device.id, "reclaiming"); - await tick(harness, 2); - - expect(readsOf(harness.driver)).toHaveLength(3); - expect(stateOf(harness, device)).toBe("reclaiming"); - expect(reclaimCalls(harness.driver)).toEqual([]); - }); - - it("a device left reclaiming by a killed daemon that doctor --fix quarantines before the watcher's tick stays quarantined, and the watcher runs no reclaim on it", async () => { - const options = { quarantineBackoffMs: 10 * DAY_MS }; - const first = await createHarness(options); - const device = await enrollIphone(first, "UDID-A"); - await first.registry.transitionDevice(device.id, "leased"); - await first.registry.transitionDevice(device.id, "reclaiming"); - const harness = await reopen( - first, - (database) => { - database.exec("UPDATE devices SET last_lease_ended_at = 1"); - }, - options, - ); - await harness.engine.convergeRunningCapacity(); - harness.clock.advance(DAY_MS); - await harness.engine.core.doctor.reconcile({ fix: true }); - expect(stateOf(harness, device)).toBe("quarantined"); - back(harness, "UDID-A"); - startWatcher(harness); - - await tick(harness, 3); - - expect(readsOf(harness.driver)).toContainEqual(["UDID-A"]); - expect(stateOf(harness, device)).toBe("quarantined"); - expect(reclaimCalls(harness.driver)).toEqual([]); - }); -}); - -describe("PhysicalDeviceWatcher and nuke maintenance", () => { - it("the device-lost ending waits while nuke maintenance is active, and runs when it ends", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - startWatcher(harness); - await harness.engine.request(physicalRequest, asking("agent")); - gone(harness, "UDID-A"); - const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; - await maintenance.beginMaintenance(); - - await tick(harness, 3); - expect(readsOf(harness.driver).length).toBeGreaterThan(0); - expect(stateOf(harness, device)).toBe("leased"); - expect(harness.registry.snapshot.leases).toHaveLength(1); - - await maintenance.endMaintenance(); - await flush(); - await tick(harness); - expect(stateOf(harness, device)).toBe("absent"); - expect(harness.registry.snapshot.leases).toEqual([]); - expect(eventsOf(harness, "lease.released")).toMatchObject([ - { payload: { reason: "device-lost" } }, - ]); - }); - - it("while nuke maintenance is active, a ready device past the grace period stays ready, an absent device read with another reason keeps its old one, and an absent device read present gets no write and no reclaim; all three are acted on once maintenance ends", async () => { - const options = { health: { stableObservations: 3 } }; - const first = await createHarness(options); - const ready = await enrollIphone(first, "UDID-R"); - const reasoned = await enrollIphone(first, "UDID-X"); - const returned = await enrollIphone(first, "UDID-Y"); - const harness = await reopen( - first, - (database) => { - markAbsent(database, "UDID-X", "locked"); - markAbsent(database, "UDID-Y", "locked"); - }, - options, - ); - gone(harness, "UDID-R"); - gone(harness, "UDID-X", "locked"); - gone(harness, "UDID-Y", "locked"); - startWatcher(harness); - await tick(harness, 2); - const writes = recordStateWrites(harness); - gone(harness, "UDID-X", "not-present"); - back(harness, "UDID-Y"); - const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; - await maintenance.beginMaintenance(); - - await tick(harness); - expect(readsOf(harness.driver)).toHaveLength(3); - expect(stateOf(harness, ready)).toBe("ready"); - expect(absentReasonOf(harness, reasoned)).toBe("locked"); - expect(stateOf(harness, returned)).toBe("absent"); - expect(writes).toHaveLength(0); - expect(reclaimCalls(harness.driver)).toHaveLength(0); - - await maintenance.endMaintenance(); - await flush(); - await tick(harness); - expect(stateOf(harness, ready)).toBe("absent"); - expect(absentReasonOf(harness, reasoned)).toBe("not-present"); - expect(reclaimCalls(harness.driver)).toHaveLength(1); - expect(["reclaiming", "ready"]).toContain(stateOf(harness, returned)); - }); - - it.each([ - { - name: "nuke's lease maintenance, begun while it runs,", - start: (harness: Harness) => - harness.engine.leasing.corePorts.leaseMaintenance.leases.beginMaintenance(), - finish: async (harness: Harness) => { - await harness.engine.leasing.corePorts.leaseMaintenance.leases.endMaintenance(); - }, - }, - { - name: "leasing.settle()", - start: (harness: Harness) => harness.engine.leasing.settle(), - finish: async (harness: Harness, device: PhysicalDevice) => { - expect(stateOf(harness, device)).toBe("ready"); - }, - }, - ])("$name resolves only after a return reclaim the watcher started has settled", async (row) => { - const { device, harness } = await absentHarness("not-present"); - back(harness, "UDID-A"); - const gate = holdNextReclaim(harness); - startWatcher(harness); - await tick(harness); - expect(reclaimCalls(harness.driver)).toHaveLength(1); - - const waiting = row.start(harness); - expect(await isSettled(waiting)).toBe(false); - - gate.resolve(); - expect(await isSettled(waiting)).toBe(true); - await row.finish(harness, device); - }); - - it("an absentReason rewrite held back by nuke maintenance is dropped when the watcher is disposed before maintenance ends", async () => { - const { device, harness } = await absentHarness("locked"); - gone(harness, "UDID-A", "not-present"); - startWatcher(harness); - const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; - await maintenance.beginMaintenance(); - await tick(harness); - expect(readsOf(harness.driver)).toHaveLength(1); - - watcherOf(harness)?.dispose(); - await maintenance.endMaintenance(); - await flush(); - - expect(absentReasonOf(harness, device)).toBe("locked"); - }); - - it("a return reclaim waiting on nuke maintenance when the watcher is disposed never starts: the device stays absent", async () => { - const { device, harness } = await absentHarness("not-present"); - back(harness, "UDID-A"); - startWatcher(harness); - const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; - await maintenance.beginMaintenance(); - await tick(harness); - expect(readsOf(harness.driver)).toHaveLength(1); - - watcherOf(harness)?.dispose(); - await maintenance.endMaintenance(); - await flush(); - - expect(stateOf(harness, device)).toBe("absent"); - expect(reclaimCalls(harness.driver)).toHaveLength(0); - }); - - it("a device-lost ending waiting on nuke maintenance when the watcher is disposed leaves the lease in place", async () => { - const harness = await createHarness(); - const device = await enrollIphone(harness, "UDID-A"); - startWatcher(harness); - await harness.engine.request(physicalRequest, asking("agent")); - gone(harness, "UDID-A"); - await tick(harness); - const maintenance = harness.engine.leasing.corePorts.leaseMaintenance.leases; - await maintenance.beginMaintenance(); - await tick(harness); - expect(readsOf(harness.driver).length).toBeGreaterThan(1); - - watcherOf(harness)?.dispose(); - await maintenance.endMaintenance(); - await flush(); - - expect(stateOf(harness, device)).toBe("leased"); - expect(harness.registry.snapshot.leases).toHaveLength(1); - }); -}); diff --git a/src/leasing/test-physical-device-watcher.ts b/src/leasing/test-physical-device-watcher.ts new file mode 100644 index 000000000..475bae823 --- /dev/null +++ b/src/leasing/test-physical-device-watcher.ts @@ -0,0 +1,327 @@ +import { vi } from "vitest"; +import { type Config, type DeviceRequest, type PhysicalDevice, Registry } from "../core/index.js"; +import { + FakeDriver, + loadTestRegistry, + openTestState, + testComponentWiring, + testStateOf, +} from "../core/testing.js"; +import { type StateDatabase } from "../state/index.js"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, FakeSystemStats, type Logger, type SqliteDatabase } from "../ports/index.js"; +import { createTestEngine, type TestEngine, type TestEngineConfig } from "./testing.js"; + +const gibibyte = 1024 ** 3; + +export const PROBE_MS = 30_000; + +export const physicalRequest = { + model: "iPhone 15 Pro", + physical: true, + platform: "ios", +} as const satisfies DeviceRequest; + +interface HarnessOptions { + readonly android?: boolean; + readonly health?: Partial; + readonly logger?: Logger; + readonly quarantineBackoffMs?: number; +} + +function config(options: HarnessOptions): TestEngineConfig { + const quarantineBackoffMs = options.quarantineBackoffMs ?? 30_000; + return { + devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, + health: { + enabled: true, + maxConcurrentRecoveries: 1, + maxRecoveryAttempts: 3, + probeIntervalMs: PROBE_MS, + recoveryBackoffMs: 5_000, + stableObservations: 2, + ...options.health, + }, + stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, + lease: { + defaultTtlMs: 3_600_000, + maxTtlMs: 14_400_000, + identity: { ios: "reusable", android: "reusable" }, + requestRetentionMs: 600_000, + endedIdRetentionMs: 600_000, + maxRequestRecords: 10_000, + maxPerRequester: 1, + endWaitMs: 600_000, + }, + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + warmPool: { + enabled: false, + maxConcurrentBoots: 1, + reserveRunning: { android: 0, ios: 0 }, + targets: [], + quarantine: { + maxRetries: 3, + maxRetryBackoffMs: Math.max(300_000, quarantineBackoffMs), + retryBackoffMs: quarantineBackoffMs, + retryBackoffMultiplier: 2, + }, + }, + }; +} + +export interface Harness { + readonly androidDriver: FakeDriver; + readonly bus: EventBus; + readonly clock: FakeClock; + readonly driver: FakeDriver; + readonly engine: TestEngine; + readonly registry: Registry; + readonly state: StateDatabase; +} + +export const harnesses: Harness[] = []; + +export async function createHarness( + options: HarnessOptions = {}, + state?: StateDatabase, +): Promise { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const androidDriver = new FakeDriver({ + availableOsVersions: ["14"], + clock, + platform: "android", + }); + const drivers = options.android === true ? [driver, androidDriver] : [driver]; + let nextId = 1; + const stateDatabase = state ?? (await openTestState()); + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => `${nextId++}` }, + state: stateDatabase, + }); + const wiring = testComponentWiring({ clock, drivers, eventBus: bus, registry }); + const engine = createTestEngine({ + ...wiring, + clock, + config: config(options), + drivers, + eventBus: bus, + idGenerator: { generate: () => `request-${nextId++}` }, + ...(options.logger === undefined ? {} : { logger: options.logger }), + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + }); + const harness = { + androidDriver, + bus, + clock, + driver, + engine, + registry, + state: stateDatabase, + }; + harnesses.push(harness); + return harness; +} + +interface WatcherHandle { + start(): void; + dispose(): void; +} + +export function watcherOf(harness: Harness): WatcherHandle | undefined { + return (harness.engine.leasing as unknown as { physicalDeviceWatcher?: WatcherHandle }) + .physicalDeviceWatcher; +} + +export function startWatcher(harness: Harness): void { + watcherOf(harness)?.start(); +} + +export async function reopen( + first: Harness, + patch: (database: SqliteDatabase) => void, + options: HarnessOptions = {}, +): Promise { + patch(first.state.database); + first.engine.dispose(); + return createHarness(options, first.state); +} + +export function markAbsent( + database: SqliteDatabase, + driverDeviceId: string, + absentReason: string, +): void { + const { changes } = database + .prepare("UPDATE devices SET state = 'absent', absent_reason = ? WHERE driver_device_id = ?") + .run(absentReason, driverDeviceId); + if (changes === 0) throw new Error(`no record for ${driverDeviceId}`); +} + +export async function enrollIphone( + harness: Harness, + driverDeviceId: string, +): Promise { + const device = await harness.registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId, + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + harness.driver.setPhysicalPresence(driverDeviceId, true); + return device; +} + +export async function absentHarness( + reason: string, + options: HarnessOptions = {}, +): Promise<{ readonly device: PhysicalDevice; readonly harness: Harness }> { + const first = await createHarness(options); + const device = await enrollIphone(first, "UDID-A"); + const harness = await reopen( + first, + (database) => markAbsent(database, "UDID-A", reason), + options, + ); + return { device, harness }; +} + +export function asking(requesterId: string) { + return { ownerId: requesterId, requesterId }; +} + +export async function flush(): Promise { + for (let count = 0; count < 500; count += 1) { + await Promise.resolve(); + } +} + +export async function tick(harness: Harness, times = 1): Promise { + for (let count = 0; count < times; count += 1) { + harness.clock.advance(PROBE_MS); + await flush(); + } +} + +export async function isSettled(promise: Promise): Promise { + return Promise.race([promise.then(() => true), flush().then(() => false)]); +} + +export function deferred(): { readonly promise: Promise; readonly resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +export function gone( + harness: Harness, + driverDeviceId: string, + reason?: "locked" | "untrusted" | "not-present", +): void { + harness.driver.setPhysicalPresence(driverDeviceId, false, reason === undefined ? {} : { reason }); +} + +export function back(harness: Harness, driverDeviceId: string): void { + harness.driver.setPhysicalPresence(driverDeviceId, true); +} + +export function stateOf(harness: Harness, device: PhysicalDevice): string | undefined { + return harness.registry.snapshot.devices.find((candidate) => candidate.id === device.id)?.state; +} + +export function absentReasonOf(harness: Harness, device: PhysicalDevice): unknown { + const record = harness.registry.snapshot.devices.find((candidate) => candidate.id === device.id); + return (record as unknown as Record | undefined)?.absentReason; +} + +export function eventsOf(harness: Harness, name: string) { + return harness.bus.replay().filter((event) => String(event.event) === name); +} + +export function readsOf(driver: FakeDriver): unknown[] { + return driver.calls + .filter((call) => call.operation === "listPresentPhysical") + .map((call) => call.arguments[0]); +} + +const heldReclaims = new WeakMap(); + +export function reclaimCalls(driver: FakeDriver) { + const held = Array.from({ length: heldReclaims.get(driver) ?? 0 }, () => ({ + arguments: [] as unknown[], + operation: "reclaim" as const, + })); + return [...driver.calls.filter((call) => call.operation === "reclaim"), ...held]; +} + +type PresenceAnswer = Awaited>; + +export function answer( + entries: readonly { + readonly driverDeviceId: string; + readonly present: boolean; + readonly osVersion?: string; + readonly reason?: string; + }[], +): PresenceAnswer { + return entries as unknown as PresenceAnswer; +} + +interface RecordedWrite { + readonly claims: Record; + readonly states: Record; +} + +export function recordStateWrites(harness: Harness): RecordedWrite[] { + const writes: RecordedWrite[] = []; + const original = harness.state.transaction.bind(harness.state); + vi.spyOn(harness.state, "transaction").mockImplementation((work) => + original(() => { + const result = work(); + const records = harness.state.database.prepare("SELECT id, state FROM devices").all() as { + id: string; + state: string; + }[]; + writes.push({ + claims: Object.fromEntries( + records.map((record) => [record.id, harness.engine.core.claims.claim(record.id)?.kind]), + ), + states: Object.fromEntries(records.map((record) => [record.id, record.state])), + }); + return result; + }), + ); + return writes; +} + +export function holdNextReclaim(harness: Harness): { readonly resolve: () => void } { + const gate = deferred(); + const original = harness.driver.reclaim.bind(harness.driver); + vi.spyOn(harness.driver, "reclaim").mockImplementationOnce(async (device, options) => { + heldReclaims.set(harness.driver, (heldReclaims.get(harness.driver) ?? 0) + 1); + await gate.promise; + heldReclaims.set(harness.driver, (heldReclaims.get(harness.driver) ?? 1) - 1); + return original(device, options); + }); + return gate; +} From 259d164426e37afe225524965979332d4ff07c3c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:29 +0100 Subject: [PATCH 40/83] test(leasing): split lease-end-participants.test.ts by topic --- ...ticipants-nuke-settle-and-restarts.test.ts | 239 ++++++ ...-participants-startup-and-ceilings.test.ts | 246 ++++++ src/leasing/lease-end-participants.test.ts | 738 +----------------- src/leasing/test-lease-end-participants.ts | 284 +++++++ 4 files changed, 789 insertions(+), 718 deletions(-) create mode 100644 src/leasing/lease-end-participants-nuke-settle-and-restarts.test.ts create mode 100644 src/leasing/lease-end-participants-startup-and-ceilings.test.ts create mode 100644 src/leasing/test-lease-end-participants.ts diff --git a/src/leasing/lease-end-participants-nuke-settle-and-restarts.test.ts b/src/leasing/lease-end-participants-nuke-settle-and-restarts.test.ts new file mode 100644 index 000000000..7bb4e2106 --- /dev/null +++ b/src/leasing/lease-end-participants-nuke-settle-and-restarts.test.ts @@ -0,0 +1,239 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { NoCapacityError } from "./index.js"; +import { + END_WAIT_MS, + TestParticipant, + type World, + asking, + createWorld, + destroys, + flush, + leaseAndRelease, + markerOf, + request, + settledOrPending, + startEngine, + started, + stateOf, + warningsFor, +} from "./test-lease-end-participants.js"; + +const otherRequest = { model: "iPhone 15", osVersion: "26.5", platform: "ios" } as const; + +function eventNames(world: World): string[] { + return world.bus.replay().map((event) => String(event.event)); +} + +afterEach(() => { + for (const engine of started.splice(0)) engine.dispose(); +}); + +describe("lease-end participants (#469)", () => { + it("advancing the clock past lease.endWaitMs after a nuke cut a wait short, and the participant then resolving, logs no ceiling warning and starts no second destroy", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + await harness.engine.nuke(false); + expect(destroys(harness.driver)).toHaveLength(1); + + harness.clock.advance(END_WAIT_MS + 1); + participant.calls[0]?.resolve(); + await flush(); + + expect(warningsFor(harness, grant.lease.id)).toEqual([]); + expect(destroys(harness.driver)).toHaveLength(1); + }); + + it("nuke with leased devices calls each participant with an already aborted signal, and completes without the clock advancing even when the participant never returns", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ + ios: { maxDevices: 2, maxRunning: 2 }, + participants: [participant], + }); + const one = await harness.engine.request(request, asking("one")); + const two = await harness.engine.request(request, asking("two")); + + const nuking = harness.engine.nuke(false); + + await expect(settledOrPending(nuking)).resolves.toMatchObject({ + releasedLeaseIds: expect.arrayContaining([one.lease.id, two.lease.id]), + }); + expect(participant.calls.map((call) => call.abortedAtCall)).toEqual([true, true]); + }); + + it("after nuke ends, a release waits for its participant again", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant] }); + await harness.engine.nuke(false); + const grant = await harness.engine.request(request, asking("agent")); + + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + + expect(participant.calls).toHaveLength(1); + expect(participant.calls[0]?.abortedAtCall).toBe(false); + expect(destroys(harness.driver)).toHaveLength(0); + }); + + it("when the participant resolves after settle(), the driver's destroy has run once, and advancing past lease.endWaitMs logs no ceiling warning", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + await harness.engine.settle(); + + participant.calls[0]?.resolve(); + harness.clock.advance(END_WAIT_MS + 1); + await flush(); + + expect(destroys(harness.driver)).toHaveLength(1); + expect(warningsFor(harness, grant.lease.id)).toEqual([]); + }); + + it("a release after settle() began calls the participant with an already aborted signal and destroys at once", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant] }); + const grant = await harness.engine.request(request, asking("agent")); + await harness.engine.settle(); + + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + + expect(participant.calls.map((call) => call.abortedAtCall)).toEqual([true]); + expect(destroys(harness.driver)).toHaveLength(1); + }); + + it("a participant that returns before the ceiling: advancing the clock past lease.endWaitMs afterwards logs no ceiling warning and aborts no signal", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + participant.calls[0]?.resolve(); + await flush(); + + harness.clock.advance(END_WAIT_MS + 1); + await flush(); + + expect(destroys(harness.driver)).toHaveLength(1); + expect(warningsFor(harness, grant.lease.id)).toEqual([]); + expect(participant.calls[0]?.signal.aborted).toBe(false); + }); + + it("dispose() with a participant still running aborts its signal; advancing the clock past lease.endWaitMs afterwards, and the participant then resolving, logs no ceiling warning and starts no destroy", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + + harness.engine.dispose(); + expect(participant.calls[0]?.signal.aborted).toBe(true); + harness.clock.advance(END_WAIT_MS + 1); + participant.calls[0]?.resolve(); + await flush(); + + expect(warningsFor(harness, grant.lease.id)).toEqual([]); + expect(destroys(harness.driver)).toEqual([]); + }); + + it("with no participant, the destroy starts without the clock advancing, the device is shut down while it runs, and the record carries no deferred-reclaim marker", async () => { + const world = await createWorld({ latencyMs: { destroy: 20 } }); + const harness = await startEngine({ world }); + const grant = await harness.engine.request(request, asking("agent")); + + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + + expect(destroys(harness.driver)).toHaveLength(1); + expect(stateOf(harness, grant.device.id)).toBe("shutdown"); + expect(markerOf(harness, grant.device.id)).toBeUndefined(); + }); + + it("with a participant, the device is reclaiming with its deferred-reclaim marker while the participant runs, and the marker is gone once the device is deleted", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + expect(stateOf(harness, grant.device.id)).toBe("reclaiming"); + expect(markerOf(harness, grant.device.id)).toBe(grant.lease.id); + + participant.calls[0]?.resolve(); + await flush(); + + expect(stateOf(harness, grant.device.id)).toBe("deleted"); + expect(markerOf(harness, grant.device.id)).toBeUndefined(); + }); + + it("with running capacity full, while a participant waits, a lease request for another spec queues, and a noWait request gets NO_CAPACITY; the queued one is granted once the wait and the destroy end", async () => { + const { harness, participant } = await leaseAndRelease({ + ios: { maxDevices: 2, maxRunning: 1 }, + }); + expect(participant.calls).toHaveLength(1); + + const queued = harness.engine.request(otherRequest, asking("queued")); + const refused = harness.engine.request(otherRequest, { ...asking("hurried"), noWait: true }); + + await expect(refused).rejects.toBeInstanceOf(NoCapacityError); + await expect(settledOrPending(queued)).resolves.toBe("still pending"); + participant.calls[0]?.resolve(); + await flush(); + await harness.engine.settle(); + await expect(settledOrPending(queued)).resolves.toMatchObject({ + lease: { requesterId: "queued" }, + }); + }); + + it("a daemon stopped while a participant has not returned: a new engine on the same registry wipes nothing, destroys the device, and calls no participant registered in the new engine", async () => { + const { grant, harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + const replacement = new TestParticipant(); + + const restarted = await startEngine({ participants: [replacement], world: harness.world }); + await restarted.engine.convergeRunningCapacity(); + await flush(); + await restarted.engine.settle(); + + expect(harness.driver.calls.filter((call) => call.operation === "reclaim")).toEqual([]); + expect(destroys(harness.driver)).toHaveLength(1); + expect(stateOf(restarted, grant.device.id)).toBe("deleted"); + expect(replacement.calls).toEqual([]); + }); + + it("a daemon stopped during the destroy that follows the wait destroys the device again at the next start", async () => { + const world = await createWorld({ latencyMs: { destroy: 20 } }); + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant], world }); + const grant = await harness.engine.request(request, asking("agent")); + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + expect(participant.calls).toHaveLength(1); + participant.calls[0]?.resolve(); + await flush(); + expect(destroys(world.driver)).toHaveLength(1); + const destroysBefore = destroys(world.driver).length; + + const restarted = await startEngine({ world }); + const converging = restarted.engine.convergeRunningCapacity(); + for (let tick = 0; tick < 5; tick += 1) { + world.clock.advance(20); + await flush(); + } + await converging; + + expect(destroys(world.driver).length - destroysBefore).toBe(1); + }); + + it("a release with a participant emits the same events in the same order as one without (lease.released, then device.deleted)", async () => { + const plain = await startEngine(); + const plainGrant = await plain.engine.request(request, asking("agent")); + const plainBefore = eventNames(plain).length; + await plain.engine.release(plainGrant.lease.id, "explicit"); + await flush(); + await plain.engine.settle(); + + const participant = new TestParticipant(); + const waiting = await startEngine({ participants: [participant] }); + const waitingGrant = await waiting.engine.request(request, asking("agent")); + const waitingBefore = eventNames(waiting).length; + await waiting.engine.release(waitingGrant.lease.id, "explicit"); + await flush(); + expect(participant.calls).toHaveLength(1); + participant.calls[0]?.resolve(); + await flush(); + await waiting.engine.settle(); + + const expected = eventNames(plain).slice(plainBefore); + expect(expected.indexOf("lease.released")).toBeLessThan(expected.indexOf("device.deleted")); + expect(eventNames(waiting).slice(waitingBefore)).toEqual(expected); + }); +}); diff --git a/src/leasing/lease-end-participants-startup-and-ceilings.test.ts b/src/leasing/lease-end-participants-startup-and-ceilings.test.ts new file mode 100644 index 000000000..39cf259ae --- /dev/null +++ b/src/leasing/lease-end-participants-startup-and-ceilings.test.ts @@ -0,0 +1,246 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { type DeviceRequest, UnknownLeaseError } from "../core/index.js"; +import { + END_WAIT_MS, + type Harness, + PROBE_MS, + TestParticipant, + asking, + destroys, + driverDeviceOf, + endStartupLease, + errorsFor, + flush, + leaseAndRelease, + markerOf, + request, + settledOrPending, + startEngine, + started, + stateOf, + withoutUnhandledRejections, +} from "./test-lease-end-participants.js"; + +const physicalRequest = { + model: "iPhone 15 Pro", + physical: true, + platform: "ios", +} as const satisfies DeviceRequest; + +afterEach(() => { + for (const engine of started.splice(0)) engine.dispose(); +}); + +describe("lease-end participants (#469)", () => { + it("a startup ending with reclaim whose participant has not returned, followed by core.converge: converge leaves the device alone, and the device is destroyed once the participant returns", async () => { + const participant = new TestParticipant(); + const { grant, second, world } = await endStartupLease({ + afterFirstStop: async (first, leased) => { + await first.driver.shutdown(driverDeviceOf(first, leased.device.id)); + }, + participant, + }); + const callsBefore = world.driver.calls.length; + + await second.engine.convergeRunningCapacity(); + await flush(); + + expect(participant.calls).toHaveLength(1); + expect( + world.driver.calls.slice(callsBefore).filter((call) => call.operation === "shutdown"), + ).toEqual([]); + expect( + world.bus + .replay() + .filter( + (event) => + String(event.event) === "device.shutdown" && + (event.payload as { initiator?: string }).initiator === "startup-interrupted-reclaim", + ), + ).toEqual([]); + expect(destroys(world.driver)).toHaveLength(0); + + participant.calls[0]?.resolve(); + await flush(); + await second.engine.settle(); + expect(destroys(world.driver)).toHaveLength(1); + expect(stateOf(second, grant.device.id)).toBe("deleted"); + }); + + it("a startup ending with wait calls the participant with its reason, starts no destroy, and leaves the device reclaiming with its deferred wipe recorded", async () => { + const participant = new TestParticipant(); + const { grant, second, world } = await endStartupLease({ + afterFirstStop: (first) => { + first.driver.failOn( + "listManaged", + first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, + new Error("cannot list"), + ); + }, + participant, + }); + + await second.engine.convergeRunningCapacity(); + await flush(); + participant.calls[0]?.resolve(); + await flush(); + + expect(participant.calls.map((call) => call.ended)).toMatchObject([ + { deviceId: grant.device.id, leaseId: grant.lease.id, reason: "device-lost" }, + ]); + expect(destroys(world.driver)).toHaveLength(0); + expect(stateOf(second, grant.device.id)).toBe("reclaiming"); + expect(markerOf(second, grant.device.id)).toBe(grant.lease.id); + }); + + it("a startup ending with missing calls the participant and marks the device missing, without waiting for it", async () => { + const participant = new TestParticipant(); + const { grant, second } = await endStartupLease({ + afterFirstStop: (first) => { + first.driver.setManagedReality({ devices: [], processes: [] }); + }, + participant, + }); + + await expect( + settledOrPending(second.engine.convergeRunningCapacity()), + ).resolves.toBeUndefined(); + + expect(participant.calls.map((call) => call.ended)).toMatchObject([ + { deviceId: grant.device.id, leaseId: grant.lease.id, reason: "device-lost" }, + ]); + expect(stateOf(second, grant.device.id)).toBe("deleted"); + expect(second.registry.snapshot.leases).toEqual([]); + }); + + it("a physical device that goes absent (through PhysicalDeviceWatcher) calls the participant with device-lost, and the device becomes absent without waiting for it", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant] }); + const device = await harness.registry.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: "UDID-A", + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + harness.driver.setPhysicalPresence("UDID-A", true); + ( + harness.engine.leasing as unknown as { physicalDeviceWatcher: { start(): void } } + ).physicalDeviceWatcher.start(); + await harness.engine.request(physicalRequest, asking("agent")); + + harness.driver.setPhysicalPresence("UDID-A", false); + for (let tick = 0; tick < 3; tick += 1) { + harness.clock.advance(PROBE_MS); + await flush(); + } + + expect(participant.calls.map((call) => call.ended)).toMatchObject([ + { deviceId: device.id, platform: "ios", reason: "device-lost" }, + ]); + expect(stateOf(harness, device.id)).toBe("absent"); + }); + + it("a call nothing waits for, still running at the ceiling, has its signal aborted then", async () => { + const participant = new TestParticipant(); + const { second } = await endStartupLease({ + afterFirstStop: (first) => { + first.driver.failOn( + "listManaged", + first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, + new Error("cannot list"), + ); + }, + participant, + }); + await second.engine.convergeRunningCapacity(); + await flush(); + expect(participant.calls).toHaveLength(1); + + second.clock.advance(END_WAIT_MS - 1); + await flush(); + expect(participant.calls[0]?.signal.aborted).toBe(false); + + second.clock.advance(1); + await flush(); + expect(participant.calls[0]?.signal.aborted).toBe(true); + }); + + it("a call nothing waits for that rejects is logged as an error with the lease id", async () => { + await withoutUnhandledRejections(async () => { + const participant = new TestParticipant(); + const { grant, second, world } = await endStartupLease({ + afterFirstStop: (first) => { + first.driver.failOn( + "listManaged", + first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, + new Error("cannot list"), + ); + }, + participant, + }); + await second.engine.convergeRunningCapacity(); + await flush(); + expect(participant.calls).toHaveLength(1); + + participant.calls[0]?.reject(new Error("rejected unwaited")); + await flush(); + + expect(errorsFor(world, grant.lease.id)).toMatchObject([ + { fields: { error: "rejected unwaited" } }, + ]); + }); + }); + + it("a release of an unknown lease calls no participant", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant] }); + await harness.engine.request(request, asking("agent")); + + await expect(harness.engine.release("no-such-lease", "explicit")).rejects.toBeInstanceOf( + UnknownLeaseError, + ); + await flush(); + + expect(participant.calls).toEqual([]); + }); + + it("an expiry whose deadline was moved by a renew calls no participant", async () => { + const participant = new TestParticipant(); + const harness = await startEngine({ + lease: { defaultTtlMs: 100 }, + participants: [participant], + }); + const grant = await harness.engine.request(request, asking("agent")); + harness.clock.advance(90); + await harness.engine.renew(grant.lease.id, 100); + + harness.clock.advance(20); + await flush(); + + expect(harness.registry.snapshot.leases).toHaveLength(1); + expect(participant.calls).toEqual([]); + }); + + it.each([ + [ + "nuke", + (harness: Harness) => harness.engine.nuke(false), + { releasedLeaseIds: [], removedBaselines: [] }, + ], + ["settle()", (harness: Harness) => harness.engine.settle(), undefined], + ] as const)( + "%s while a participant has not returned aborts its signal, starts the destroy at once, and completes without the clock advancing", + async (_call, run, completed) => { + const { harness, participant } = await leaseAndRelease(); + expect(participant.calls).toHaveLength(1); + + const running = run(harness); + + await expect(settledOrPending(running)).resolves.toEqual(completed); + expect(participant.calls[0]?.signal.aborted).toBe(true); + expect(destroys(harness.driver)).toHaveLength(1); + }, + ); +}); diff --git a/src/leasing/lease-end-participants.test.ts b/src/leasing/lease-end-participants.test.ts index b07ed5fe3..f1d19d25a 100644 --- a/src/leasing/lease-end-participants.test.ts +++ b/src/leasing/lease-end-participants.test.ts @@ -1,74 +1,25 @@ import { afterEach, describe, expect, it } from "vitest"; -import { EventBus } from "../bus/index.js"; +import { type LeaseGrant } from "../core/index.js"; +import type { LeaseEndParticipant } from "./lease-ports.js"; import { - type Config, - type DeviceRequest, - type DriverDevice, - type LeaseGrant, - type Registry, - UnknownLeaseError, -} from "../core/index.js"; -import { - FakeDriver, - type FakeDriverOptions, - loadTestRegistry, - openTestState, - testComponentWiring, - testStateOf, -} from "../core/testing.js"; -import type { StateDatabase } from "../state/index.js"; -import { FakeClock, FakeSystemStats, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; -import { NoCapacityError } from "./lease-acquisition-coordinator.js"; -import type { EndedLease, LeaseEndParticipant } from "./lease-ports.js"; -import { createTestEngine, type TestEngine, type TestEngineConfig } from "./testing.js"; - -const gibibyte = 1024 ** 3; -const END_WAIT_MS = 600_000; -const PROBE_MS = 30_000; - -const request = { model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; -const otherRequest = { model: "iPhone 15", osVersion: "26.5", platform: "ios" } as const; -const physicalRequest = { - model: "iPhone 15 Pro", - physical: true, - platform: "ios", -} as const satisfies DeviceRequest; - -interface Observed { - readonly claimed: boolean; - readonly deviceState: string | undefined; - readonly events: readonly string[]; - readonly leaseCount: number; - readonly marker: string | undefined; -} - -interface RecordedCall { - readonly abortedAtCall: boolean; - readonly ended: EndedLease; - readonly observed: Observed | undefined; - readonly signal: AbortSignal; - reject(error: Error): void; - resolve(): void; -} - -class TestParticipant implements LeaseEndParticipant { - readonly calls: RecordedCall[] = []; - observer: ((ended: EndedLease) => Observed) | undefined; - - leaseEnded(ended: EndedLease, signal: AbortSignal): Promise { - const observed = this.observer?.(ended); - return new Promise((resolve, reject) => { - this.calls.push({ - abortedAtCall: signal.aborted, - ended, - observed, - reject, - resolve, - signal, - }); - }); - } -} + END_WAIT_MS, + PROBE_MS, + TestParticipant, + asking, + destroys, + driverDeviceOf, + endStartupLease, + errorsFor, + flush, + leaseAndRelease, + request, + settledOrPending, + startEngine, + started, + stateOf, + warningsFor, + withoutUnhandledRejections, +} from "./test-lease-end-participants.js"; class ThrowingParticipant implements LeaseEndParticipant { leaseEnded(): Promise { @@ -76,239 +27,10 @@ class ThrowingParticipant implements LeaseEndParticipant { } } -interface World { - readonly bus: EventBus; - readonly clock: FakeClock; - readonly driver: FakeDriver; - readonly state: StateDatabase; - readonly ids: { next: number }; - readonly sink: MemoryLogSink; -} - -async function createWorld(driverOptions: Partial = {}): Promise { - const clock = new FakeClock(1_000); - return { - bus: new EventBus(clock), - clock, - driver: new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - platform: "ios", - ...driverOptions, - }), - ids: { next: 1 }, - sink: new MemoryLogSink(), - state: await openTestState(), - }; -} - -interface EngineOptions { - readonly ios?: { readonly maxDevices: number; readonly maxRunning: number }; - readonly lease?: Partial; - readonly participants?: readonly LeaseEndParticipant[]; - readonly world?: World; -} - -function config(options: EngineOptions): TestEngineConfig { - const ios = options.ios ?? { maxDevices: 1, maxRunning: 1 }; - return { - devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, - health: { - enabled: true, - maxConcurrentRecoveries: 1, - maxRecoveryAttempts: 3, - probeIntervalMs: PROBE_MS, - recoveryBackoffMs: 5_000, - stableObservations: 2, - }, - stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, - idle: { deleteAfterMs: 48 * 60 * 60_000, shutdownAfterMs: 24 * 60 * 60_000 }, - lease: { - defaultTtlMs: 3_600_000, - maxTtlMs: 14_400_000, - identity: { ios: "reusable", android: "reusable" }, - requestRetentionMs: 600_000, - endedIdRetentionMs: 600_000, - maxRequestRecords: 10_000, - maxPerRequester: 1, - endWaitMs: END_WAIT_MS, - ...options.lease, - }, - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios, - maxRunning: ios.maxRunning + 1, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - }, - warmPool: { - enabled: true, - maxConcurrentBoots: 1, - reserveRunning: { android: 0, ios: 0 }, - targets: [], - quarantine: { - maxRetries: 3, - maxRetryBackoffMs: 300_000, - retryBackoffMs: 30_000, - retryBackoffMultiplier: 2, - }, - }, - }; -} - -interface Harness extends World { - readonly engine: TestEngine; - readonly registry: Registry; - readonly world: World; -} - -const started: TestEngine[] = []; - afterEach(() => { for (const engine of started.splice(0)) engine.dispose(); }); -async function startEngine(options: EngineOptions = {}): Promise { - const world = options.world ?? (await createWorld()); - const { bus, clock, driver, ids, sink, state } = world; - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => `${ids.next++}` }, - state, - }); - const engine = createTestEngine({ - ...testComponentWiring({ clock, drivers: [driver], eventBus: bus, registry }), - clock, - config: config(options), - drivers: [driver], - eventBus: bus, - idGenerator: { generate: () => `request-${ids.next++}` }, - ...(options.participants === undefined ? {} : { leaseEndParticipants: options.participants }), - logger: new JsonLinesLogger({ clock, level: "debug", sink }), - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - }); - started.push(engine); - for (const participant of options.participants ?? []) { - if (participant instanceof TestParticipant) { - participant.observer = (ended) => { - const device = registry.snapshot.devices.find( - (candidate) => candidate.id === ended.deviceId, - ); - return { - claimed: engine.claimReader.isClaimed(ended.deviceId), - deviceState: device?.state, - events: bus.replay().map((event) => String(event.event)), - leaseCount: registry.snapshot.leases.length, - marker: (device as { deferredReclaimLeaseId?: string } | undefined) - ?.deferredReclaimLeaseId, - }; - }; - } - } - return { ...world, engine, registry, world }; -} - -function asking(requesterId: string) { - return { ownerId: `${requesterId}-owner`, requesterId }; -} - -async function flush(): Promise { - for (let count = 0; count < 500; count += 1) { - await Promise.resolve(); - } -} - -async function settledOrPending(promise: Promise): Promise { - return Promise.race([ - promise.catch((error: unknown) => error), - flush().then(() => "still pending"), - ]); -} - -/** why: a released virtual device is destroyed, never reclaimed, so the destroy marks the end of the wait. */ -function destroys(driver: FakeDriver) { - return driver.calls.filter((call) => call.operation === "destroy"); -} - -function stateOf(harness: Harness, deviceId: string): string | undefined { - return harness.registry.snapshot.devices.find((device) => device.id === deviceId)?.state; -} - -function markerOf(harness: Harness, deviceId: string): string | undefined { - const device = harness.registry.snapshot.devices.find((candidate) => candidate.id === deviceId); - return (device as { deferredReclaimLeaseId?: string } | undefined)?.deferredReclaimLeaseId; -} - -function warningsFor(world: World, leaseId: string) { - return world.sink.records.filter( - (record) => record.level === "warn" && record.fields?.leaseId === leaseId, - ); -} - -function errorsFor(world: World, leaseId: string) { - return world.sink.records.filter( - (record) => record.level === "error" && record.fields?.leaseId === leaseId, - ); -} - -function eventNames(world: World): string[] { - return world.bus.replay().map((event) => String(event.event)); -} - -async function withoutUnhandledRejections(run: () => Promise): Promise { - const seen: unknown[] = []; - const listener = (reason: unknown): void => { - seen.push(reason); - }; - process.on("unhandledRejection", listener); - try { - await run(); - await new Promise((resolve) => setImmediate(resolve)); - } finally { - process.off("unhandledRejection", listener); - } - expect(seen).toEqual([]); -} - -async function leaseAndRelease(options: EngineOptions = {}) { - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant], ...options }); - const grant = await harness.engine.request(request, asking("agent")); - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - return { grant, harness, participant }; -} - -async function endStartupLease(options: { - readonly afterFirstStop?: (first: Harness, grant: LeaseGrant) => void | Promise; - readonly participant: TestParticipant; - readonly ttlMs?: number; -}) { - const world = await createWorld(); - const first = await startEngine({ world }); - const grant = await first.engine.request(request, { - ...asking("agent"), - ...(options.ttlMs === undefined ? {} : { ttlMs: options.ttlMs }), - }); - first.engine.dispose(); - await options.afterFirstStop?.(first, grant); - const second = await startEngine({ participants: [options.participant], world }); - return { grant, second, world }; -} - -function driverDeviceOf(harness: Harness, deviceId: string): DriverDevice { - const device = harness.registry.snapshot.devices.find((candidate) => candidate.id === deviceId); - if (device === undefined) throw new Error(`no device ${deviceId}`); - return { address: "", deviceId: device.driverDeviceId, driverData: device.driverData }; -} - describe("lease-end participants (#469)", () => { it("a release answers its caller while the participant has not returned, and the participant gets the lease id, owner, simlock device id, platform and reason explicit", async () => { const participant = new TestParticipant(); @@ -610,424 +332,4 @@ describe("lease-end participants (#469)", () => { expect(destroys(world.driver)).toHaveLength(0); }, ); - - it("a startup ending with reclaim whose participant has not returned, followed by core.converge: converge leaves the device alone, and the device is destroyed once the participant returns", async () => { - const participant = new TestParticipant(); - const { grant, second, world } = await endStartupLease({ - afterFirstStop: async (first, leased) => { - await first.driver.shutdown(driverDeviceOf(first, leased.device.id)); - }, - participant, - }); - const callsBefore = world.driver.calls.length; - - await second.engine.convergeRunningCapacity(); - await flush(); - - expect(participant.calls).toHaveLength(1); - expect( - world.driver.calls.slice(callsBefore).filter((call) => call.operation === "shutdown"), - ).toEqual([]); - expect( - world.bus - .replay() - .filter( - (event) => - String(event.event) === "device.shutdown" && - (event.payload as { initiator?: string }).initiator === "startup-interrupted-reclaim", - ), - ).toEqual([]); - expect(destroys(world.driver)).toHaveLength(0); - - participant.calls[0]?.resolve(); - await flush(); - await second.engine.settle(); - expect(destroys(world.driver)).toHaveLength(1); - expect(stateOf(second, grant.device.id)).toBe("deleted"); - }); - - it("a startup ending with wait calls the participant with its reason, starts no destroy, and leaves the device reclaiming with its deferred wipe recorded", async () => { - const participant = new TestParticipant(); - const { grant, second, world } = await endStartupLease({ - afterFirstStop: (first) => { - first.driver.failOn( - "listManaged", - first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, - new Error("cannot list"), - ); - }, - participant, - }); - - await second.engine.convergeRunningCapacity(); - await flush(); - participant.calls[0]?.resolve(); - await flush(); - - expect(participant.calls.map((call) => call.ended)).toMatchObject([ - { deviceId: grant.device.id, leaseId: grant.lease.id, reason: "device-lost" }, - ]); - expect(destroys(world.driver)).toHaveLength(0); - expect(stateOf(second, grant.device.id)).toBe("reclaiming"); - expect(markerOf(second, grant.device.id)).toBe(grant.lease.id); - }); - - it("a startup ending with missing calls the participant and marks the device missing, without waiting for it", async () => { - const participant = new TestParticipant(); - const { grant, second } = await endStartupLease({ - afterFirstStop: (first) => { - first.driver.setManagedReality({ devices: [], processes: [] }); - }, - participant, - }); - - await expect( - settledOrPending(second.engine.convergeRunningCapacity()), - ).resolves.toBeUndefined(); - - expect(participant.calls.map((call) => call.ended)).toMatchObject([ - { deviceId: grant.device.id, leaseId: grant.lease.id, reason: "device-lost" }, - ]); - expect(stateOf(second, grant.device.id)).toBe("deleted"); - expect(second.registry.snapshot.leases).toEqual([]); - }); - - it("a physical device that goes absent (through PhysicalDeviceWatcher) calls the participant with device-lost, and the device becomes absent without waiting for it", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant] }); - const device = await harness.registry.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: "UDID-A", - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - harness.driver.setPhysicalPresence("UDID-A", true); - ( - harness.engine.leasing as unknown as { physicalDeviceWatcher: { start(): void } } - ).physicalDeviceWatcher.start(); - await harness.engine.request(physicalRequest, asking("agent")); - - harness.driver.setPhysicalPresence("UDID-A", false); - for (let tick = 0; tick < 3; tick += 1) { - harness.clock.advance(PROBE_MS); - await flush(); - } - - expect(participant.calls.map((call) => call.ended)).toMatchObject([ - { deviceId: device.id, platform: "ios", reason: "device-lost" }, - ]); - expect(stateOf(harness, device.id)).toBe("absent"); - }); - - it("a call nothing waits for, still running at the ceiling, has its signal aborted then", async () => { - const participant = new TestParticipant(); - const { second } = await endStartupLease({ - afterFirstStop: (first) => { - first.driver.failOn( - "listManaged", - first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, - new Error("cannot list"), - ); - }, - participant, - }); - await second.engine.convergeRunningCapacity(); - await flush(); - expect(participant.calls).toHaveLength(1); - - second.clock.advance(END_WAIT_MS - 1); - await flush(); - expect(participant.calls[0]?.signal.aborted).toBe(false); - - second.clock.advance(1); - await flush(); - expect(participant.calls[0]?.signal.aborted).toBe(true); - }); - - it("a call nothing waits for that rejects is logged as an error with the lease id", async () => { - await withoutUnhandledRejections(async () => { - const participant = new TestParticipant(); - const { grant, second, world } = await endStartupLease({ - afterFirstStop: (first) => { - first.driver.failOn( - "listManaged", - first.driver.calls.filter((call) => call.operation === "listManaged").length + 1, - new Error("cannot list"), - ); - }, - participant, - }); - await second.engine.convergeRunningCapacity(); - await flush(); - expect(participant.calls).toHaveLength(1); - - participant.calls[0]?.reject(new Error("rejected unwaited")); - await flush(); - - expect(errorsFor(world, grant.lease.id)).toMatchObject([ - { fields: { error: "rejected unwaited" } }, - ]); - }); - }); - - it("a release of an unknown lease calls no participant", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant] }); - await harness.engine.request(request, asking("agent")); - - await expect(harness.engine.release("no-such-lease", "explicit")).rejects.toBeInstanceOf( - UnknownLeaseError, - ); - await flush(); - - expect(participant.calls).toEqual([]); - }); - - it("an expiry whose deadline was moved by a renew calls no participant", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ - lease: { defaultTtlMs: 100 }, - participants: [participant], - }); - const grant = await harness.engine.request(request, asking("agent")); - harness.clock.advance(90); - await harness.engine.renew(grant.lease.id, 100); - - harness.clock.advance(20); - await flush(); - - expect(harness.registry.snapshot.leases).toHaveLength(1); - expect(participant.calls).toEqual([]); - }); - - it.each([ - [ - "nuke", - (harness: Harness) => harness.engine.nuke(false), - { releasedLeaseIds: [], removedBaselines: [] }, - ], - ["settle()", (harness: Harness) => harness.engine.settle(), undefined], - ] as const)( - "%s while a participant has not returned aborts its signal, starts the destroy at once, and completes without the clock advancing", - async (_call, run, completed) => { - const { harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - - const running = run(harness); - - await expect(settledOrPending(running)).resolves.toEqual(completed); - expect(participant.calls[0]?.signal.aborted).toBe(true); - expect(destroys(harness.driver)).toHaveLength(1); - }, - ); - - it("advancing the clock past lease.endWaitMs after a nuke cut a wait short, and the participant then resolving, logs no ceiling warning and starts no second destroy", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - await harness.engine.nuke(false); - expect(destroys(harness.driver)).toHaveLength(1); - - harness.clock.advance(END_WAIT_MS + 1); - participant.calls[0]?.resolve(); - await flush(); - - expect(warningsFor(harness, grant.lease.id)).toEqual([]); - expect(destroys(harness.driver)).toHaveLength(1); - }); - - it("nuke with leased devices calls each participant with an already aborted signal, and completes without the clock advancing even when the participant never returns", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ - ios: { maxDevices: 2, maxRunning: 2 }, - participants: [participant], - }); - const one = await harness.engine.request(request, asking("one")); - const two = await harness.engine.request(request, asking("two")); - - const nuking = harness.engine.nuke(false); - - await expect(settledOrPending(nuking)).resolves.toMatchObject({ - releasedLeaseIds: expect.arrayContaining([one.lease.id, two.lease.id]), - }); - expect(participant.calls.map((call) => call.abortedAtCall)).toEqual([true, true]); - }); - - it("after nuke ends, a release waits for its participant again", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant] }); - await harness.engine.nuke(false); - const grant = await harness.engine.request(request, asking("agent")); - - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - - expect(participant.calls).toHaveLength(1); - expect(participant.calls[0]?.abortedAtCall).toBe(false); - expect(destroys(harness.driver)).toHaveLength(0); - }); - - it("when the participant resolves after settle(), the driver's destroy has run once, and advancing past lease.endWaitMs logs no ceiling warning", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - await harness.engine.settle(); - - participant.calls[0]?.resolve(); - harness.clock.advance(END_WAIT_MS + 1); - await flush(); - - expect(destroys(harness.driver)).toHaveLength(1); - expect(warningsFor(harness, grant.lease.id)).toEqual([]); - }); - - it("a release after settle() began calls the participant with an already aborted signal and destroys at once", async () => { - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant] }); - const grant = await harness.engine.request(request, asking("agent")); - await harness.engine.settle(); - - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - - expect(participant.calls.map((call) => call.abortedAtCall)).toEqual([true]); - expect(destroys(harness.driver)).toHaveLength(1); - }); - - it("a participant that returns before the ceiling: advancing the clock past lease.endWaitMs afterwards logs no ceiling warning and aborts no signal", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - participant.calls[0]?.resolve(); - await flush(); - - harness.clock.advance(END_WAIT_MS + 1); - await flush(); - - expect(destroys(harness.driver)).toHaveLength(1); - expect(warningsFor(harness, grant.lease.id)).toEqual([]); - expect(participant.calls[0]?.signal.aborted).toBe(false); - }); - - it("dispose() with a participant still running aborts its signal; advancing the clock past lease.endWaitMs afterwards, and the participant then resolving, logs no ceiling warning and starts no destroy", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - - harness.engine.dispose(); - expect(participant.calls[0]?.signal.aborted).toBe(true); - harness.clock.advance(END_WAIT_MS + 1); - participant.calls[0]?.resolve(); - await flush(); - - expect(warningsFor(harness, grant.lease.id)).toEqual([]); - expect(destroys(harness.driver)).toEqual([]); - }); - - it("with no participant, the destroy starts without the clock advancing, the device is shut down while it runs, and the record carries no deferred-reclaim marker", async () => { - const world = await createWorld({ latencyMs: { destroy: 20 } }); - const harness = await startEngine({ world }); - const grant = await harness.engine.request(request, asking("agent")); - - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - - expect(destroys(harness.driver)).toHaveLength(1); - expect(stateOf(harness, grant.device.id)).toBe("shutdown"); - expect(markerOf(harness, grant.device.id)).toBeUndefined(); - }); - - it("with a participant, the device is reclaiming with its deferred-reclaim marker while the participant runs, and the marker is gone once the device is deleted", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - expect(stateOf(harness, grant.device.id)).toBe("reclaiming"); - expect(markerOf(harness, grant.device.id)).toBe(grant.lease.id); - - participant.calls[0]?.resolve(); - await flush(); - - expect(stateOf(harness, grant.device.id)).toBe("deleted"); - expect(markerOf(harness, grant.device.id)).toBeUndefined(); - }); - - it("with running capacity full, while a participant waits, a lease request for another spec queues, and a noWait request gets NO_CAPACITY; the queued one is granted once the wait and the destroy end", async () => { - const { harness, participant } = await leaseAndRelease({ - ios: { maxDevices: 2, maxRunning: 1 }, - }); - expect(participant.calls).toHaveLength(1); - - const queued = harness.engine.request(otherRequest, asking("queued")); - const refused = harness.engine.request(otherRequest, { ...asking("hurried"), noWait: true }); - - await expect(refused).rejects.toBeInstanceOf(NoCapacityError); - await expect(settledOrPending(queued)).resolves.toBe("still pending"); - participant.calls[0]?.resolve(); - await flush(); - await harness.engine.settle(); - await expect(settledOrPending(queued)).resolves.toMatchObject({ - lease: { requesterId: "queued" }, - }); - }); - - it("a daemon stopped while a participant has not returned: a new engine on the same registry wipes nothing, destroys the device, and calls no participant registered in the new engine", async () => { - const { grant, harness, participant } = await leaseAndRelease(); - expect(participant.calls).toHaveLength(1); - const replacement = new TestParticipant(); - - const restarted = await startEngine({ participants: [replacement], world: harness.world }); - await restarted.engine.convergeRunningCapacity(); - await flush(); - await restarted.engine.settle(); - - expect(harness.driver.calls.filter((call) => call.operation === "reclaim")).toEqual([]); - expect(destroys(harness.driver)).toHaveLength(1); - expect(stateOf(restarted, grant.device.id)).toBe("deleted"); - expect(replacement.calls).toEqual([]); - }); - - it("a daemon stopped during the destroy that follows the wait destroys the device again at the next start", async () => { - const world = await createWorld({ latencyMs: { destroy: 20 } }); - const participant = new TestParticipant(); - const harness = await startEngine({ participants: [participant], world }); - const grant = await harness.engine.request(request, asking("agent")); - await harness.engine.release(grant.lease.id, "explicit"); - await flush(); - expect(participant.calls).toHaveLength(1); - participant.calls[0]?.resolve(); - await flush(); - expect(destroys(world.driver)).toHaveLength(1); - const destroysBefore = destroys(world.driver).length; - - const restarted = await startEngine({ world }); - const converging = restarted.engine.convergeRunningCapacity(); - for (let tick = 0; tick < 5; tick += 1) { - world.clock.advance(20); - await flush(); - } - await converging; - - expect(destroys(world.driver).length - destroysBefore).toBe(1); - }); - - it("a release with a participant emits the same events in the same order as one without (lease.released, then device.deleted)", async () => { - const plain = await startEngine(); - const plainGrant = await plain.engine.request(request, asking("agent")); - const plainBefore = eventNames(plain).length; - await plain.engine.release(plainGrant.lease.id, "explicit"); - await flush(); - await plain.engine.settle(); - - const participant = new TestParticipant(); - const waiting = await startEngine({ participants: [participant] }); - const waitingGrant = await waiting.engine.request(request, asking("agent")); - const waitingBefore = eventNames(waiting).length; - await waiting.engine.release(waitingGrant.lease.id, "explicit"); - await flush(); - expect(participant.calls).toHaveLength(1); - participant.calls[0]?.resolve(); - await flush(); - await waiting.engine.settle(); - - const expected = eventNames(plain).slice(plainBefore); - expect(expected.indexOf("lease.released")).toBeLessThan(expected.indexOf("device.deleted")); - expect(eventNames(waiting).slice(waitingBefore)).toEqual(expected); - }); }); diff --git a/src/leasing/test-lease-end-participants.ts b/src/leasing/test-lease-end-participants.ts new file mode 100644 index 000000000..98b68d72e --- /dev/null +++ b/src/leasing/test-lease-end-participants.ts @@ -0,0 +1,284 @@ +import { expect } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { type Config, type DriverDevice, type LeaseGrant, type Registry } from "../core/index.js"; +import { + FakeDriver, + type FakeDriverOptions, + loadTestRegistry, + openTestState, + testComponentWiring, + testStateOf, +} from "../core/testing.js"; +import { type StateDatabase } from "../state/index.js"; +import { FakeClock, FakeSystemStats, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; +import { type EndedLease, type LeaseEndParticipant } from "./lease-ports.js"; +import { createTestEngine, type TestEngine, type TestEngineConfig } from "./testing.js"; + +const gibibyte = 1024 ** 3; + +export const END_WAIT_MS = 600_000; + +export const PROBE_MS = 30_000; + +export const request = { model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; + +interface Observed { + readonly claimed: boolean; + readonly deviceState: string | undefined; + readonly events: readonly string[]; + readonly leaseCount: number; + readonly marker: string | undefined; +} + +interface RecordedCall { + readonly abortedAtCall: boolean; + readonly ended: EndedLease; + readonly observed: Observed | undefined; + readonly signal: AbortSignal; + reject(error: Error): void; + resolve(): void; +} + +export class TestParticipant implements LeaseEndParticipant { + readonly calls: RecordedCall[] = []; + observer: ((ended: EndedLease) => Observed) | undefined; + + leaseEnded(ended: EndedLease, signal: AbortSignal): Promise { + const observed = this.observer?.(ended); + return new Promise((resolve, reject) => { + this.calls.push({ + abortedAtCall: signal.aborted, + ended, + observed, + reject, + resolve, + signal, + }); + }); + } +} + +export interface World { + readonly bus: EventBus; + readonly clock: FakeClock; + readonly driver: FakeDriver; + readonly state: StateDatabase; + readonly ids: { next: number }; + readonly sink: MemoryLogSink; +} + +export async function createWorld(driverOptions: Partial = {}): Promise { + const clock = new FakeClock(1_000); + return { + bus: new EventBus(clock), + clock, + driver: new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + platform: "ios", + ...driverOptions, + }), + ids: { next: 1 }, + sink: new MemoryLogSink(), + state: await openTestState(), + }; +} + +interface EngineOptions { + readonly ios?: { readonly maxDevices: number; readonly maxRunning: number }; + readonly lease?: Partial; + readonly participants?: readonly LeaseEndParticipant[]; + readonly world?: World; +} + +function config(options: EngineOptions): TestEngineConfig { + const ios = options.ios ?? { maxDevices: 1, maxRunning: 1 }; + return { + devices: { autoEnroll: false, deletedRetentionMs: 3_600_000 }, + health: { + enabled: true, + maxConcurrentRecoveries: 1, + maxRecoveryAttempts: 3, + probeIntervalMs: PROBE_MS, + recoveryBackoffMs: 5_000, + stableObservations: 2, + }, + stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + idle: { deleteAfterMs: 48 * 60 * 60_000, shutdownAfterMs: 24 * 60 * 60_000 }, + lease: { + defaultTtlMs: 3_600_000, + maxTtlMs: 14_400_000, + identity: { ios: "reusable", android: "reusable" }, + requestRetentionMs: 600_000, + endedIdRetentionMs: 600_000, + maxRequestRecords: 10_000, + maxPerRequester: 1, + endWaitMs: END_WAIT_MS, + ...options.lease, + }, + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios, + maxRunning: ios.maxRunning + 1, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + warmPool: { + enabled: true, + maxConcurrentBoots: 1, + reserveRunning: { android: 0, ios: 0 }, + targets: [], + quarantine: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + }, + }; +} + +export interface Harness extends World { + readonly engine: TestEngine; + readonly registry: Registry; + readonly world: World; +} + +export const started: TestEngine[] = []; + +export async function startEngine(options: EngineOptions = {}): Promise { + const world = options.world ?? (await createWorld()); + const { bus, clock, driver, ids, sink, state } = world; + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => `${ids.next++}` }, + state, + }); + const engine = createTestEngine({ + ...testComponentWiring({ clock, drivers: [driver], eventBus: bus, registry }), + clock, + config: config(options), + drivers: [driver], + eventBus: bus, + idGenerator: { generate: () => `request-${ids.next++}` }, + ...(options.participants === undefined ? {} : { leaseEndParticipants: options.participants }), + logger: new JsonLinesLogger({ clock, level: "debug", sink }), + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + }); + started.push(engine); + for (const participant of options.participants ?? []) { + if (participant instanceof TestParticipant) { + participant.observer = (ended) => { + const device = registry.snapshot.devices.find( + (candidate) => candidate.id === ended.deviceId, + ); + return { + claimed: engine.claimReader.isClaimed(ended.deviceId), + deviceState: device?.state, + events: bus.replay().map((event) => String(event.event)), + leaseCount: registry.snapshot.leases.length, + marker: (device as { deferredReclaimLeaseId?: string } | undefined) + ?.deferredReclaimLeaseId, + }; + }; + } + } + return { ...world, engine, registry, world }; +} + +export function asking(requesterId: string) { + return { ownerId: `${requesterId}-owner`, requesterId }; +} + +export async function flush(): Promise { + for (let count = 0; count < 500; count += 1) { + await Promise.resolve(); + } +} + +export async function settledOrPending(promise: Promise): Promise { + return Promise.race([ + promise.catch((error: unknown) => error), + flush().then(() => "still pending"), + ]); +} + +export /** why: a released virtual device is destroyed, never reclaimed, so the destroy marks the end of the wait. */ +function destroys(driver: FakeDriver) { + return driver.calls.filter((call) => call.operation === "destroy"); +} + +export function stateOf(harness: Harness, deviceId: string): string | undefined { + return harness.registry.snapshot.devices.find((device) => device.id === deviceId)?.state; +} + +export function markerOf(harness: Harness, deviceId: string): string | undefined { + const device = harness.registry.snapshot.devices.find((candidate) => candidate.id === deviceId); + return (device as { deferredReclaimLeaseId?: string } | undefined)?.deferredReclaimLeaseId; +} + +export function warningsFor(world: World, leaseId: string) { + return world.sink.records.filter( + (record) => record.level === "warn" && record.fields?.leaseId === leaseId, + ); +} + +export function errorsFor(world: World, leaseId: string) { + return world.sink.records.filter( + (record) => record.level === "error" && record.fields?.leaseId === leaseId, + ); +} + +export async function withoutUnhandledRejections(run: () => Promise): Promise { + const seen: unknown[] = []; + const listener = (reason: unknown): void => { + seen.push(reason); + }; + process.on("unhandledRejection", listener); + try { + await run(); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + process.off("unhandledRejection", listener); + } + expect(seen).toEqual([]); +} + +export async function leaseAndRelease(options: EngineOptions = {}) { + const participant = new TestParticipant(); + const harness = await startEngine({ participants: [participant], ...options }); + const grant = await harness.engine.request(request, asking("agent")); + await harness.engine.release(grant.lease.id, "explicit"); + await flush(); + return { grant, harness, participant }; +} + +export async function endStartupLease(options: { + readonly afterFirstStop?: (first: Harness, grant: LeaseGrant) => void | Promise; + readonly participant: TestParticipant; + readonly ttlMs?: number; +}) { + const world = await createWorld(); + const first = await startEngine({ world }); + const grant = await first.engine.request(request, { + ...asking("agent"), + ...(options.ttlMs === undefined ? {} : { ttlMs: options.ttlMs }), + }); + first.engine.dispose(); + await options.afterFirstStop?.(first, grant); + const second = await startEngine({ participants: [options.participant], world }); + return { grant, second, world }; +} + +export function driverDeviceOf(harness: Harness, deviceId: string): DriverDevice { + const device = harness.registry.snapshot.devices.find((candidate) => candidate.id === deviceId); + if (device === undefined) throw new Error(`no device ${deviceId}`); + return { address: "", deviceId: device.driverDeviceId, driverData: device.driverData }; +} From dfd16a82c5296f7e1ae66ba0d65170eff7b6d5b6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:45 +0100 Subject: [PATCH 41/83] test(gateway): split service.test.ts by topic --- src/gateway/service-link-lifecycle.test.ts | 528 ++++++ .../service-relays-and-refresh.test.ts | 468 +++++ src/gateway/service-worker-protocol.test.ts | 524 ++++++ src/gateway/service.test.ts | 1601 +---------------- src/gateway/test-service.ts | 118 ++ 5 files changed, 1640 insertions(+), 1599 deletions(-) create mode 100644 src/gateway/service-link-lifecycle.test.ts create mode 100644 src/gateway/service-relays-and-refresh.test.ts create mode 100644 src/gateway/service-worker-protocol.test.ts create mode 100644 src/gateway/test-service.ts diff --git a/src/gateway/service-link-lifecycle.test.ts b/src/gateway/service-link-lifecycle.test.ts new file mode 100644 index 000000000..699d56330 --- /dev/null +++ b/src/gateway/service-link-lifecycle.test.ts @@ -0,0 +1,528 @@ +import { describe, expect, it, vi } from "vitest"; +import { negotiateProtocolVersion, PROTOCOL_VERSION_RANGE } from "../contract/index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import { REJECTION_COALESCE_WINDOW_MS } from "./service.js"; +import { + deviceFixture, + hostFixture, + leaseFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { MAX_CONSECUTIVE_REFRESH_TIMEOUTS, WORKER_CALL_TIMEOUT_MS } from "./worker-link.js"; +import { eventNames, fleet, REFRESH_MS, RETENTION_MS } from "./test-service.js"; + +// why: the 8 MiB the gateway accepts for one unfinished worker reply line, written out so the test pins the number. +const REPLY_LINE_CAP = 8 * 1024 * 1024; + +describe("GatewayService", () => { + it("flips a view to disconnected the moment its uplink closes -- no polling", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + const workerEnd = await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + await workerEnd.close(); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), + ); + expect(eventNames(harness.events)).toContain("worker.disconnected"); + + await harness.service.stop(); + }); + + it("does not let a stale link's late close mark a reconnected worker disconnected (D1)", async () => { + const harness = fleet(); + await harness.service.start(); + + const clientA = new ScriptedWorkerClient("admin", "0.1.0"); + const workerEndA = await harness.join("wrk_1", clientA); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"), + ); + + clientA.hangUnsubscribe = true; + + const clientB = new ScriptedWorkerClient("admin", "9.9.9"); + await harness.join("wrk_1", clientB); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + + await workerEndA.close(); + + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); + expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9"); + expect(eventNames(harness.events)).not.toContain("worker.disconnected"); + + await harness.service.stop(); + }); + + it("does not hang close() forever behind an events.unsubscribe that never answers (D2)", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + worker.hangUnsubscribe = true; + + const stopped = harness.service.stop(); + const stoppedState = promiseState(stopped); + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + + await vi.waitFor(() => expect(stoppedState.state).toBe("fulfilled")); + await expect(stopped).resolves.toBeUndefined(); + expect(worker.closed).toBe(true); + }); + + it("does not latch refreshing forever when a round trip hangs, and recovers past the timeout (D3)", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + worker.hangingCalls.add("status.get"); + worker.status = statusFixture({ leases: [leaseFixture("lease_9", "dev_9")] }); + worker.pushEvent({ event: "lease.granted" }); + + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + + worker.hangingCalls.delete("status.get"); + worker.pushEvent({ event: "lease.granted" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.leases).toEqual([ + expect.objectContaining({ id: "lease_9" }), + ]), + ); + + await harness.service.stop(); + }); + + it("does not let a stale link's late-arriving refresh overwrite its successor's fresher view (H2)", async () => { + const harness = fleet(); + await harness.service.start(); + + const clientA = new ScriptedWorkerClient("admin", "0.1.0"); + await harness.join("wrk_1", clientA); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("0.1.0")); + + let resolveStatus: ((status: ReturnType) => void) | undefined; + clientA.getStatus = () => + new Promise((resolve) => { + resolveStatus = resolve; + }); + clientA.pushEvent({ event: "lease.granted" }); + await vi.waitFor(() => expect(resolveStatus).toBeDefined()); + + const clientB = new ScriptedWorkerClient("admin", "9.9.9"); + await harness.join("wrk_1", clientB); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + + resolveStatus?.(statusFixture()); + // why: A few real event-loop turns for A's now-resolved promise chain to run to completion (or, + // with the fix, to stop itself before writing) -- nothing here is clock-scheduled, so there + // is no fake-clock timer to advance instead. + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9"); + + await harness.service.stop(); + }); + + it("does not let a stale link's late starting answer overwrite its successor's view", async () => { + const harness = fleet(); + await harness.service.start(); + const clientA = new ScriptedWorkerClient("admin", "0.1.0"); + await harness.join("wrk_1", clientA); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("0.1.0")); + const release = clientA.holdStatus(); + clientA.pushEvent({ event: "lease.granted" }); + await vi.waitFor(() => expect(clientA.calls.at(-1)).toBe("status.get")); + const clientB = new ScriptedWorkerClient("admin", "9.9.9"); + clientB.devices = [deviceFixture("dev_b", "ready")]; + await harness.join("wrk_1", clientB); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + + clientA.status = { + daemon: { + health: "starting", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: hostFixture(), + }; + release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + devices: [{ id: "dev_b" }], + health: "running", + version: "9.9.9", + }); + await harness.service.stop(); + }); + + it("closes the uplink once refreshes keep timing out, so a half-open worker stops reporting connected (P1)", async () => { + expect(MAX_CONSECUTIVE_REFRESH_TIMEOUTS).toBeGreaterThan(1); + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + worker.hangingCalls.add("status.get"); + + for (let attempt = 1; attempt < MAX_CONSECUTIVE_REFRESH_TIMEOUTS; attempt++) { + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + // why: Wait for the *next* tick to actually be armed, not just for the (already, trivially + // true) `connected` check to pass -- that value never changes on a timeout, so checking + // it alone would resolve before the tick's own promise chain (refresh's catch, then + // `pruneExpired`, then rescheduling) has actually finished, and the next iteration's + // `advance(REFRESH_MS)` would fast-forward the clock past a timer that has not been + // registered yet -- silently skipping a whole tick instead of exercising it. + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); + } + + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), + ); + expect(eventNames(harness.events)).toContain("worker.disconnected"); + + await harness.service.stop(); + }); + + it("does not close the link when status.get keeps answering but the dispatcher is busy (C-1)", async () => { + // why: A refreshIntervalMs far outside this test's clock advances: `#runTick` only reschedules + // its own timer once a refresh settles (`service.ts`'s `.finally(() => this.#scheduleTick())`), + // so on a link whose refresh keeps timing out the tick's real cadence drifts to roughly + // `refreshIntervalMs + WORKER_CALL_TIMEOUT_MS` per cycle -- exercising this via event-driven + // refreshes instead (as D3 and H2 above do) keeps the periodic tick out of the picture + // entirely, rather than fighting its drift with hand-tuned clock advances. + const harness = fleet({ refreshIntervalMs: 10_000_000 }); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + worker.hangingCalls.add("list.get:devices"); + + for (let attempt = 1; attempt <= MAX_CONSECUTIVE_REFRESH_TIMEOUTS + 2; attempt++) { + const listCallsBefore = worker.calls.filter((call) => call === "list.get:devices").length; + worker.pushEvent({ event: "lease.granted" }); + // why: Waits for `list.get` to have actually been called -- proof `status.get`'s real promise + // was given a genuine chance to settle and `#rebuildView` moved on to the hung batch. + // Waiting on `pendingTimerCount` instead would be a false positive: `status.get`'s own + // bounded wait registers a timer *synchronously*, before its real response has had any + // chance to arrive, so a bare "a timer exists" check can pass and race the fake clock's + // `advance` below against `status.get`'s still-pending real resolution -- timing out the + // liveness call itself instead of the batch behind it, which is exactly the distinction + // this fix exists to preserve. + await vi.waitFor(() => + expect(worker.calls.filter((call) => call === "list.get:devices").length).toBeGreaterThan( + listCallsBefore, + ), + ); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); + } + expect(eventNames(harness.events)).not.toContain("worker.disconnected"); + + await harness.service.stop(); + }); + + it("closes exactly the link a token authorized, and leaves every other link alone (C-2)", async () => { + const harness = fleet({ + authenticate: (credential) => + credential === undefined + ? "unauthenticated" + : { outcome: "accept" as const, tokenId: credential }, + }); + await harness.service.start(); + + const workerA = new ScriptedWorkerClient(); + await harness.join("wrk_a", workerA, undefined, "tok_a"); + const workerB = new ScriptedWorkerClient(); + await harness.join("wrk_b", workerB, undefined, "tok_b"); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"), + ); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_b")?.connection).toBe("connected"), + ); + + await harness.service.closeLinksForToken("tok_a"); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_a")?.connection).toBe("disconnected"), + ); + expect(harness.service.workers.view("wrk_b")?.connection).toBe("connected"); + expect(eventNames(harness.events)).toEqual( + expect.arrayContaining(["worker.connected", "worker.connected", "worker.disconnected"]), + ); + + await harness.service.stop(); + }); + + it("does not hang the handshake forever when hello itself never answers (C3)", async () => { + const harness = fleet(); + await harness.service.start(); + + const workerEnd = await harness.joinSilent("wrk_1", "mac-mini-1"); + await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); + harness.clock.advance(WORKER_CALL_TIMEOUT_MS); + + await vi.waitFor(() => expect(workerEnd.closed).toBe(true)); + + expect(eventNames(harness.events)).not.toContain("worker.connected"); + expect(harness.service.workers.view("wrk_1")).toBeUndefined(); + + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"), + ); + + await harness.service.stop(); + }); + + it("closes an uplink whose worker sends an unfinished line past the reply cap, and keeps one at the cap", async () => { + const harness = fleet({ realConnect: true }); + await harness.service.start(); + const workerEnd = await harness.transport.connect({ + token: "join-secret", + url: "ws://gateway.test", + workerId: "wrk_1", + }); + let helloId: number | string | undefined; + workerEnd.onData((chunk) => { + const frame = JSON.parse(chunk.trim().split("\n")[0] ?? "{}") as { + id?: number | string; + type?: string; + }; + if (frame.type === "hello") helloId = frame.id; + }); + await vi.waitFor(() => expect(helloId).toBeDefined()); + await workerEnd.write( + `${JSON.stringify({ + id: helloId, + ok: true, + payload: { + daemonProtocolRange: PROTOCOL_VERSION_RANGE, + principal: "gw:instance-1", + protocolVersion: negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, PROTOCOL_VERSION_RANGE), + role: "admin", + version: "0.3.0", + }, + })}\n`, + ); + const chunk = "a".repeat(64 * 1024); + for (let sent = 0; sent < REPLY_LINE_CAP; sent += chunk.length) { + await workerEnd.write(chunk); + } + expect(workerEnd.closed).toBe(false); + await workerEnd.write(chunk); + await vi.waitFor(() => expect(workerEnd.closed).toBe(true)); + + await harness.service.stop(); + }); + + it("sweeps a retired view on the tick, once retention has passed", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + const workerEnd = await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + await workerEnd.close(); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), + ); + + harness.clock.advance(RETENTION_MS + REFRESH_MS); + + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")).toBeUndefined()); + expect(eventNames(harness.events)).toContain("worker.removed"); + + await harness.service.stop(); + }); + + it.each(["unauthenticated", "forbidden", "worker-mismatch"] as const)( + "reports a %s uplink it turned away, without inventing a worker for it", + async (reason) => { + const harness = fleet({ authenticate: () => reason }); + await harness.service.start(); + + await expect( + harness.join("wrk_1", new ScriptedWorkerClient(), "mac-mini-1"), + ).rejects.toMatchObject({ + code: "rejected", + }); + + expect(harness.service.workers.views()).toEqual([]); + expect(harness.events.at(-1)).toMatchObject({ + event: "worker.rejected", + payload: { label: "mac-mini-1", reason, workerId: "wrk_1" }, + }); + + await harness.service.stop(); + }, + ); + + it("refuses a second worker id on a token pinned to another worker as worker-mismatch, leaving that worker's link alone, while the pinned worker's own redial replaces its link", async () => { + const pins = new Map(); + const harness = fleet({ + authenticate: (credential, workerId) => { + if (!pins.has(credential)) pins.set(credential, workerId); + return pins.get(credential) === workerId + ? { outcome: "accept" as const, tokenId: "tok_a" } + : "worker-mismatch"; + }, + }); + await harness.service.start(); + const workerEndA = await harness.join("wrk_a", new ScriptedWorkerClient("admin", "1.0.0")); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"), + ); + + await expect( + harness.join("wrk_b", new ScriptedWorkerClient(), "intruder"), + ).rejects.toMatchObject({ code: "rejected" }); + + expect(harness.events.filter((event) => event.event === "worker.rejected")).toMatchObject([ + { payload: { label: "intruder", reason: "worker-mismatch", workerId: "wrk_b" } }, + ]); + expect(harness.service.workers.view("wrk_b")).toBeUndefined(); + expect(workerEndA.closed).toBe(false); + expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"); + + await harness.join("wrk_a", new ScriptedWorkerClient("admin", "2.0.0")); + + await vi.waitFor(() => expect(workerEndA.closed).toBe(true)); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_a")?.version).toBe("2.0.0")); + expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"); + + await harness.service.stop(); + }); + + it("coalesces a flood of identical refusals into one event per window, with a count only above one (P-2)", async () => { + const harness = fleet({ authenticate: () => "unauthenticated" }); + await harness.service.start(); + const rejectedEvents = () => + harness.events.filter((event) => event.event === "worker.rejected"); + + await expect( + harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), + ).rejects.toMatchObject({ code: "rejected" }); + await expect( + harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), + ).rejects.toMatchObject({ code: "rejected" }); + + expect(rejectedEvents()).toHaveLength(1); + expect(rejectedEvents()[0]?.payload).toMatchObject({ workerId: "wrk_flood" }); + expect(rejectedEvents()[0]?.payload).not.toHaveProperty("count"); + + harness.clock.advance(REJECTION_COALESCE_WINDOW_MS); + await expect( + harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), + ).rejects.toMatchObject({ code: "rejected" }); + + expect(rejectedEvents()).toHaveLength(2); + expect(rejectedEvents()[1]?.payload).toMatchObject({ count: 2, workerId: "wrk_flood" }); + + await expect( + harness.join("wrk_other", new ScriptedWorkerClient(), "mac-mini-2"), + ).rejects.toMatchObject({ code: "rejected" }); + expect(rejectedEvents()).toHaveLength(3); + expect(rejectedEvents()[2]?.payload).not.toHaveProperty("count"); + + // why: both EVENTS.md cuts document `count` as "present only above one", so a lone refusal + // whose predecessor window absorbed exactly one -- the normal path for a worker redialling + // on its backoff, which lands outside every window by design -- must not carry `count: 1`. + // Presence is the flood signal (#677). + harness.clock.advance(REJECTION_COALESCE_WINDOW_MS); + await expect( + harness.join("wrk_other", new ScriptedWorkerClient(), "mac-mini-2"), + ).rejects.toMatchObject({ code: "rejected" }); + expect(rejectedEvents()).toHaveLength(4); + expect(rejectedEvents()[3]?.payload).toMatchObject({ workerId: "wrk_other" }); + expect(rejectedEvents()[3]?.payload).not.toHaveProperty("count"); + + await harness.service.stop(); + }); + + it("closes the uplink when a worker does not grant the gateway admin", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient("agent"); + + const workerEnd = await harness.join("wrk_1", worker); + + await vi.waitFor(() => expect(worker.closed).toBe(true)); + expect(workerEnd.closed).toBe(true); + expect(harness.service.workers.views()).toEqual([]); + + await harness.service.stop(); + }); + + it("stops the tick and closes every link on stop", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + await harness.service.stop(); + + expect(worker.closed).toBe(true); + const callsAfterStop = worker.calls.length; + harness.clock.advance(10 * REFRESH_MS); + expect(worker.calls).toHaveLength(callsAfterStop); + }); + + describe("target() / WorkerDispatchTarget (fleet-ports)", () => { + it("has no client before the handshake completes", async () => { + const harness = fleet(); + await harness.service.start(); + + await harness.joinSilent("wrk_1"); + const target = harness.service.target("wrk_1"); + + expect(target).toBeDefined(); + expect(target?.reachable).toBe(true); + expect(target?.client()).toBeUndefined(); + + await harness.service.stop(); + }); + + it("has no client once the link has closed", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + + const target = harness.service.target("wrk_1"); + expect(target?.client()).toBeDefined(); + expect(target?.reachable).toBe(true); + + await harness.service.stop(); + + expect(target?.reachable).toBe(false); + expect(target?.client()).toBeUndefined(); + }); + }); +}); diff --git a/src/gateway/service-relays-and-refresh.test.ts b/src/gateway/service-relays-and-refresh.test.ts new file mode 100644 index 000000000..b676951b3 --- /dev/null +++ b/src/gateway/service-relays-and-refresh.test.ts @@ -0,0 +1,468 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryEndedLeaseIds } from "./index.js"; +import { FleetLeaseCoordinator } from "./fleet-coordinator.js"; +import { FleetLeaseIndex } from "./lease-index.js"; +import { createRoutingPolicy } from "./routing.js"; +import { + catalogFixture, + hostFixture, + leaseFixture, + physicalDeviceFixture, + type PhysicalPresenceFixture, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { eventNames, fleet, REFRESH_MS, RecordingLogger } from "./test-service.js"; + +// why: the documented bound (src/gateway/AGENTS.md); a payload nested this deep is refused. +const MAX_RELAYED_PAYLOAD_DEPTH = 64; + +describe("GatewayService", () => { + it("a relayed event differs from the worker's only in seq and payload.workerId", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + const sent = { + event: "lease.granted", + id: "evt_worker-made", + module: "lease-engine", + payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, + seq: 41, + timestamp: 777, + }; + + worker.pushEvent(sent); + + const relayed = harness.events.find((event) => event.event === "lease.granted"); + const { seq: relayedSeq, payload: relayedPayload, ...relayedRest } = relayed ?? {}; + const { seq: sentSeq, payload: sentPayload, ...sentRest } = sent; + expect(relayedRest).toEqual(sentRest); + expect(relayedPayload).toEqual({ ...sentPayload, workerId: "wrk_1" }); + expect(relayedSeq).not.toBe(sentSeq); + + await harness.service.stop(); + }); + + it.each([ + { levels: MAX_RELAYED_PAYLOAD_DEPTH - 1, relayed: true }, + { levels: MAX_RELAYED_PAYLOAD_DEPTH, relayed: false }, + { levels: 6_189, relayed: false }, + { levels: 20_000, relayed: false }, + ])( + "a relayed event whose payload nests $levels levels is relayed: $relayed, and a refused one logs one error while the next event arrives", + async ({ levels, relayed }) => { + const logger = new RecordingLogger(); + const harness = fleet({ logger }); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + const nested = JSON.parse("[".repeat(levels) + "]".repeat(levels)) as unknown; + + worker.pushEvent({ + event: "queue.changed", + id: "evt_nested", + module: "wait-queue", + payload: { depth: 1, nested }, + }); + worker.pushEvent({ + event: "queue.changed", + id: "evt_after", + module: "wait-queue", + payload: { depth: 2 }, + }); + + const ids = harness.events + .filter((event) => event.event === "queue.changed") + .map((event) => event.id); + expect(ids).toEqual(relayed ? ["evt_nested", "evt_after"] : ["evt_after"]); + expect(logger.errors).toHaveLength(relayed ? 0 : 1); + + await harness.service.stop(); + }, + ); + + it("republishes a worker's queue.changed with its workerId and emits no queue.changed of its own", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + + worker.pushEvent({ + event: "queue.changed", + id: "evt_worker-queue", + module: "wait-queue", + payload: { depth: 3 }, + seq: 5, + timestamp: 900, + }); + + expect(harness.events.filter((event) => event.event === "queue.changed")).toMatchObject([ + { module: "wait-queue", payload: { depth: 3, workerId: "wrk_1" } }, + ]); + + await harness.service.stop(); + }); + + // why: nothing about the admin protocol proves that a pushed event's *name* is genuinely the + // worker's own -- `workerId` is merged in last so it cannot be spoofed, but a worker (or + // anything speaking its protocol) could otherwise forge a fact only a gateway authors, such as + // `worker.drain-ended`, straight into the operator's audit trail. + it.each([ + { + event: "worker.drain-ended", + module: "gateway", + payload: { workerId: "wrk_forged", label: "not-this-worker" }, + }, + { + event: "request.granted", + module: "fleet-lease-coordinator", + payload: { leaseId: "wrk_1.lease_1", requestId: "req_forged", worker: "wrk_1" }, + }, + ])( + "refuses to republish a $event a worker pushes under a name reserved for the gateway itself (ADR 0021 §4)", + async (forgedEvent) => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + + worker.pushEvent(forgedEvent); + worker.pushEvent({ + event: "lease.granted", + module: "lease-engine", + payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, + }); + await vi.waitFor(() => + expect(harness.events.some((event) => event.event === "lease.granted")).toBe(true), + ); + + // why: never forwarded, and never with the forged `workerId` a real gateway-authored event + // for a *different* worker would otherwise be indistinguishable from. + expect(harness.events.filter((event) => event.event === forgedEvent.event)).toEqual([]); + + await harness.service.stop(); + }, + ); + + it("refuses to republish a request.dispatched a worker pushes, which only the gateway itself emits", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + + worker.pushEvent({ + event: "request.dispatched", + module: "fleet-lease-coordinator", + payload: { + platform: "ios", + queuedMs: 0, + reason: "warm-hit", + requestId: "req_forged", + requesterId: "agent-1", + stage: "warm-hit", + workerId: "wrk_forged", + }, + }); + worker.pushEvent({ + event: "lease.granted", + module: "lease-engine", + payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, + }); + await vi.waitFor(() => + expect(harness.events.some((event) => event.event === "lease.granted")).toBe(true), + ); + + expect(harness.events.filter((event) => event.event === "request.dispatched")).toEqual([]); + + await harness.service.stop(); + }); + + it("refreshes the view on a worker event that changes capacity or leases", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + const callsBefore = worker.calls.length; + worker.status = statusFixture({ leases: [leaseFixture("lease_9", "dev_9")] }); + + worker.pushEvent({ event: "lease.granted" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.leases).toEqual([ + expect.objectContaining({ id: "lease_9" }), + ]), + ); + expect(worker.calls.length).toBeGreaterThan(callsBefore); + + await harness.service.stop(); + }); + + it("refreshes a worker's view at once on its memory.pressure-changed", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + const capacity = statusFixture().capacity; + worker.status = statusFixture({ + capacity: { ...capacity, memoryPressure: { step: "new-devices-paused" } }, + }); + + worker.pushEvent({ event: "memory.pressure-changed" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.capacity?.memoryPressure?.step).toBe( + "new-devices-paused", + ), + ); + + await harness.service.stop(); + }); + + it("asks a worker that refused a request again once its memory.pressure-changed lifts the step, and not before", async () => { + const harness = fleet(); + await harness.service.start(); + const coordinator = new FleetLeaseCoordinator({ + clock: harness.clock, + endedLeaseIds: new MemoryEndedLeaseIds({ clock: harness.clock, retentionMs: 86_400_000 }), + describeFailure: (error) => ({ code: "INTERNAL", message: String(error) }), + directory: harness.service, + eventBus: harness.eventBus, + execTimeoutMs: 11 * 60_000, + idGenerator: { + generate: (() => { + let next = 1; + return () => `${next++}`; + })(), + }, + leaseIndex: new FleetLeaseIndex("gw:instance-1:"), + leaseRequestLimits: { maxRecords: 10_000, retentionMs: 600_000 }, + leaseRequestTimeoutMs: 5 * 60_000, + routing: createRoutingPolicy("warm-then-free"), + views: harness.service.workers, + }); + const worker = new ScriptedWorkerClient(); + const capacity = statusFixture().capacity; + worker.status = statusFixture({ + capacity: { ...capacity, memoryPressure: { step: "new-devices-paused" } }, + }); + worker.catalog = catalogFixture([ + { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, + ]); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.capacity?.memoryPressure?.step).toBe( + "new-devices-paused", + ), + ); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.catalog).toHaveLength(1)); + const leaseRequests = () => worker.calls.filter((call) => call.startsWith("lease.request")); + + void coordinator + .request( + { model: "iPhone 17", platform: "ios" }, + { allowDownload: false, noWait: false, ownerId: "agent-1", requesterId: "agent-1" }, + ) + .catch(() => undefined); + await vi.waitFor(() => expect(leaseRequests()).toHaveLength(1)); + + worker.pushEvent({ event: "memory.pressure-changed" }); + await vi.waitFor(() => + expect(worker.calls.filter((c) => c === "status.get").length).toBeGreaterThan(1), + ); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(leaseRequests()).toHaveLength(1); + + worker.status = statusFixture({ capacity: { ...capacity, memoryPressure: { step: "none" } } }); + worker.pushEvent({ event: "memory.pressure-changed" }); + + await vi.waitFor(() => expect(leaseRequests()).toHaveLength(2)); + + await harness.service.stop(); + }); + + it("does not re-read the fleet for an event that changes neither", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + const callsBefore = worker.calls.length; + + worker.pushEvent({ event: "doctor.reconciled" }); + await vi.waitFor(() => expect(eventNames(harness.events)).toContain("doctor.reconciled")); + + expect(worker.calls).toHaveLength(callsBefore); + + await harness.service.stop(); + }); + + it("refreshes every view on the periodic tick, catalog included", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); + worker.catalog = catalogFixture([ + { models: ["iPhone 17", "iPad Pro"], platform: "ios", runtimes: ["26.0"] }, + ]); + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.models).toEqual([ + "iPhone 17", + "iPad Pro", + ]), + ); + + await harness.service.stop(); + }); + + it("sends a waiting physical request its worker refused to that worker again on the first periodic refresh after the worker's view reports the device present, with no worker event", async () => { + const harness = fleet(); + await harness.service.start(); + const coordinator = new FleetLeaseCoordinator({ + clock: harness.clock, + endedLeaseIds: new MemoryEndedLeaseIds({ clock: harness.clock, retentionMs: 86_400_000 }), + describeFailure: (error) => ({ code: "INTERNAL", message: String(error) }), + directory: harness.service, + eventBus: harness.eventBus, + execTimeoutMs: 11 * 60_000, + idGenerator: { + generate: (() => { + let next = 1; + return () => `${next++}`; + })(), + }, + leaseIndex: new FleetLeaseIndex("gw:instance-1:"), + leaseRequestLimits: { maxRecords: 10_000, retentionMs: 600_000 }, + leaseRequestTimeoutMs: 5 * 60_000, + routing: createRoutingPolicy("warm-then-free"), + views: harness.service.workers, + }); + const worker = new ScriptedWorkerClient(); + const phone = (presence: PhysicalPresenceFixture) => + physicalDeviceFixture("dev_p", "ready", { presence }); + worker.devices = [phone({ present: true })]; + worker.catalog = catalogFixture([ + { models: ["iPhone 15 Pro"], platform: "ios", runtimes: ["18.4"] }, + ]); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.devices).toHaveLength(1)); + const leaseRequests = () => worker.calls.filter((call) => call.startsWith("lease.request")); + + void coordinator + .request( + { physical: true, platform: "ios" }, + { allowDownload: false, noWait: false, ownerId: "agent-1", requesterId: "agent-1" }, + ) + .catch(() => undefined); + await vi.waitFor(() => expect(leaseRequests()).toHaveLength(1)); + const callsBefore = worker.calls.length; + + worker.devices = [phone({ present: false, reason: "not-present" })]; + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(worker.calls.length).toBeGreaterThan(callsBefore)); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.devices?.[0]).toMatchObject({ + presence: { present: false, reason: "not-present" }, + }), + ); + expect(leaseRequests()).toHaveLength(1); + + worker.devices = [phone({ present: true })]; + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => expect(leaseRequests()).toHaveLength(2)); + + await harness.service.stop(); + }); + + it("shows a pairing that changes on a worker on its view after the next refresh tick", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.catalog = catalogFixture([ + { + modelRuntimes: { "iPhone 17": ["26.0"] }, + models: ["iPhone 17"], + platform: "ios", + runtimes: ["25.4", "26.0"], + }, + ]); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.modelRuntimes).toEqual({ + "iPhone 17": ["26.0"], + }), + ); + worker.catalog = catalogFixture([ + { + modelRuntimes: { "iPhone 17": ["25.4", "26.0"] }, + models: ["iPhone 17"], + platform: "ios", + runtimes: ["25.4", "26.0"], + }, + ]); + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.modelRuntimes).toEqual({ + "iPhone 17": ["25.4", "26.0"], + }), + ); + + await harness.service.stop(); + }); + + it("carries the host facts its worker last reported on the view", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.status = statusFixture({ host: hostFixture({ arch: "x64", osVersion: "14.7" }) }); + + await harness.join("wrk_1", worker); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.host).toEqual( + hostFixture({ arch: "x64", osVersion: "14.7" }), + ), + ); + + await harness.service.stop(); + }); + + it("shows a changed tool version on the view after the next refresh", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.host?.tools[0]?.version).toBe("16.4"), + ); + worker.status = statusFixture({ + host: hostFixture({ + tools: [{ build: "17A324", name: "xcode", platform: "ios", version: "26.0" }], + }), + }); + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.host?.tools[0]?.version).toBe("26.0"), + ); + + await harness.service.stop(); + }); +}); diff --git a/src/gateway/service-worker-protocol.test.ts b/src/gateway/service-worker-protocol.test.ts new file mode 100644 index 000000000..8baba0a41 --- /dev/null +++ b/src/gateway/service-worker-protocol.test.ts @@ -0,0 +1,524 @@ +import { describe, expect, it, vi } from "vitest"; +import { negotiateProtocolVersion, PROTOCOL_VERSION_RANGE } from "../contract/index.js"; +import { + catalogFixture, + protocolMismatchError, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { eventNames, fleet, REFRESH_MS, RecordingLogger } from "./test-service.js"; + +const previousProtocol = { + min: PROTOCOL_VERSION_RANGE.min - 1, + max: PROTOCOL_VERSION_RANGE.max - 1, +}; + +describe("GatewayService", () => { + describe("waiting requests on the view", () => { + it("a worker's lease.queued event refreshes its view's waiting list on the gateway", async () => { + const worker = new ScriptedWorkerClient(); + const harness = fleet(); + await harness.service.start(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.downloads).toBeDefined(), + ); + expect(harness.service.workers.view("wrk_1")?.waiting).toEqual([]); + + const queued = { + createdAt: 500, + id: "req_1", + queuePosition: 1, + requesterId: "local-agent", + spec: { model: "iPhone 17", platform: "ios" as const }, + stage: "queued" as const, + }; + worker.status = statusFixture({ queueDepth: 1, waiting: [queued] }); + worker.pushEvent({ event: "lease.queued" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.waiting).toEqual([queued]), + ); + await harness.service.stop(); + }); + }); + + describe("installs in progress on the view (ADR 0010 §7)", () => { + const install = { + component: "26.4", + platform: "ios" as const, + since: 500, + state: "downloading" as const, + waiters: 1, + }; + + async function joined(worker: ScriptedWorkerClient) { + const harness = fleet(); + await harness.service.start(); + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(worker.subscribed).toBe(true)); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.downloads).toBeDefined(), + ); + return harness; + } + + it("shows a worker's install after component.install-started and drops it after component.installed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + const harness = await joined(worker); + expect(harness.service.workers.view("wrk_1")?.installs).toEqual([]); + + worker.status = statusFixture({ installs: [install] }); + worker.pushEvent({ event: "component.install-started" }); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.installs).toEqual([install]), + ); + + worker.status = statusFixture({ installs: [] }); + worker.pushEvent({ event: "component.installed" }); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.installs).toEqual([])); + + await harness.service.stop(); + }); + + it("shows a worker's update after daemon.update-started and its failure after daemon.update-failed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + const harness = await joined(worker); + const base = statusFixture(); + const update = { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm failed", + stage: "installing", + to: "0.4.0", + } as const; + + worker.status = { ...base, daemon: { ...base.daemon, update } }; + worker.pushEvent({ event: "daemon.update-started" }); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toEqual(update)); + + worker.status = { ...base, daemon: { ...base.daemon, lastUpdateFailure: failure } }; + worker.pushEvent({ event: "daemon.update-failed" }); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + update: null, + }), + ); + + await harness.service.stop(); + }); + + it("drops a worker's install after component.install-failed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + worker.status = statusFixture({ installs: [install] }); + const harness = await joined(worker); + expect(harness.service.workers.view("wrk_1")?.installs).toEqual([install]); + + worker.status = statusFixture({ installs: [] }); + worker.pushEvent({ event: "component.install-failed" }); + + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.installs).toEqual([])); + await harness.service.stop(); + }); + + it("lists the new runtime in the view's catalog after component.installed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + worker.catalog = catalogFixture([ + { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, + ]); + const harness = await joined(worker); + + worker.catalog = catalogFixture([ + { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0", "26.4"] }, + ]); + worker.pushEvent({ event: "component.installed" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.runtimes).toEqual([ + "26.0", + "26.4", + ]), + ); + await harness.service.stop(); + }); + + it.each(["baseline.saved", "baseline.removed", "baseline.stale"])( + "lists the worker's baselines in its view, and reads them again after %s, without waiting for the periodic refresh", + async (event) => { + const entry = { + mode: "full" as const, + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android" as const, + savedAt: "2026-11-02T13:10:00Z", + version: 1, + }; + const worker = new ScriptedWorkerClient(); + worker.baselines = [entry]; + const harness = await joined(worker); + expect(harness.service.workers.view("wrk_1")?.baselines).toEqual([entry]); + + worker.baselines = [{ ...entry, version: 2 }]; + worker.pushEvent({ event }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.baselines?.[0]?.version).toBe(2), + ); + await harness.service.stop(); + }, + ); + + it("re-reads the catalog for a component.installed that queues behind a refresh in flight, and keeps it when another event queues after it", async () => { + const worker = new ScriptedWorkerClient(); + worker.catalog = catalogFixture([ + { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, + ]); + const harness = await joined(worker); + + worker.catalog = catalogFixture([ + { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0", "26.4"] }, + ]); + worker.pushEvent({ event: "lease.granted" }); + worker.pushEvent({ event: "component.installed" }); + worker.pushEvent({ event: "lease.released" }); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.runtimes).toEqual([ + "26.0", + "26.4", + ]), + ); + await harness.service.stop(); + }); + + it("yields a view with no installs, still connected, for a worker whose status has no installs field", async () => { + const worker = new ScriptedWorkerClient(); + const { installs: _absent, ...withoutInstalls } = statusFixture(); + worker.status = { ...withoutInstalls, queueDepth: 3 }; + + const harness = await joined(worker); + + const view = harness.service.workers.view("wrk_1"); + expect(view?.queueDepth).toBe(3); + expect(view?.installs).toEqual([]); + expect(view?.connection).toBe("connected"); + await harness.service.stop(); + }); + + it("truncates a worker's installs to 16 entries and an over-long component to 64 characters, rather than storing them as sent", async () => { + const worker = new ScriptedWorkerClient(); + worker.status = statusFixture({ + installs: Array.from({ length: 20 }, (_, index) => ({ + ...install, + component: `${String(index).padStart(2, "0")}${"x".repeat(100)}`, + since: index, + })), + }); + + const harness = await joined(worker); + + const installs = harness.service.workers.view("wrk_1")?.installs ?? []; + expect(installs).toHaveLength(16); + expect(installs.map((entry) => entry.component)).toEqual( + Array.from({ length: 16 }, (_, index) => + `${String(index).padStart(2, "0")}${"x".repeat(100)}`.slice(0, 64), + ), + ); + expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); + await harness.service.stop(); + }); + }); + + describe("warning on a worker's lower lease.maxTtlMs (ADR 0005 §15)", () => { + it("warns once the joining worker's own config.get reports a lower cap", async () => { + const logger = new RecordingLogger(); + const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.leaseMaxTtlMs = 1_800_000; + + await harness.join("wrk_1", worker, "mac-mini-1"); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); + + const warning = logger.warnings.find((entry) => entry.fields?.workerId === "wrk_1"); + expect(warning?.fields).toMatchObject({ + gatewayMaxTtlMs: 3_600_000, + label: "mac-mini-1", + workerMaxTtlMs: 1_800_000, + }); + + await harness.service.stop(); + }); + + it("does not warn when the joining worker's own cap is at or above the gateway's", async () => { + const logger = new RecordingLogger(); + const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.leaseMaxTtlMs = 3_600_000; + + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); + + expect(logger.warnings.some((entry) => entry.fields?.workerId === "wrk_1")).toBe(false); + + await harness.service.stop(); + }); + + it("does not repeat the warning on the periodic tick's re-read of an unchanged cap", async () => { + const logger = new RecordingLogger(); + const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.leaseMaxTtlMs = 1_800_000; + + await harness.join("wrk_1", worker); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); + const warningsAfterJoin = logger.warnings.filter( + (entry) => entry.fields?.workerId === "wrk_1", + ).length; + expect(warningsAfterJoin).toBe(1); + + const callsBefore = worker.calls.filter((call) => call === "config.get").length; + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => + expect(worker.calls.filter((call) => call === "config.get").length).toBeGreaterThan( + callsBefore, + ), + ); + + expect(logger.warnings.filter((entry) => entry.fields?.workerId === "wrk_1").length).toBe( + warningsAfterJoin, + ); + + await harness.service.stop(); + }); + }); + + it("puts the warm pool block a worker's status read carries on its view", async () => { + const warmPool = { enabled: true, reserveRunning: { android: 0, ios: 2 }, targets: [] }; + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.status = statusFixture({ warmPool }); + + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.warmPool).toEqual(warmPool), + ); + + await harness.service.stop(); + }); + + it("marks a worker that speaks only the previous protocol version incompatible with both ranges, and asks it nothing else", async () => { + const previous = { min: 5, max: 5 }; + expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, previous)).toBeUndefined(); + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError(previous); + + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: previous }, + }); + expect(worker.calls).toEqual(["status.get"]); + expect(worker.subscribed).toBe(false); + + await harness.service.stop(); + }); + + it("reads an incompatible worker's update with daemon.update query on the tick and keeps the last view when the query fails", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + const update = { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const lastUpdateFailure = { + at: 500, + from: "0.1.0", + message: "npm failed", + stage: "installing", + to: "0.2.0", + } as const; + worker.updateAnswer = { + ...worker.updateAnswer, + lastUpdateFailure, + latestVersion: "0.4.0", + update, + }; + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure, + latestVersion: "0.4.0", + update, + }), + ); + expect(worker.updateInputs).toEqual([{ query: true }]); + + worker.updateFailWith = new Error("worker went quiet"); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(worker.updateInputs).toHaveLength(2)); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + + await harness.service.stop(); + }); + + it("does not let a replaced incompatible link's late update query overwrite its successor's view", async () => { + const harness = fleet(); + await harness.service.start(); + const stale = new ScriptedWorkerClient(); + stale.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", stale); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + let answerQuery: (() => void) | undefined; + stale.updateDaemon = () => + new Promise((resolve) => { + answerQuery = () => + resolve({ + ...stale.updateAnswer, + update: { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + }); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(answerQuery).toBeDefined()); + + await harness.join("wrk_1", new ScriptedWorkerClient("admin", "9.9.9")); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + answerQuery?.(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + connection: "connected", + update: null, + version: "9.9.9", + }); + await harness.service.stop(); + }); + + it("shows a worker's update, last update failure and latest version from its status, and a later status without an update clears it", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + const update = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.2.0", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + const base = statusFixture(); + worker.status = { + ...base, + daemon: { ...base.daemon, lastUpdateFailure: failure, latestVersion: "0.4.0", update }, + }; + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + update, + }), + ); + + worker.status = { ...worker.status, daemon: { ...worker.status.daemon, update: null } }; + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toBeNull()); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + }); + + await harness.service.stop(); + }); + + it("a worker advertising protocol 10 is incompatible on the gateway and nothing it pushes reaches the gateway's bus", async () => { + const worker10 = { min: 10, max: 10 }; + expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, worker10)).toBeUndefined(); + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError(worker10); + worker.subscribeDespiteFailure = true; + + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + worker.pushEvent({ event: "lease.granted", payload: { leaseId: "l1" } }); + + expect(worker.subscribed).toBe(false); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: worker10 }, + }); + expect(eventNames(harness.events)).not.toContain("lease.granted"); + + await harness.service.stop(); + }); + + it("marks a worker incompatible when hello finds no overlapping range, and asks it nothing else", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError(previousProtocol); + + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: previousProtocol }, + }); + expect(worker.calls).toEqual(["status.get"]); + expect(worker.subscribed).toBe(false); + expect(eventNames(harness.events)).not.toContain("worker.rejected"); + expect(eventNames(harness.events)).not.toContain("worker.connected"); + + await harness.service.stop(); + }); +}); diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index b8af0e64a..42857805b 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1,146 +1,15 @@ import { describe, expect, it, vi } from "vitest"; - -import { EventBus, type EventEnvelope } from "../bus/index.js"; -import { negotiateProtocolVersion, PROTOCOL_VERSION_RANGE } from "../contract/index.js"; -import { - FakeClock, - MemoryUplinkTransport, - type Logger, - type UplinkAuthOutcome, - type UplinkAuthResult, -} from "../ports/index.js"; -import { promiseState } from "../test-support/promise-state.js"; -import { MemoryDrainStore } from "./drain-store.js"; -import { MemoryEndedLeaseIds } from "./index.js"; -import { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import { FleetLeaseIndex } from "./lease-index.js"; -import { createRoutingPolicy } from "./routing.js"; -import { GatewayService, REJECTION_COALESCE_WINDOW_MS } from "./service.js"; import { catalogFixture, deviceFixture, hostFixture, leaseFixture, - physicalDeviceFixture, - type PhysicalPresenceFixture, protocolMismatchError, ScriptedWorkerClient, statusFixture, } from "./test-support.js"; -import { MAX_CONSECUTIVE_REFRESH_TIMEOUTS, WORKER_CALL_TIMEOUT_MS } from "./worker-link.js"; - -// why: the 8 MiB the gateway accepts for one unfinished worker reply line, written out so the test pins the number. -const REPLY_LINE_CAP = 8 * 1024 * 1024; -// why: the documented bound (src/gateway/AGENTS.md); a payload nested this deep is refused. -const MAX_RELAYED_PAYLOAD_DEPTH = 64; -const RETENTION_MS = 24 * 60 * 60_000; -const previousProtocol = { - min: PROTOCOL_VERSION_RANGE.min - 1, - max: PROTOCOL_VERSION_RANGE.max - 1, -}; -const REFRESH_MS = 30_000; -const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; - -class RecordingLogger implements Logger { - readonly warnings: Array<{ message: string; fields?: Record }> = []; - - debug(): void {} - info(): void {} - warn(message: string, fields?: Record): void { - this.warnings.push(fields === undefined ? { message } : { fields, message }); - } - readonly errors: string[] = []; - error(message: string): void { - this.errors.push(message); - } - child(): Logger { - return this; - } -} - -function fleet( - options: { - readonly authenticate?: ( - credential: string | undefined, - workerId: string | undefined, - ) => UplinkAuthOutcome | UplinkAuthResult; - readonly leaseMaxTtlMs?: number; - readonly logger?: Logger; - readonly refreshIntervalMs?: number; - /** why: leaves `connect` unset so the gateway builds its real client over the uplink. */ - readonly realConnect?: boolean; - } = {}, -) { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const events: EventEnvelope[] = []; - eventBus.subscribeAll((envelope) => events.push(envelope)); - const transport = new MemoryUplinkTransport(); - const clients: ScriptedWorkerClient[] = []; - const connectState = { hangNext: false }; - const service = new GatewayService({ - authenticate: async (credential, workerId) => - options.authenticate?.(credential, workerId) ?? "accept", - clock, - ...(options.realConnect === true - ? {} - : { - connect: async () => { - if (connectState.hangNext) { - connectState.hangNext = false; - return new Promise(() => {}); - } - const client = clients.at(-1); - if (client === undefined) - throw new Error("no scripted worker was queued for this uplink"); - return client.asClient(); - }, - }), - drainStore: new MemoryDrainStore(), - eventBus, - leaseMaxTtlMs: options.leaseMaxTtlMs ?? DEFAULT_LEASE_MAX_TTL_MS, - ...(options.logger === undefined ? {} : { logger: options.logger }), - principal: "gw:instance-1", - refreshIntervalMs: options.refreshIntervalMs ?? REFRESH_MS, - retentionMs: RETENTION_MS, - uplinks: transport, - }); - return { - clients, - clock, - eventBus, - events, - service, - transport, - join: async ( - workerId: string, - client: ScriptedWorkerClient, - label?: string, - token = "join-secret", - ) => { - clients.push(client); - return transport.connect({ - token, - url: "ws://gateway.test", - workerId, - ...(label === undefined ? {} : { label }), - }); - }, - joinSilent: async (workerId: string, label?: string) => { - connectState.hangNext = true; - return transport.connect({ - token: "join-secret", - url: "ws://gateway.test", - workerId, - ...(label === undefined ? {} : { label }), - }); - }, - }; -} - -function eventNames(events: readonly EventEnvelope[]): string[] { - return events.map((event) => event.event); -} +import { WORKER_CALL_TIMEOUT_MS } from "./worker-link.js"; +import { eventNames, fleet, REFRESH_MS } from "./test-service.js"; describe("GatewayService", () => { it("builds a view from the four calls ADR 0005 §7 prescribes", async () => { @@ -593,1472 +462,6 @@ describe("GatewayService", () => { await harness.service.stop(); }); - - it("a relayed event differs from the worker's only in seq and payload.workerId", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - const sent = { - event: "lease.granted", - id: "evt_worker-made", - module: "lease-engine", - payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, - seq: 41, - timestamp: 777, - }; - - worker.pushEvent(sent); - - const relayed = harness.events.find((event) => event.event === "lease.granted"); - const { seq: relayedSeq, payload: relayedPayload, ...relayedRest } = relayed ?? {}; - const { seq: sentSeq, payload: sentPayload, ...sentRest } = sent; - expect(relayedRest).toEqual(sentRest); - expect(relayedPayload).toEqual({ ...sentPayload, workerId: "wrk_1" }); - expect(relayedSeq).not.toBe(sentSeq); - - await harness.service.stop(); - }); - - it.each([ - { levels: MAX_RELAYED_PAYLOAD_DEPTH - 1, relayed: true }, - { levels: MAX_RELAYED_PAYLOAD_DEPTH, relayed: false }, - { levels: 6_189, relayed: false }, - { levels: 20_000, relayed: false }, - ])( - "a relayed event whose payload nests $levels levels is relayed: $relayed, and a refused one logs one error while the next event arrives", - async ({ levels, relayed }) => { - const logger = new RecordingLogger(); - const harness = fleet({ logger }); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - const nested = JSON.parse("[".repeat(levels) + "]".repeat(levels)) as unknown; - - worker.pushEvent({ - event: "queue.changed", - id: "evt_nested", - module: "wait-queue", - payload: { depth: 1, nested }, - }); - worker.pushEvent({ - event: "queue.changed", - id: "evt_after", - module: "wait-queue", - payload: { depth: 2 }, - }); - - const ids = harness.events - .filter((event) => event.event === "queue.changed") - .map((event) => event.id); - expect(ids).toEqual(relayed ? ["evt_nested", "evt_after"] : ["evt_after"]); - expect(logger.errors).toHaveLength(relayed ? 0 : 1); - - await harness.service.stop(); - }, - ); - - it("republishes a worker's queue.changed with its workerId and emits no queue.changed of its own", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - - worker.pushEvent({ - event: "queue.changed", - id: "evt_worker-queue", - module: "wait-queue", - payload: { depth: 3 }, - seq: 5, - timestamp: 900, - }); - - expect(harness.events.filter((event) => event.event === "queue.changed")).toMatchObject([ - { module: "wait-queue", payload: { depth: 3, workerId: "wrk_1" } }, - ]); - - await harness.service.stop(); - }); - - // why: nothing about the admin protocol proves that a pushed event's *name* is genuinely the - // worker's own -- `workerId` is merged in last so it cannot be spoofed, but a worker (or - // anything speaking its protocol) could otherwise forge a fact only a gateway authors, such as - // `worker.drain-ended`, straight into the operator's audit trail. - it.each([ - { - event: "worker.drain-ended", - module: "gateway", - payload: { workerId: "wrk_forged", label: "not-this-worker" }, - }, - { - event: "request.granted", - module: "fleet-lease-coordinator", - payload: { leaseId: "wrk_1.lease_1", requestId: "req_forged", worker: "wrk_1" }, - }, - ])( - "refuses to republish a $event a worker pushes under a name reserved for the gateway itself (ADR 0021 §4)", - async (forgedEvent) => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - - worker.pushEvent(forgedEvent); - worker.pushEvent({ - event: "lease.granted", - module: "lease-engine", - payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, - }); - await vi.waitFor(() => - expect(harness.events.some((event) => event.event === "lease.granted")).toBe(true), - ); - - // why: never forwarded, and never with the forged `workerId` a real gateway-authored event - // for a *different* worker would otherwise be indistinguishable from. - expect(harness.events.filter((event) => event.event === forgedEvent.event)).toEqual([]); - - await harness.service.stop(); - }, - ); - - it("refuses to republish a request.dispatched a worker pushes, which only the gateway itself emits", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - - worker.pushEvent({ - event: "request.dispatched", - module: "fleet-lease-coordinator", - payload: { - platform: "ios", - queuedMs: 0, - reason: "warm-hit", - requestId: "req_forged", - requesterId: "agent-1", - stage: "warm-hit", - workerId: "wrk_forged", - }, - }); - worker.pushEvent({ - event: "lease.granted", - module: "lease-engine", - payload: { deviceId: "dev_1", leaseId: "lease_1", requester: "agent-1" }, - }); - await vi.waitFor(() => - expect(harness.events.some((event) => event.event === "lease.granted")).toBe(true), - ); - - expect(harness.events.filter((event) => event.event === "request.dispatched")).toEqual([]); - - await harness.service.stop(); - }); - - it("refreshes the view on a worker event that changes capacity or leases", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - const callsBefore = worker.calls.length; - worker.status = statusFixture({ leases: [leaseFixture("lease_9", "dev_9")] }); - - worker.pushEvent({ event: "lease.granted" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.leases).toEqual([ - expect.objectContaining({ id: "lease_9" }), - ]), - ); - expect(worker.calls.length).toBeGreaterThan(callsBefore); - - await harness.service.stop(); - }); - - it("refreshes a worker's view at once on its memory.pressure-changed", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - const capacity = statusFixture().capacity; - worker.status = statusFixture({ - capacity: { ...capacity, memoryPressure: { step: "new-devices-paused" } }, - }); - - worker.pushEvent({ event: "memory.pressure-changed" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.capacity?.memoryPressure?.step).toBe( - "new-devices-paused", - ), - ); - - await harness.service.stop(); - }); - - it("asks a worker that refused a request again once its memory.pressure-changed lifts the step, and not before", async () => { - const harness = fleet(); - await harness.service.start(); - const coordinator = new FleetLeaseCoordinator({ - clock: harness.clock, - endedLeaseIds: new MemoryEndedLeaseIds({ clock: harness.clock, retentionMs: 86_400_000 }), - describeFailure: (error) => ({ code: "INTERNAL", message: String(error) }), - directory: harness.service, - eventBus: harness.eventBus, - execTimeoutMs: 11 * 60_000, - idGenerator: { - generate: (() => { - let next = 1; - return () => `${next++}`; - })(), - }, - leaseIndex: new FleetLeaseIndex("gw:instance-1:"), - leaseRequestLimits: { maxRecords: 10_000, retentionMs: 600_000 }, - leaseRequestTimeoutMs: 5 * 60_000, - routing: createRoutingPolicy("warm-then-free"), - views: harness.service.workers, - }); - const worker = new ScriptedWorkerClient(); - const capacity = statusFixture().capacity; - worker.status = statusFixture({ - capacity: { ...capacity, memoryPressure: { step: "new-devices-paused" } }, - }); - worker.catalog = catalogFixture([ - { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, - ]); - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.capacity?.memoryPressure?.step).toBe( - "new-devices-paused", - ), - ); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.catalog).toHaveLength(1)); - const leaseRequests = () => worker.calls.filter((call) => call.startsWith("lease.request")); - - void coordinator - .request( - { model: "iPhone 17", platform: "ios" }, - { allowDownload: false, noWait: false, ownerId: "agent-1", requesterId: "agent-1" }, - ) - .catch(() => undefined); - await vi.waitFor(() => expect(leaseRequests()).toHaveLength(1)); - - worker.pushEvent({ event: "memory.pressure-changed" }); - await vi.waitFor(() => - expect(worker.calls.filter((c) => c === "status.get").length).toBeGreaterThan(1), - ); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(leaseRequests()).toHaveLength(1); - - worker.status = statusFixture({ capacity: { ...capacity, memoryPressure: { step: "none" } } }); - worker.pushEvent({ event: "memory.pressure-changed" }); - - await vi.waitFor(() => expect(leaseRequests()).toHaveLength(2)); - - await harness.service.stop(); - }); - - it("does not re-read the fleet for an event that changes neither", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - const callsBefore = worker.calls.length; - - worker.pushEvent({ event: "doctor.reconciled" }); - await vi.waitFor(() => expect(eventNames(harness.events)).toContain("doctor.reconciled")); - - expect(worker.calls).toHaveLength(callsBefore); - - await harness.service.stop(); - }); - - it("refreshes every view on the periodic tick, catalog included", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - worker.catalog = catalogFixture([ - { models: ["iPhone 17", "iPad Pro"], platform: "ios", runtimes: ["26.0"] }, - ]); - - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.models).toEqual([ - "iPhone 17", - "iPad Pro", - ]), - ); - - await harness.service.stop(); - }); - - it("sends a waiting physical request its worker refused to that worker again on the first periodic refresh after the worker's view reports the device present, with no worker event", async () => { - const harness = fleet(); - await harness.service.start(); - const coordinator = new FleetLeaseCoordinator({ - clock: harness.clock, - endedLeaseIds: new MemoryEndedLeaseIds({ clock: harness.clock, retentionMs: 86_400_000 }), - describeFailure: (error) => ({ code: "INTERNAL", message: String(error) }), - directory: harness.service, - eventBus: harness.eventBus, - execTimeoutMs: 11 * 60_000, - idGenerator: { - generate: (() => { - let next = 1; - return () => `${next++}`; - })(), - }, - leaseIndex: new FleetLeaseIndex("gw:instance-1:"), - leaseRequestLimits: { maxRecords: 10_000, retentionMs: 600_000 }, - leaseRequestTimeoutMs: 5 * 60_000, - routing: createRoutingPolicy("warm-then-free"), - views: harness.service.workers, - }); - const worker = new ScriptedWorkerClient(); - const phone = (presence: PhysicalPresenceFixture) => - physicalDeviceFixture("dev_p", "ready", { presence }); - worker.devices = [phone({ present: true })]; - worker.catalog = catalogFixture([ - { models: ["iPhone 15 Pro"], platform: "ios", runtimes: ["18.4"] }, - ]); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.devices).toHaveLength(1)); - const leaseRequests = () => worker.calls.filter((call) => call.startsWith("lease.request")); - - void coordinator - .request( - { physical: true, platform: "ios" }, - { allowDownload: false, noWait: false, ownerId: "agent-1", requesterId: "agent-1" }, - ) - .catch(() => undefined); - await vi.waitFor(() => expect(leaseRequests()).toHaveLength(1)); - const callsBefore = worker.calls.length; - - worker.devices = [phone({ present: false, reason: "not-present" })]; - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => expect(worker.calls.length).toBeGreaterThan(callsBefore)); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.devices?.[0]).toMatchObject({ - presence: { present: false, reason: "not-present" }, - }), - ); - expect(leaseRequests()).toHaveLength(1); - - worker.devices = [phone({ present: true })]; - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => expect(leaseRequests()).toHaveLength(2)); - - await harness.service.stop(); - }); - - it("shows a pairing that changes on a worker on its view after the next refresh tick", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.catalog = catalogFixture([ - { - modelRuntimes: { "iPhone 17": ["26.0"] }, - models: ["iPhone 17"], - platform: "ios", - runtimes: ["25.4", "26.0"], - }, - ]); - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.modelRuntimes).toEqual({ - "iPhone 17": ["26.0"], - }), - ); - worker.catalog = catalogFixture([ - { - modelRuntimes: { "iPhone 17": ["25.4", "26.0"] }, - models: ["iPhone 17"], - platform: "ios", - runtimes: ["25.4", "26.0"], - }, - ]); - - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.modelRuntimes).toEqual({ - "iPhone 17": ["25.4", "26.0"], - }), - ); - - await harness.service.stop(); - }); - - it("carries the host facts its worker last reported on the view", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.status = statusFixture({ host: hostFixture({ arch: "x64", osVersion: "14.7" }) }); - - await harness.join("wrk_1", worker); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.host).toEqual( - hostFixture({ arch: "x64", osVersion: "14.7" }), - ), - ); - - await harness.service.stop(); - }); - - it("shows a changed tool version on the view after the next refresh", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.host?.tools[0]?.version).toBe("16.4"), - ); - worker.status = statusFixture({ - host: hostFixture({ - tools: [{ build: "17A324", name: "xcode", platform: "ios", version: "26.0" }], - }), - }); - - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.host?.tools[0]?.version).toBe("26.0"), - ); - - await harness.service.stop(); - }); - - describe("waiting requests on the view", () => { - it("a worker's lease.queued event refreshes its view's waiting list on the gateway", async () => { - const worker = new ScriptedWorkerClient(); - const harness = fleet(); - await harness.service.start(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.downloads).toBeDefined(), - ); - expect(harness.service.workers.view("wrk_1")?.waiting).toEqual([]); - - const queued = { - createdAt: 500, - id: "req_1", - queuePosition: 1, - requesterId: "local-agent", - spec: { model: "iPhone 17", platform: "ios" as const }, - stage: "queued" as const, - }; - worker.status = statusFixture({ queueDepth: 1, waiting: [queued] }); - worker.pushEvent({ event: "lease.queued" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.waiting).toEqual([queued]), - ); - await harness.service.stop(); - }); - }); - - describe("installs in progress on the view (ADR 0010 §7)", () => { - const install = { - component: "26.4", - platform: "ios" as const, - since: 500, - state: "downloading" as const, - waiters: 1, - }; - - async function joined(worker: ScriptedWorkerClient) { - const harness = fleet(); - await harness.service.start(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.downloads).toBeDefined(), - ); - return harness; - } - - it("shows a worker's install after component.install-started and drops it after component.installed, without waiting for the periodic refresh", async () => { - const worker = new ScriptedWorkerClient(); - const harness = await joined(worker); - expect(harness.service.workers.view("wrk_1")?.installs).toEqual([]); - - worker.status = statusFixture({ installs: [install] }); - worker.pushEvent({ event: "component.install-started" }); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.installs).toEqual([install]), - ); - - worker.status = statusFixture({ installs: [] }); - worker.pushEvent({ event: "component.installed" }); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.installs).toEqual([])); - - await harness.service.stop(); - }); - - it("shows a worker's update after daemon.update-started and its failure after daemon.update-failed, without waiting for the periodic refresh", async () => { - const worker = new ScriptedWorkerClient(); - const harness = await joined(worker); - const base = statusFixture(); - const update = { - forced: false, - from: "0.3.0", - stage: "installing", - startedAt: 1_000, - to: "0.4.0", - } as const; - const failure = { - at: 2_000, - from: "0.3.0", - message: "npm failed", - stage: "installing", - to: "0.4.0", - } as const; - - worker.status = { ...base, daemon: { ...base.daemon, update } }; - worker.pushEvent({ event: "daemon.update-started" }); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toEqual(update)); - - worker.status = { ...base, daemon: { ...base.daemon, lastUpdateFailure: failure } }; - worker.pushEvent({ event: "daemon.update-failed" }); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - lastUpdateFailure: failure, - update: null, - }), - ); - - await harness.service.stop(); - }); - - it("drops a worker's install after component.install-failed, without waiting for the periodic refresh", async () => { - const worker = new ScriptedWorkerClient(); - worker.status = statusFixture({ installs: [install] }); - const harness = await joined(worker); - expect(harness.service.workers.view("wrk_1")?.installs).toEqual([install]); - - worker.status = statusFixture({ installs: [] }); - worker.pushEvent({ event: "component.install-failed" }); - - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.installs).toEqual([])); - await harness.service.stop(); - }); - - it("lists the new runtime in the view's catalog after component.installed, without waiting for the periodic refresh", async () => { - const worker = new ScriptedWorkerClient(); - worker.catalog = catalogFixture([ - { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, - ]); - const harness = await joined(worker); - - worker.catalog = catalogFixture([ - { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0", "26.4"] }, - ]); - worker.pushEvent({ event: "component.installed" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.runtimes).toEqual([ - "26.0", - "26.4", - ]), - ); - await harness.service.stop(); - }); - - it.each(["baseline.saved", "baseline.removed", "baseline.stale"])( - "lists the worker's baselines in its view, and reads them again after %s, without waiting for the periodic refresh", - async (event) => { - const entry = { - mode: "full" as const, - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android" as const, - savedAt: "2026-11-02T13:10:00Z", - version: 1, - }; - const worker = new ScriptedWorkerClient(); - worker.baselines = [entry]; - const harness = await joined(worker); - expect(harness.service.workers.view("wrk_1")?.baselines).toEqual([entry]); - - worker.baselines = [{ ...entry, version: 2 }]; - worker.pushEvent({ event }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.baselines?.[0]?.version).toBe(2), - ); - await harness.service.stop(); - }, - ); - - it("re-reads the catalog for a component.installed that queues behind a refresh in flight, and keeps it when another event queues after it", async () => { - const worker = new ScriptedWorkerClient(); - worker.catalog = catalogFixture([ - { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0"] }, - ]); - const harness = await joined(worker); - - worker.catalog = catalogFixture([ - { models: ["iPhone 17"], platform: "ios", runtimes: ["26.0", "26.4"] }, - ]); - worker.pushEvent({ event: "lease.granted" }); - worker.pushEvent({ event: "component.installed" }); - worker.pushEvent({ event: "lease.released" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.catalog?.[0]?.runtimes).toEqual([ - "26.0", - "26.4", - ]), - ); - await harness.service.stop(); - }); - - it("yields a view with no installs, still connected, for a worker whose status has no installs field", async () => { - const worker = new ScriptedWorkerClient(); - const { installs: _absent, ...withoutInstalls } = statusFixture(); - worker.status = { ...withoutInstalls, queueDepth: 3 }; - - const harness = await joined(worker); - - const view = harness.service.workers.view("wrk_1"); - expect(view?.queueDepth).toBe(3); - expect(view?.installs).toEqual([]); - expect(view?.connection).toBe("connected"); - await harness.service.stop(); - }); - - it("truncates a worker's installs to 16 entries and an over-long component to 64 characters, rather than storing them as sent", async () => { - const worker = new ScriptedWorkerClient(); - worker.status = statusFixture({ - installs: Array.from({ length: 20 }, (_, index) => ({ - ...install, - component: `${String(index).padStart(2, "0")}${"x".repeat(100)}`, - since: index, - })), - }); - - const harness = await joined(worker); - - const installs = harness.service.workers.view("wrk_1")?.installs ?? []; - expect(installs).toHaveLength(16); - expect(installs.map((entry) => entry.component)).toEqual( - Array.from({ length: 16 }, (_, index) => - `${String(index).padStart(2, "0")}${"x".repeat(100)}`.slice(0, 64), - ), - ); - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); - await harness.service.stop(); - }); - }); - - describe("warning on a worker's lower lease.maxTtlMs (ADR 0005 §15)", () => { - it("warns once the joining worker's own config.get reports a lower cap", async () => { - const logger = new RecordingLogger(); - const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.leaseMaxTtlMs = 1_800_000; - - await harness.join("wrk_1", worker, "mac-mini-1"); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); - - const warning = logger.warnings.find((entry) => entry.fields?.workerId === "wrk_1"); - expect(warning?.fields).toMatchObject({ - gatewayMaxTtlMs: 3_600_000, - label: "mac-mini-1", - workerMaxTtlMs: 1_800_000, - }); - - await harness.service.stop(); - }); - - it("does not warn when the joining worker's own cap is at or above the gateway's", async () => { - const logger = new RecordingLogger(); - const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.leaseMaxTtlMs = 3_600_000; - - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); - - expect(logger.warnings.some((entry) => entry.fields?.workerId === "wrk_1")).toBe(false); - - await harness.service.stop(); - }); - - it("does not repeat the warning on the periodic tick's re-read of an unchanged cap", async () => { - const logger = new RecordingLogger(); - const harness = fleet({ leaseMaxTtlMs: 3_600_000, logger }); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.leaseMaxTtlMs = 1_800_000; - - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.lease).toBeDefined()); - const warningsAfterJoin = logger.warnings.filter( - (entry) => entry.fields?.workerId === "wrk_1", - ).length; - expect(warningsAfterJoin).toBe(1); - - const callsBefore = worker.calls.filter((call) => call === "config.get").length; - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => - expect(worker.calls.filter((call) => call === "config.get").length).toBeGreaterThan( - callsBefore, - ), - ); - - expect(logger.warnings.filter((entry) => entry.fields?.workerId === "wrk_1").length).toBe( - warningsAfterJoin, - ); - - await harness.service.stop(); - }); - }); - - it("puts the warm pool block a worker's status read carries on its view", async () => { - const warmPool = { enabled: true, reserveRunning: { android: 0, ios: 2 }, targets: [] }; - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.status = statusFixture({ warmPool }); - - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.warmPool).toEqual(warmPool), - ); - - await harness.service.stop(); - }); - - it("marks a worker that speaks only the previous protocol version incompatible with both ranges, and asks it nothing else", async () => { - const previous = { min: 5, max: 5 }; - expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, previous)).toBeUndefined(); - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.failWith = protocolMismatchError(previous); - - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), - ); - - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: previous }, - }); - expect(worker.calls).toEqual(["status.get"]); - expect(worker.subscribed).toBe(false); - - await harness.service.stop(); - }); - - it("reads an incompatible worker's update with daemon.update query on the tick and keeps the last view when the query fails", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.failWith = protocolMismatchError({ min: 5, max: 5 }); - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), - ); - const update = { - forced: false, - from: "0.2.0", - stage: "installing", - startedAt: 1_000, - to: "0.4.0", - } as const; - const lastUpdateFailure = { - at: 500, - from: "0.1.0", - message: "npm failed", - stage: "installing", - to: "0.2.0", - } as const; - worker.updateAnswer = { - ...worker.updateAnswer, - lastUpdateFailure, - latestVersion: "0.4.0", - update, - }; - - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - lastUpdateFailure, - latestVersion: "0.4.0", - update, - }), - ); - expect(worker.updateInputs).toEqual([{ query: true }]); - - worker.updateFailWith = new Error("worker went quiet"); - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => expect(worker.updateInputs).toHaveLength(2)); - expect(harness.service.workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); - - await harness.service.stop(); - }); - - it("does not let a replaced incompatible link's late update query overwrite its successor's view", async () => { - const harness = fleet(); - await harness.service.start(); - const stale = new ScriptedWorkerClient(); - stale.failWith = protocolMismatchError({ min: 5, max: 5 }); - await harness.join("wrk_1", stale); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), - ); - let answerQuery: (() => void) | undefined; - stale.updateDaemon = () => - new Promise((resolve) => { - answerQuery = () => - resolve({ - ...stale.updateAnswer, - update: { - forced: false, - from: "0.2.0", - stage: "installing", - startedAt: 1, - to: "0.4.0", - }, - }); - }); - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => expect(answerQuery).toBeDefined()); - - await harness.join("wrk_1", new ScriptedWorkerClient("admin", "9.9.9")); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); - answerQuery?.(); - await new Promise((resolve) => setTimeout(resolve, 20)); - - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - connection: "connected", - update: null, - version: "9.9.9", - }); - await harness.service.stop(); - }); - - it("shows a worker's update, last update failure and latest version from its status, and a later status without an update clears it", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - const update = { - forced: false, - from: "0.3.0", - leases: 2, - stage: "waiting", - startedAt: 1_000, - to: "0.4.0", - } as const; - const failure = { - at: 500, - from: "0.2.0", - message: "npm failed", - stage: "installing", - to: "0.3.0", - } as const; - const base = statusFixture(); - worker.status = { - ...base, - daemon: { ...base.daemon, lastUpdateFailure: failure, latestVersion: "0.4.0", update }, - }; - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - lastUpdateFailure: failure, - latestVersion: "0.4.0", - update, - }), - ); - - worker.status = { ...worker.status, daemon: { ...worker.status.daemon, update: null } }; - harness.clock.advance(REFRESH_MS); - - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toBeNull()); - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - lastUpdateFailure: failure, - latestVersion: "0.4.0", - }); - - await harness.service.stop(); - }); - - it("a worker advertising protocol 10 is incompatible on the gateway and nothing it pushes reaches the gateway's bus", async () => { - const worker10 = { min: 10, max: 10 }; - expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, worker10)).toBeUndefined(); - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.failWith = protocolMismatchError(worker10); - worker.subscribeDespiteFailure = true; - - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), - ); - worker.pushEvent({ event: "lease.granted", payload: { leaseId: "l1" } }); - - expect(worker.subscribed).toBe(false); - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: worker10 }, - }); - expect(eventNames(harness.events)).not.toContain("lease.granted"); - - await harness.service.stop(); - }); - - it("marks a worker incompatible when hello finds no overlapping range, and asks it nothing else", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - worker.failWith = protocolMismatchError(previousProtocol); - - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), - ); - - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - protocol: { gateway: PROTOCOL_VERSION_RANGE, worker: previousProtocol }, - }); - expect(worker.calls).toEqual(["status.get"]); - expect(worker.subscribed).toBe(false); - expect(eventNames(harness.events)).not.toContain("worker.rejected"); - expect(eventNames(harness.events)).not.toContain("worker.connected"); - - await harness.service.stop(); - }); - - it("flips a view to disconnected the moment its uplink closes -- no polling", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - const workerEnd = await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - await workerEnd.close(); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), - ); - expect(eventNames(harness.events)).toContain("worker.disconnected"); - - await harness.service.stop(); - }); - - it("does not let a stale link's late close mark a reconnected worker disconnected (D1)", async () => { - const harness = fleet(); - await harness.service.start(); - - const clientA = new ScriptedWorkerClient("admin", "0.1.0"); - const workerEndA = await harness.join("wrk_1", clientA); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"), - ); - - clientA.hangUnsubscribe = true; - - const clientB = new ScriptedWorkerClient("admin", "9.9.9"); - await harness.join("wrk_1", clientB); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); - - await workerEndA.close(); - - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); - expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9"); - expect(eventNames(harness.events)).not.toContain("worker.disconnected"); - - await harness.service.stop(); - }); - - it("does not hang close() forever behind an events.unsubscribe that never answers (D2)", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(worker.subscribed).toBe(true)); - worker.hangUnsubscribe = true; - - const stopped = harness.service.stop(); - const stoppedState = promiseState(stopped); - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - - await vi.waitFor(() => expect(stoppedState.state).toBe("fulfilled")); - await expect(stopped).resolves.toBeUndefined(); - expect(worker.closed).toBe(true); - }); - - it("does not latch refreshing forever when a round trip hangs, and recovers past the timeout (D3)", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - worker.hangingCalls.add("status.get"); - worker.status = statusFixture({ leases: [leaseFixture("lease_9", "dev_9")] }); - worker.pushEvent({ event: "lease.granted" }); - - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - - worker.hangingCalls.delete("status.get"); - worker.pushEvent({ event: "lease.granted" }); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.leases).toEqual([ - expect.objectContaining({ id: "lease_9" }), - ]), - ); - - await harness.service.stop(); - }); - - it("does not let a stale link's late-arriving refresh overwrite its successor's fresher view (H2)", async () => { - const harness = fleet(); - await harness.service.start(); - - const clientA = new ScriptedWorkerClient("admin", "0.1.0"); - await harness.join("wrk_1", clientA); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("0.1.0")); - - let resolveStatus: ((status: ReturnType) => void) | undefined; - clientA.getStatus = () => - new Promise((resolve) => { - resolveStatus = resolve; - }); - clientA.pushEvent({ event: "lease.granted" }); - await vi.waitFor(() => expect(resolveStatus).toBeDefined()); - - const clientB = new ScriptedWorkerClient("admin", "9.9.9"); - await harness.join("wrk_1", clientB); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); - - resolveStatus?.(statusFixture()); - // why: A few real event-loop turns for A's now-resolved promise chain to run to completion (or, - // with the fix, to stop itself before writing) -- nothing here is clock-scheduled, so there - // is no fake-clock timer to advance instead. - await new Promise((resolve) => setTimeout(resolve, 20)); - - expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9"); - - await harness.service.stop(); - }); - - it("does not let a stale link's late starting answer overwrite its successor's view", async () => { - const harness = fleet(); - await harness.service.start(); - const clientA = new ScriptedWorkerClient("admin", "0.1.0"); - await harness.join("wrk_1", clientA); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("0.1.0")); - const release = clientA.holdStatus(); - clientA.pushEvent({ event: "lease.granted" }); - await vi.waitFor(() => expect(clientA.calls.at(-1)).toBe("status.get")); - const clientB = new ScriptedWorkerClient("admin", "9.9.9"); - clientB.devices = [deviceFixture("dev_b", "ready")]; - await harness.join("wrk_1", clientB); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); - - clientA.status = { - daemon: { - health: "starting", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: hostFixture(), - }; - release(); - await new Promise((resolve) => setTimeout(resolve, 20)); - - expect(harness.service.workers.view("wrk_1")).toMatchObject({ - devices: [{ id: "dev_b" }], - health: "running", - version: "9.9.9", - }); - await harness.service.stop(); - }); - - it("closes the uplink once refreshes keep timing out, so a half-open worker stops reporting connected (P1)", async () => { - expect(MAX_CONSECUTIVE_REFRESH_TIMEOUTS).toBeGreaterThan(1); - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - worker.hangingCalls.add("status.get"); - - for (let attempt = 1; attempt < MAX_CONSECUTIVE_REFRESH_TIMEOUTS; attempt++) { - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - // why: Wait for the *next* tick to actually be armed, not just for the (already, trivially - // true) `connected` check to pass -- that value never changes on a timeout, so checking - // it alone would resolve before the tick's own promise chain (refresh's catch, then - // `pruneExpired`, then rescheduling) has actually finished, and the next iteration's - // `advance(REFRESH_MS)` would fast-forward the clock past a timer that has not been - // registered yet -- silently skipping a whole tick instead of exercising it. - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); - } - - harness.clock.advance(REFRESH_MS); - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), - ); - expect(eventNames(harness.events)).toContain("worker.disconnected"); - - await harness.service.stop(); - }); - - it("does not close the link when status.get keeps answering but the dispatcher is busy (C-1)", async () => { - // why: A refreshIntervalMs far outside this test's clock advances: `#runTick` only reschedules - // its own timer once a refresh settles (`service.ts`'s `.finally(() => this.#scheduleTick())`), - // so on a link whose refresh keeps timing out the tick's real cadence drifts to roughly - // `refreshIntervalMs + WORKER_CALL_TIMEOUT_MS` per cycle -- exercising this via event-driven - // refreshes instead (as D3 and H2 above do) keeps the periodic tick out of the picture - // entirely, rather than fighting its drift with hand-tuned clock advances. - const harness = fleet({ refreshIntervalMs: 10_000_000 }); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - worker.hangingCalls.add("list.get:devices"); - - for (let attempt = 1; attempt <= MAX_CONSECUTIVE_REFRESH_TIMEOUTS + 2; attempt++) { - const listCallsBefore = worker.calls.filter((call) => call === "list.get:devices").length; - worker.pushEvent({ event: "lease.granted" }); - // why: Waits for `list.get` to have actually been called -- proof `status.get`'s real promise - // was given a genuine chance to settle and `#rebuildView` moved on to the hung batch. - // Waiting on `pendingTimerCount` instead would be a false positive: `status.get`'s own - // bounded wait registers a timer *synchronously*, before its real response has had any - // chance to arrive, so a bare "a timer exists" check can pass and race the fake clock's - // `advance` below against `status.get`'s still-pending real resolution -- timing out the - // liveness call itself instead of the batch behind it, which is exactly the distinction - // this fix exists to preserve. - await vi.waitFor(() => - expect(worker.calls.filter((call) => call === "list.get:devices").length).toBeGreaterThan( - listCallsBefore, - ), - ); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"); - } - expect(eventNames(harness.events)).not.toContain("worker.disconnected"); - - await harness.service.stop(); - }); - - it("closes exactly the link a token authorized, and leaves every other link alone (C-2)", async () => { - const harness = fleet({ - authenticate: (credential) => - credential === undefined - ? "unauthenticated" - : { outcome: "accept" as const, tokenId: credential }, - }); - await harness.service.start(); - - const workerA = new ScriptedWorkerClient(); - await harness.join("wrk_a", workerA, undefined, "tok_a"); - const workerB = new ScriptedWorkerClient(); - await harness.join("wrk_b", workerB, undefined, "tok_b"); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"), - ); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_b")?.connection).toBe("connected"), - ); - - await harness.service.closeLinksForToken("tok_a"); - - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_a")?.connection).toBe("disconnected"), - ); - expect(harness.service.workers.view("wrk_b")?.connection).toBe("connected"); - expect(eventNames(harness.events)).toEqual( - expect.arrayContaining(["worker.connected", "worker.connected", "worker.disconnected"]), - ); - - await harness.service.stop(); - }); - - it("does not hang the handshake forever when hello itself never answers (C3)", async () => { - const harness = fleet(); - await harness.service.start(); - - const workerEnd = await harness.joinSilent("wrk_1", "mac-mini-1"); - await vi.waitFor(() => expect(harness.clock.pendingTimerCount).toBeGreaterThan(0)); - harness.clock.advance(WORKER_CALL_TIMEOUT_MS); - - await vi.waitFor(() => expect(workerEnd.closed).toBe(true)); - - expect(eventNames(harness.events)).not.toContain("worker.connected"); - expect(harness.service.workers.view("wrk_1")).toBeUndefined(); - - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("connected"), - ); - - await harness.service.stop(); - }); - - it("closes an uplink whose worker sends an unfinished line past the reply cap, and keeps one at the cap", async () => { - const harness = fleet({ realConnect: true }); - await harness.service.start(); - const workerEnd = await harness.transport.connect({ - token: "join-secret", - url: "ws://gateway.test", - workerId: "wrk_1", - }); - let helloId: number | string | undefined; - workerEnd.onData((chunk) => { - const frame = JSON.parse(chunk.trim().split("\n")[0] ?? "{}") as { - id?: number | string; - type?: string; - }; - if (frame.type === "hello") helloId = frame.id; - }); - await vi.waitFor(() => expect(helloId).toBeDefined()); - await workerEnd.write( - `${JSON.stringify({ - id: helloId, - ok: true, - payload: { - daemonProtocolRange: PROTOCOL_VERSION_RANGE, - principal: "gw:instance-1", - protocolVersion: negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, PROTOCOL_VERSION_RANGE), - role: "admin", - version: "0.3.0", - }, - })}\n`, - ); - const chunk = "a".repeat(64 * 1024); - for (let sent = 0; sent < REPLY_LINE_CAP; sent += chunk.length) { - await workerEnd.write(chunk); - } - expect(workerEnd.closed).toBe(false); - await workerEnd.write(chunk); - await vi.waitFor(() => expect(workerEnd.closed).toBe(true)); - - await harness.service.stop(); - }); - - it("sweeps a retired view on the tick, once retention has passed", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - const workerEnd = await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - await workerEnd.close(); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_1")?.connection).toBe("disconnected"), - ); - - harness.clock.advance(RETENTION_MS + REFRESH_MS); - - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")).toBeUndefined()); - expect(eventNames(harness.events)).toContain("worker.removed"); - - await harness.service.stop(); - }); - - it.each(["unauthenticated", "forbidden", "worker-mismatch"] as const)( - "reports a %s uplink it turned away, without inventing a worker for it", - async (reason) => { - const harness = fleet({ authenticate: () => reason }); - await harness.service.start(); - - await expect( - harness.join("wrk_1", new ScriptedWorkerClient(), "mac-mini-1"), - ).rejects.toMatchObject({ - code: "rejected", - }); - - expect(harness.service.workers.views()).toEqual([]); - expect(harness.events.at(-1)).toMatchObject({ - event: "worker.rejected", - payload: { label: "mac-mini-1", reason, workerId: "wrk_1" }, - }); - - await harness.service.stop(); - }, - ); - - it("refuses a second worker id on a token pinned to another worker as worker-mismatch, leaving that worker's link alone, while the pinned worker's own redial replaces its link", async () => { - const pins = new Map(); - const harness = fleet({ - authenticate: (credential, workerId) => { - if (!pins.has(credential)) pins.set(credential, workerId); - return pins.get(credential) === workerId - ? { outcome: "accept" as const, tokenId: "tok_a" } - : "worker-mismatch"; - }, - }); - await harness.service.start(); - const workerEndA = await harness.join("wrk_a", new ScriptedWorkerClient("admin", "1.0.0")); - await vi.waitFor(() => - expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"), - ); - - await expect( - harness.join("wrk_b", new ScriptedWorkerClient(), "intruder"), - ).rejects.toMatchObject({ code: "rejected" }); - - expect(harness.events.filter((event) => event.event === "worker.rejected")).toMatchObject([ - { payload: { label: "intruder", reason: "worker-mismatch", workerId: "wrk_b" } }, - ]); - expect(harness.service.workers.view("wrk_b")).toBeUndefined(); - expect(workerEndA.closed).toBe(false); - expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"); - - await harness.join("wrk_a", new ScriptedWorkerClient("admin", "2.0.0")); - - await vi.waitFor(() => expect(workerEndA.closed).toBe(true)); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_a")?.version).toBe("2.0.0")); - expect(harness.service.workers.view("wrk_a")?.connection).toBe("connected"); - - await harness.service.stop(); - }); - - it("coalesces a flood of identical refusals into one event per window, with a count only above one (P-2)", async () => { - const harness = fleet({ authenticate: () => "unauthenticated" }); - await harness.service.start(); - const rejectedEvents = () => - harness.events.filter((event) => event.event === "worker.rejected"); - - await expect( - harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), - ).rejects.toMatchObject({ code: "rejected" }); - await expect( - harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), - ).rejects.toMatchObject({ code: "rejected" }); - - expect(rejectedEvents()).toHaveLength(1); - expect(rejectedEvents()[0]?.payload).toMatchObject({ workerId: "wrk_flood" }); - expect(rejectedEvents()[0]?.payload).not.toHaveProperty("count"); - - harness.clock.advance(REJECTION_COALESCE_WINDOW_MS); - await expect( - harness.join("wrk_flood", new ScriptedWorkerClient(), "mac-mini-1"), - ).rejects.toMatchObject({ code: "rejected" }); - - expect(rejectedEvents()).toHaveLength(2); - expect(rejectedEvents()[1]?.payload).toMatchObject({ count: 2, workerId: "wrk_flood" }); - - await expect( - harness.join("wrk_other", new ScriptedWorkerClient(), "mac-mini-2"), - ).rejects.toMatchObject({ code: "rejected" }); - expect(rejectedEvents()).toHaveLength(3); - expect(rejectedEvents()[2]?.payload).not.toHaveProperty("count"); - - // why: both EVENTS.md cuts document `count` as "present only above one", so a lone refusal - // whose predecessor window absorbed exactly one -- the normal path for a worker redialling - // on its backoff, which lands outside every window by design -- must not carry `count: 1`. - // Presence is the flood signal (#677). - harness.clock.advance(REJECTION_COALESCE_WINDOW_MS); - await expect( - harness.join("wrk_other", new ScriptedWorkerClient(), "mac-mini-2"), - ).rejects.toMatchObject({ code: "rejected" }); - expect(rejectedEvents()).toHaveLength(4); - expect(rejectedEvents()[3]?.payload).toMatchObject({ workerId: "wrk_other" }); - expect(rejectedEvents()[3]?.payload).not.toHaveProperty("count"); - - await harness.service.stop(); - }); - - it("closes the uplink when a worker does not grant the gateway admin", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient("agent"); - - const workerEnd = await harness.join("wrk_1", worker); - - await vi.waitFor(() => expect(worker.closed).toBe(true)); - expect(workerEnd.closed).toBe(true); - expect(harness.service.workers.views()).toEqual([]); - - await harness.service.stop(); - }); - - it("stops the tick and closes every link on stop", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - await harness.service.stop(); - - expect(worker.closed).toBe(true); - const callsAfterStop = worker.calls.length; - harness.clock.advance(10 * REFRESH_MS); - expect(worker.calls).toHaveLength(callsAfterStop); - }); - - describe("target() / WorkerDispatchTarget (fleet-ports)", () => { - it("has no client before the handshake completes", async () => { - const harness = fleet(); - await harness.service.start(); - - await harness.joinSilent("wrk_1"); - const target = harness.service.target("wrk_1"); - - expect(target).toBeDefined(); - expect(target?.reachable).toBe(true); - expect(target?.client()).toBeUndefined(); - - await harness.service.stop(); - }); - - it("has no client once the link has closed", async () => { - const harness = fleet(); - await harness.service.start(); - const worker = new ScriptedWorkerClient(); - await harness.join("wrk_1", worker); - await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.capacity).toBeDefined()); - - const target = harness.service.target("wrk_1"); - expect(target?.client()).toBeDefined(); - expect(target?.reachable).toBe(true); - - await harness.service.stop(); - - expect(target?.reachable).toBe(false); - expect(target?.client()).toBeUndefined(); - }); - }); }); describe("GatewayService: a worker's capabilities in its view", () => { diff --git a/src/gateway/test-service.ts b/src/gateway/test-service.ts new file mode 100644 index 000000000..ebad49479 --- /dev/null +++ b/src/gateway/test-service.ts @@ -0,0 +1,118 @@ +import { EventBus, type EventEnvelope } from "../bus/index.js"; +import { + FakeClock, + MemoryUplinkTransport, + type Logger, + type UplinkAuthOutcome, + type UplinkAuthResult, +} from "../ports/index.js"; +import { MemoryDrainStore } from "./drain-store.js"; +import { GatewayService } from "./service.js"; +import { ScriptedWorkerClient } from "./test-support.js"; + +export const RETENTION_MS = 24 * 60 * 60_000; + +export const REFRESH_MS = 30_000; + +export const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; + +export class RecordingLogger implements Logger { + readonly warnings: Array<{ message: string; fields?: Record }> = []; + + debug(): void {} + info(): void {} + warn(message: string, fields?: Record): void { + this.warnings.push(fields === undefined ? { message } : { fields, message }); + } + readonly errors: string[] = []; + error(message: string): void { + this.errors.push(message); + } + child(): Logger { + return this; + } +} + +export function fleet( + options: { + readonly authenticate?: ( + credential: string | undefined, + workerId: string | undefined, + ) => UplinkAuthOutcome | UplinkAuthResult; + readonly leaseMaxTtlMs?: number; + readonly logger?: Logger; + readonly refreshIntervalMs?: number; + /** why: leaves `connect` unset so the gateway builds its real client over the uplink. */ + readonly realConnect?: boolean; + } = {}, +) { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const events: EventEnvelope[] = []; + eventBus.subscribeAll((envelope) => events.push(envelope)); + const transport = new MemoryUplinkTransport(); + const clients: ScriptedWorkerClient[] = []; + const connectState = { hangNext: false }; + const service = new GatewayService({ + authenticate: async (credential, workerId) => + options.authenticate?.(credential, workerId) ?? "accept", + clock, + ...(options.realConnect === true + ? {} + : { + connect: async () => { + if (connectState.hangNext) { + connectState.hangNext = false; + return new Promise(() => {}); + } + const client = clients.at(-1); + if (client === undefined) + throw new Error("no scripted worker was queued for this uplink"); + return client.asClient(); + }, + }), + drainStore: new MemoryDrainStore(), + eventBus, + leaseMaxTtlMs: options.leaseMaxTtlMs ?? DEFAULT_LEASE_MAX_TTL_MS, + ...(options.logger === undefined ? {} : { logger: options.logger }), + principal: "gw:instance-1", + refreshIntervalMs: options.refreshIntervalMs ?? REFRESH_MS, + retentionMs: RETENTION_MS, + uplinks: transport, + }); + return { + clients, + clock, + eventBus, + events, + service, + transport, + join: async ( + workerId: string, + client: ScriptedWorkerClient, + label?: string, + token = "join-secret", + ) => { + clients.push(client); + return transport.connect({ + token, + url: "ws://gateway.test", + workerId, + ...(label === undefined ? {} : { label }), + }); + }, + joinSilent: async (workerId: string, label?: string) => { + connectState.hangNext = true; + return transport.connect({ + token: "join-secret", + url: "ws://gateway.test", + workerId, + ...(label === undefined ? {} : { label }), + }); + }, + }; +} + +export function eventNames(events: readonly EventEnvelope[]): string[] { + return events.map((event) => event.event); +} From f6641b801e4bb497da2795b4d516216bb77282be Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:30:09 +0100 Subject: [PATCH 42/83] test(gateway): split fleet-coordinator.test.ts by topic --- src/gateway/fleet-coordinator-exec.test.ts | 385 +++++ .../fleet-coordinator-forwarding.test.ts | 429 +++++ ...fleet-coordinator-view-and-release.test.ts | 323 ++++ src/gateway/fleet-coordinator-walk.test.ts | 399 +++++ src/gateway/fleet-coordinator.test.ts | 1467 ----------------- 5 files changed, 1536 insertions(+), 1467 deletions(-) create mode 100644 src/gateway/fleet-coordinator-exec.test.ts create mode 100644 src/gateway/fleet-coordinator-forwarding.test.ts create mode 100644 src/gateway/fleet-coordinator-view-and-release.test.ts create mode 100644 src/gateway/fleet-coordinator-walk.test.ts diff --git a/src/gateway/fleet-coordinator-exec.test.ts b/src/gateway/fleet-coordinator-exec.test.ts new file mode 100644 index 000000000..1ff7c753d --- /dev/null +++ b/src/gateway/fleet-coordinator-exec.test.ts @@ -0,0 +1,385 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../contract/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import { promiseState } from "../test-support/promise-state.js"; +import { grantFixture, ScriptedWorkerClient } from "./test-support.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator dispatch", () => { + it("forwards device.exec with the namespaced requesterId reaching the scripted worker client", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { + ...grantFixture().lease, + id: "lse_9", + ownerId: "agent-1", + requesterId: `${GATEWAY_PREFIX}agent-1`, + }, + }), + kind: "grant", + }); + + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ exitCode: 0, kind: "ok" }); + + await coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent-1", + role: "agent", + }, + ); + + expect(client.calls).toContain(`device.exec:${GATEWAY_PREFIX}agent-1`); + }); + + it("announces started from the worker's own push, before any output (ADR §19a)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ + exitCode: 0, + kind: "ok", + output: [{ chunk: "hi", stream: "stdout" }], + started: true, + }); + + const order: string[] = []; + await coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onOutput: () => { + order.push("output"); + }, + onStarted: () => { + order.push("started"); + }, + principal: "agent-1", + role: "agent", + }, + ); + + expect(order).toEqual(["started", "output"]); + expect(client.calls).toContain(`device.exec:${GATEWAY_PREFIX}agent-1`); + }); + + it.each([ + ["an empty queue", 0], + ["another request queued", 1], + ] as const)( + "rejects a noWait request refused by its only worker with NO_CAPACITY and lease.rejected no-wait, refreshes that worker's view, and never queues it (no lease.queued, no queued progress), with %s -- the answer does not depend on queue depth (ADR §10, architecture rule 10)", + async (_queue, queued) => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + if (queued > 0) { + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + await coordinator.request(REQUEST, requestOptions()); + void coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), + ); + await tick(); + } + expect(coordinator.queueDepth).toBe(queued); + const rejected: unknown[] = []; + eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); + const events: string[] = []; + eventBus.subscribe("lease.queued", () => events.push("lease.queued")); + eventBus.subscribe("lease.rejected", () => events.push("lease.rejected")); + const progress: string[] = []; + const refreshesBefore = directory.refreshCalls.length; + + const refused = coordinator.request( + REQUEST, + requestOptions({ + noWait: true, + onProgress: (update) => progress.push(update.stage), + ownerId: "agent-3", + requesterId: "agent-3", + }), + ); + const refusedState = promiseState(refused); + await tick(); + + expect(refusedState.state).toBe("rejected"); + await expect(refused).rejects.toMatchObject({ code: "NO_CAPACITY" }); + expect(client.calls).toContain(`lease.request:${GATEWAY_PREFIX}agent-3`); + expect(rejected).toEqual([expect.objectContaining({ reason: "no-wait" })]); + expect(events).toEqual(["lease.rejected"]); + expect(progress).toEqual([]); + expect(directory.refreshCalls.slice(refreshesBefore)).toEqual(["wrk_a"]); + expect(coordinator.queueDepth).toBe(queued); + }, + ); + + it("relays the worker's own started push for a silent command that never answers (ADR §19a/§19b/§19e)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ kind: "hang", started: true }); + + let started = false; + void coordinator.exec( + { args: ["install", "/path/to.app"], leaseId: grant.lease.id, tool: "simctl" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onStarted: () => { + started = true; + }, + principal: "agent-1", + role: "agent", + }, + ); + await tick(); + + expect(started).toBe(true); + }); + + it("does not announce started for a command the worker refuses before any process exists -- a pre-process refusal keeps its own real status (ADR §19a′)", async () => { + const { clock, coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ + error: new SimlockError("FORBIDDEN", "domain", "Lease belongs to a different requester", {}), + kind: "error", + }); + + let started = false; + const rejection = await coordinator + .exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onStarted: () => { + started = true; + }, + principal: "agent-1", + role: "agent", + }, + ) + .catch((error: unknown) => error); + + expect(rejection).toBeInstanceOf(DispatchError); + expect((rejection as DispatchError).code).toBe("FORBIDDEN"); + expect(started).toBe(false); + + clock.advance(10_000); + expect(started).toBe(false); + }); + + it("times out a forwarded device.exec after gateway.execTimeoutMs when the worker never answers at all (P5)", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { + ...grantFixture().lease, + id: "lse_9", + ownerId: "agent-1", + requesterId: `${GATEWAY_PREFIX}agent-1`, + }, + }), + kind: "grant", + }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ kind: "hang" }); + + const execution = coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent-1", + role: "agent", + }, + ); + const executionState = promiseState(execution); + const rejection = execution.catch((error: unknown) => error); + await tick(); + clock.advance(4_999); + await tick(); + expect(executionState.state).toBe("pending"); + + clock.advance(1); + await tick(); + expect(executionState.state).toBe("rejected"); + const error = await rejection; + expect(error).toBeInstanceOf(DispatchError); + expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); + }); + + it("times out a forwarded device.rotate after gateway.execTimeoutMs when the worker never answers", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.hangingCalls.add("device.rotate"); + + const rotation = coordinator.rotate( + { leaseId: grant.lease.id, orientation: "landscape_left" }, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent-1", + role: "agent", + }, + ); + const rotationState = promiseState(rotation); + const rejection = rotation.catch((error: unknown) => error); + await tick(); + clock.advance(4_999); + await tick(); + expect(rotationState.state).toBe("pending"); + + clock.advance(1); + await tick(); + expect(rotationState.state).toBe("rejected"); + const error = await rejection; + expect(error).toBeInstanceOf(DispatchError); + expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); + }); + + it("stops relaying a worker's device.exec output once gateway.execTimeoutMs has already rejected the call (H4, round 3 review)", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { + ...grantFixture().lease, + id: "lse_9", + ownerId: "agent-1", + requesterId: `${GATEWAY_PREFIX}agent-1`, + }, + }), + kind: "grant", + }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ kind: "hang" }); + + const relayed: Array<{ stream: string; chunk: string }> = []; + const execution = coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onOutput: (stream, chunk) => { + relayed.push({ chunk, stream }); + }, + principal: "agent-1", + role: "agent", + }, + ); + const executionState = promiseState(execution); + const rejection = execution.catch((error: unknown) => error); + await tick(); + clock.advance(5_000); + await tick(); + expect(executionState.state).toBe("rejected"); + const error = await rejection; + expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); + + client.lastExecOptions?.onOutput?.({ chunk: "late output", stream: "stdout" }); + await tick(); + + expect(relayed).toEqual([]); + }); + + it("does not announce started from a worker frame that arrives after gateway.execTimeoutMs already rejected the call (H4, round 6 review)", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ kind: "hang" }); + + let startedCount = 0; + const execution = coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onStarted: () => { + startedCount += 1; + }, + principal: "agent-1", + role: "agent", + }, + ); + const executionState = promiseState(execution); + const rejection = execution.catch((error: unknown) => error); + await tick(); + clock.advance(5_000); + await tick(); + expect(executionState.state).toBe("rejected"); + const error = await rejection; + expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); + + client.lastExecOptions?.onStarted?.(); + await tick(); + + expect(startedCount).toBe(0); + }); + + it("announces started once even if the worker sends the frame more than once", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request(REQUEST, requestOptions()); + client.execQueue.push({ exitCode: 0, kind: "ok", started: true }); + + let startedCount = 0; + await coordinator.exec( + { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, + { + manageEventSubscription: () => undefined, + via: "socket", + onStarted: () => { + startedCount += 1; + }, + principal: "agent-1", + role: "agent", + }, + ); + client.lastExecOptions?.onStarted?.(); + await tick(); + + expect(startedCount).toBe(1); + }); +}); diff --git a/src/gateway/fleet-coordinator-forwarding.test.ts b/src/gateway/fleet-coordinator-forwarding.test.ts new file mode 100644 index 000000000..151006e3c --- /dev/null +++ b/src/gateway/fleet-coordinator-forwarding.test.ts @@ -0,0 +1,429 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../contract/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import { promiseState } from "../test-support/promise-state.js"; +import { + deviceFixture, + grantFixture, + noCapacityError, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator dispatch", () => { + it("bounds a forwarded lease.request against gateway.leaseRequestTimeoutMs when the worker never answers at all (P2, round 2 review)", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ leaseRequestTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ kind: "hang" }); + + const request = coordinator.request(REQUEST, requestOptions()); + const requestState = promiseState(request); + const rejection = request.catch((error: unknown) => error); + await tick(); + expect(coordinator.queueDepth).toBe(0); + expect(directory.refreshCalls).toEqual([]); + + clock.advance(4_999); + await tick(); + expect(requestState.state).toBe("pending"); + + clock.advance(1); + await tick(); + expect(requestState.state).toBe("rejected"); + const error = await rejection; + + expect(error).toBeInstanceOf(DispatchError); + expect((error as DispatchError).code).toBe("WORKER_UNREACHABLE"); + expect(coordinator.queueDepth).toBe(0); + }); + + it("does not emit request.dispatched for a progress push that arrives after gateway.leaseRequestTimeoutMs already rejected the waiter (P4, round 3 review)", async () => { + const { clock, coordinator, directory, eventBus, workers } = buildFleet({ + leaseRequestTimeoutMs: 5_000, + }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ kind: "hang" }); + const events: string[] = []; + eventBus.subscribe("request.dispatched", () => events.push("request.dispatched")); + eventBus.subscribe("lease.rejected", () => events.push("lease.rejected")); + + const request = coordinator.request(REQUEST, requestOptions()); + const requestState = promiseState(request); + const rejection = request.catch((error: unknown) => error); + await tick(); + + clock.advance(5_000); + await tick(); + expect(requestState.state).toBe("rejected"); + const error = await rejection; + expect((error as DispatchError).code).toBe("WORKER_UNREACHABLE"); + expect(events).toEqual(["lease.rejected"]); + + client.lastRequestLeaseOptions?.onProgress?.({ etaMs: 5_000, stage: "provisioning" }); + await tick(); + + expect(events).toEqual(["lease.rejected"]); + }); + + it("passes over a request only a busy worker can serve instead of blocking behind it, and never lets a brand-new admission jump an older, equally-eligible waiter still queued behind it (C2, round 3 review)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_ios", client); + directory.add("wrk_android", new ScriptedWorkerClient()); + client.requestLeaseQueue.push({ kind: "hang" }); + + const capacity = statusFixture().capacity; + connectWorker(workers, "wrk_ios", { capacity: saturatedIos() }); + connectWorker(workers, "wrk_android", { + capacity: { ...capacity, android: { ...capacity.android, maxRunning: 0 } }, + models: ["Pixel 9"], + platform: "android", + }); + const androidPromise = coordinator.request( + { model: "Pixel 9", platform: "android" }, + requestOptions({ ownerId: "agent-android", requesterId: "agent-android" }), + ); + const oldFirstPromise = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-old-1", requesterId: "agent-old-1" }), + ); + const oldSecondPromise = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-old-2", requesterId: "agent-old-2" }), + ); + await tick(); + expect(coordinator.queueDepth).toBe(3); + + workers.refresh("wrk_ios", { capacity: statusFixture().capacity }); + await tick(); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + + let androidSettled = false; + void androidPromise.then( + () => { + androidSettled = true; + }, + () => { + androidSettled = true; + }, + ); + await tick(); + expect(androidSettled).toBe(false); + + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const newPromise = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-new", requesterId: "agent-new" }), + ); + await tick(); + + const leaseRequests = client.calls.filter((call) => call.startsWith("lease.request")); + expect(leaseRequests.length).toBeGreaterThanOrEqual(2); + expect(leaseRequests[1]).toContain("agent-old-2"); + + const oldSecondGrant = await oldSecondPromise; + expect(oldSecondGrant.lease.worker?.id).toBe("wrk_ios"); + expect(androidSettled).toBe(false); + void oldFirstPromise; + void newPromise; + }); + + it("indexes a fresh grant under the ownerId it forwarded, not the worker's own echo of it (H6)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture({ + lease: { + ...grantFixture().lease, + ownerId: "worker-rewrote-this", + }, + }), + kind: "grant", + }); + + const grant = await coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), + ); + + expect(coordinator.ownerId(grant.lease.id)).toBe("agent-1"); + expect(grant.lease.ownerId).toBe("agent-1"); + }); + + it("emits request.dispatched on the worker's first progress push even when the request never settles at all (§11, C4 round 2 review)", async () => { + const { coordinator, eventBus, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + kind: "hang", + progress: [{ etaMs: 5_000, stage: "provisioning" }], + }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + let settled = false; + void coordinator.request(REQUEST, requestOptions()).then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await tick(); + + expect(settled).toBe(false); + expect(dispatched).toEqual([ + { + model: "iPhone 17", + platform: "ios", + queuedMs: expect.any(Number) as number, + reason: "free-capacity", + requestId: expect.any(String) as string, + requesterId: "agent-1", + stage: "free-capacity", + workerId: "wrk_a", + }, + ]); + }); + + it("emits request.dispatched exactly once, on the worker's first progress push, not again when the grant lands afterward (§11)", async () => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture(), + kind: "grant", + progress: [ + { etaMs: 5_000, stage: "provisioning" }, + { etaMs: 1_000, stage: "booting" }, + ], + }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + const grant = await coordinator.request(REQUEST, requestOptions()); + + expect(dispatched).toEqual([ + { + model: "iPhone 17", + platform: "ios", + queuedMs: expect.any(Number) as number, + reason: "free-capacity", + requestId: expect.any(String) as string, + requesterId: "agent-1", + stage: "free-capacity", + workerId: "wrk_a", + }, + ]); + expect(grant.lease.worker?.id).toBe("wrk_a"); + }); + + it.each(["full", "slim"] as const)( + "request.dispatched carries the requested mode %s", + async (mode) => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + grant: grantFixture(), + kind: "grant", + progress: [{ etaMs: 5_000, stage: "provisioning" }], + }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + await coordinator.request({ ...REQUEST, mode }, requestOptions()); + + expect(dispatched).toEqual([expect.objectContaining({ mode })]); + }, + ); + + it("request.dispatched carries no mode and no physical key for a plain virtual request", async () => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const dispatched: object[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + await coordinator.request(REQUEST, requestOptions()); + + expect(dispatched).toHaveLength(1); + expect(dispatched[0]).not.toHaveProperty("mode"); + expect(dispatched[0]).not.toHaveProperty("physical"); + }); + + it("relays a worker's downloading pushes to the requester unchanged", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + const pushed = [ + { component: "26.4", stage: "downloading", waiting: true }, + { component: "26.4", stage: "downloading", waiting: false }, + { component: "26.4", percent: 41, stage: "downloading", waiting: false }, + { etaMs: 5_000, stage: "provisioning" }, + ] as const; + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant", progress: pushed }); + const heard: unknown[] = []; + + await coordinator.request( + REQUEST, + requestOptions({ allowDownload: true, onProgress: (progress) => heard.push(progress) }), + ); + + expect(heard).toEqual(pushed); + }); + + it("emits request.dispatched when a grant lands with no progress push at all -- the ADR §11 rule's other half", async () => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + await coordinator.request(REQUEST, requestOptions()); + + expect(dispatched).toEqual([ + { + model: "iPhone 17", + platform: "ios", + queuedMs: expect.any(Number) as number, + reason: "free-capacity", + requestId: expect.any(String) as string, + requesterId: "agent-1", + stage: "free-capacity", + workerId: "wrk_a", + }, + ]); + }); + + it("reports request.dispatched's reason as warm-hit when routing picked the worker for an unleased ready device, and queuedMs as the real wait (C2, round 2 review)", async () => { + const { clock, coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + const noFree = { ...statusFixture().capacity.ios, maxRunning: 0, running: 0 }; + connectWorker(workers, "wrk_a", { capacity: { ...statusFixture().capacity, ios: noFree } }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + clock.advance(2_500); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + workers.refresh("wrk_a", { + capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5 } }, + devices: [deviceFixture("dev_1", "ready")], + }); + + await grantPromise; + + expect(dispatched).toEqual([ + { + model: "iPhone 17", + platform: "ios", + queuedMs: 2_500, + reason: "warm-hit", + requestId: expect.any(String) as string, + requesterId: "agent-1", + stage: "warm-hit", + workerId: "wrk_a", + }, + ]); + }); + + it("request.dispatched carries the routing decision's stage name and reason", async () => { + const { coordinator, directory, eventBus, workers } = buildFleet({ + routing: () => ({ + assess: () => ({ kind: "route-or-wait" }), + explain: () => [], + select: (_request, views) => + views[0] === undefined + ? undefined + : { reason: "warm-hit", stage: "test-rank", workerId: views[0].id }, + }), + }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + await coordinator.request(REQUEST, requestOptions()); + + expect(dispatched).toEqual([ + expect.objectContaining({ reason: "warm-hit", stage: "test-rank", workerId: "wrk_a" }), + ]); + }); + + it("does not emit request.dispatched for a stale-view NO_CAPACITY -- the request stayed queued, never actually dispatched", async () => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ error: noCapacityError(), kind: "error" }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + void coordinator.request(REQUEST, requestOptions()); + await tick(); + + expect(dispatched).toEqual([]); + expect(coordinator.queueDepth).toBe(1); + }); + + it("handles a worker's DAEMON_UPDATING answer as NO_CAPACITY: the view is read again, the request stays queued and is placed once the view changes", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + error: new SimlockError("DAEMON_UPDATING", "domain", "The daemon is updating", {}), + kind: "error", + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const outcome = coordinator.request(REQUEST, requestOptions()); + const state = promiseState(outcome); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.queueDepth).toBe(1); + expect(directory.refreshCalls).toEqual(["wrk_a"]); + + workers.refresh("wrk_a", { + update: { forced: false, from: "0.3.0", stage: "installing", startedAt: 1, to: "0.4.0" }, + }); + await tick(); + expect(state.state).toBe("pending"); + + workers.refresh("wrk_a", { update: null }); + await tick(); + + expect((await outcome).lease.worker?.id).toBe("wrk_a"); + }); +}); diff --git a/src/gateway/fleet-coordinator-view-and-release.test.ts b/src/gateway/fleet-coordinator-view-and-release.test.ts new file mode 100644 index 000000000..5cd239c0e --- /dev/null +++ b/src/gateway/fleet-coordinator-view-and-release.test.ts @@ -0,0 +1,323 @@ +import { describe, expect, it } from "vitest"; +import type { LeaseRecord } from "../admin/index.js"; +import { SimlockError } from "../contract/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; +import { RequesterAlreadyLeasedError } from "./queue.js"; +import { createRoutingPolicy } from "./index.js"; +import { grantFixture, leaseFixture, ScriptedWorkerClient, statusFixture } from "./test-support.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator dispatch", () => { + it("runs another pass for a view change raised from inside the walk, rather than dropping it (round 4 finding 6, round 6 review)", async () => { + let selectCalls = 0; + const { coordinator, directory, workers } = buildFleet({ + routing: (registry) => { + const real = createRoutingPolicy("warm-then-free"); + return { + assess: real.assess, + explain: real.explain, + select(request, workerViews) { + selectCalls += 1; + if (selectCalls === 2) { + registry.refresh("wrk_a", {}); + return undefined; + } + return real.select(request, workerViews); + }, + }; + }, + }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + workers.refresh("wrk_a", { capacity: statusFixture().capacity }); + await tick(); + + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + expect(selectCalls).toBe(3); + const grant = await grantPromise; + expect(grant.lease.worker?.id).toBe("wrk_a"); + expect(coordinator.queueDepth).toBe(0); + }); + + it.each([ + { + code: "WORKER_UNREACHABLE", + details: { workerId: "wrk_a" }, + error: new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + "Daemon connection is closed", + {}, + ), + row: "a lost connection", + }, + { + code: "WORKER_UNREACHABLE", + details: { workerId: "wrk_a" }, + error: new SimlockError("DAEMON_STOPPING", "transport", "The daemon is stopping", {}), + row: "a stopping daemon (#514)", + }, + { + code: "INTERNAL", + details: undefined, + error: new TypeError("boom"), + row: "a non-SimlockError (H1, round 2 review)", + }, + ])( + "maps $row on a forwarded lease.request to $code, never the uplink client's own error code verbatim (P4)", + async ({ code, details, error }) => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ error, kind: "error" }); + + const rejection = await coordinator + .request(REQUEST, requestOptions()) + .catch((caught: unknown) => caught); + expect(rejection).toBeInstanceOf(DispatchError); + expect(rejection).toMatchObject({ code, ...(details === undefined ? {} : { details }) }); + }, + ); + + it.each([ + { + code: "WORKER_UNREACHABLE", + error: new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + "Daemon connection is closed", + {}, + ), + row: "a transport failure", + }, + { + code: "INTERNAL", + error: new TypeError("boom"), + row: "a non-SimlockError (H1, round 2 review)", + }, + ])( + "maps $row on a forwarded lease.renew to $code, never the other (P4)", + async ({ code, error }) => { + const { coordinator, directory, workers, leaseIndex } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + leaseIndex.add({ + gatewayLeaseId: "wrk_a.lse_1", + grantedAt: 1, + ownerId: "agent-1", + requesterId: "agent-1", + workerId: "wrk_a", + workerLeaseId: "lse_1", + }); + client.renewLeaseQueue.push({ error, kind: "error" }); + + const rejection = await coordinator + .renew("wrk_a.lse_1", undefined) + .catch((caught: unknown) => caught); + expect(rejection).toBeInstanceOf(DispatchError); + expect((rejection as DispatchError).code).toBe(code); + }, + ); + + it.each([ + { + answer: (client: ScriptedWorkerClient, error: SimlockError<"UNKNOWN_LEASE">) => + client.releaseLeaseQueue.push({ error, kind: "error" }), + call: (coordinator: FleetLeaseCoordinator) => coordinator.release("wrk_a.lse_1"), + verb: "release", + }, + { + answer: (client: ScriptedWorkerClient, error: SimlockError<"UNKNOWN_LEASE">) => + client.renewLeaseQueue.push({ error, kind: "error" }), + call: (coordinator: FleetLeaseCoordinator) => coordinator.renew("wrk_a.lse_1", undefined), + verb: "renew", + }, + ])( + "drops its own index entry when the worker answers UNKNOWN_LEASE to a $verb, instead of leaving a zombie behind (C3)", + async ({ answer, call }) => { + const { coordinator, directory, workers, leaseIndex } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + leaseIndex.add({ + gatewayLeaseId: "wrk_a.lse_1", + grantedAt: 1, + ownerId: "agent-1", + requesterId: "agent-1", + workerId: "wrk_a", + workerLeaseId: "lse_1", + }); + answer( + client, + new SimlockError("UNKNOWN_LEASE", "domain", "Unknown lease: lse_1", { leaseId: "lse_1" }), + ); + + await expect(call(coordinator)).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); + + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + await expect(coordinator.request(REQUEST, requestOptions())).resolves.toBeDefined(); + }, + ); + + it("release-all treats a worker's UNKNOWN_LEASE as already-released rather than a failure that blocks every future call (C3)", async () => { + const { coordinator, directory, workers, leaseIndex } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + leaseIndex.add({ + gatewayLeaseId: "wrk_a.lse_zombie", + grantedAt: 1, + ownerId: "agent-1", + requesterId: "agent-1", + workerId: "wrk_a", + workerLeaseId: "lse_zombie", + }); + client.releaseLeaseQueue.push({ + error: new SimlockError("UNKNOWN_LEASE", "domain", "Unknown lease: lse_zombie", { + leaseId: "lse_zombie", + }), + kind: "error", + }); + + await expect(coordinator.releaseAll()).resolves.toEqual(["wrk_a.lse_zombie"]); + await expect(coordinator.releaseAll()).resolves.toEqual([]); + }); + + it("does not evict a lease on one worker when only a different worker's view changes repeatedly (C1, round 2 review)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b"); + clientA.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await coordinator.request(REQUEST, requestOptions()); + expect(grant.lease.worker?.id).toBe("wrk_a"); + expect(workers.view("wrk_a")?.leases).toEqual([]); + + workers.refresh("wrk_b", {}); + workers.refresh("wrk_b", {}); + + const retry = await coordinator + .request(REQUEST, requestOptions({ noWait: true })) + .catch((error: unknown) => error); + expect(retry).toBeInstanceOf(RequesterAlreadyLeasedError); + expect((retry as RequesterAlreadyLeasedError).existingLeaseId).toBe(grant.lease.id); + }); + + it("grants a requester's next lease when a worker view read before its release lands after it (#898)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const first = await coordinator.request(REQUEST, requestOptions()); + client.releaseLeaseQueue.push({ kind: "ok" }); + await coordinator.release(first.lease.id); + + workers.refresh("wrk_a", { + leases: [{ ...leaseFixture("lse_1", "dev_1"), requesterId: `${GATEWAY_PREFIX}agent-1` }], + }); + workers.refresh("wrk_a", { leases: [] }); + + client.requestLeaseQueue.push({ + grant: grantFixture({ lease: leaseFixture("lse_2", "dev_1") as LeaseRecord }), + kind: "grant", + }); + const second = await coordinator + .request(REQUEST, requestOptions({ noWait: true })) + .catch((error: unknown) => error); + expect(second).toMatchObject({ lease: { id: "wrk_a.lse_2" } }); + }); + + it.each(["release", "releaseAll"] as const)( + "grants the next lease when %s's relayed event came before the worker's answer and a stale view follows (#898)", + async (how) => { + const { coordinator, directory, workers, leaseIndex } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const first = await coordinator.request(REQUEST, requestOptions()); + client.onReleaseLease = (leaseId) => leaseIndex.removeByWorkerLease("wrk_a", leaseId); + if (how === "release") await coordinator.release(first.lease.id); + else await coordinator.releaseAll(); + + workers.refresh("wrk_a", { + leases: [{ ...leaseFixture("lse_1", "dev_1"), requesterId: `${GATEWAY_PREFIX}agent-1` }], + }); + workers.refresh("wrk_a", { leases: [] }); + + client.requestLeaseQueue.push({ + grant: grantFixture({ lease: leaseFixture("lse_2", "dev_1") as LeaseRecord }), + kind: "grant", + }); + const second = await coordinator + .request(REQUEST, requestOptions({ noWait: true })) + .catch((error: unknown) => error); + expect(second).toMatchObject({ lease: { id: "wrk_a.lse_2" } }); + }, + ); + + it("release-all attaches the leases it already released to a later failure's own details (H8)", async () => { + const { coordinator, directory, workers, leaseIndex } = buildFleet(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b"); + leaseIndex.add({ + gatewayLeaseId: "wrk_a.lse_1", + grantedAt: 1, + ownerId: "agent-1", + requesterId: "agent-1", + workerId: "wrk_a", + workerLeaseId: "lse_1", + }); + leaseIndex.add({ + gatewayLeaseId: "wrk_b.lse_1", + grantedAt: 1, + ownerId: "agent-2", + requesterId: "agent-2", + workerId: "wrk_b", + workerLeaseId: "lse_1", + }); + clientA.releaseLeaseQueue.push({ kind: "ok" }); + directory.unreachable.add("wrk_b"); + + const rejection = await coordinator.releaseAll().catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(DispatchError); + expect((rejection as DispatchError).code).toBe("WORKER_UNREACHABLE"); + expect((rejection as DispatchError).details).toMatchObject({ + releasedLeaseIds: ["wrk_a.lse_1"], + }); + await expect(coordinator.renew("wrk_a.lse_1", undefined)).rejects.toMatchObject({ + code: "UNKNOWN_LEASE", + }); + expect(clientA.calls.filter((call) => call.startsWith("lease.renew"))).toEqual([]); + await expect(coordinator.renew("wrk_b.lse_1", undefined)).rejects.toMatchObject({ + code: "WORKER_UNREACHABLE", + }); + }); +}); diff --git a/src/gateway/fleet-coordinator-walk.test.ts b/src/gateway/fleet-coordinator-walk.test.ts new file mode 100644 index 000000000..9202a6954 --- /dev/null +++ b/src/gateway/fleet-coordinator-walk.test.ts @@ -0,0 +1,399 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../contract/index.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import { promiseState } from "../test-support/promise-state.js"; +import { + grantFixture, + noCapacityError, + ScriptedWorkerClient, + statusFixture, +} from "./test-support.js"; +import { + GATEWAY_PREFIX, + buildFleet, + connectWorker, + tick, + REQUEST, + saturatedIos, + requestOptions, +} from "./test-fleet.js"; + +describe("FleetLeaseCoordinator dispatch", () => { + it("grants a queued request from whichever worker frees first -- never dispatching to the one routing did not prefer", async () => { + const { coordinator, directory, workers } = buildFleet(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + const noFree = { ...statusFixture().capacity.android, maxRunning: 0, running: 0 }; + connectWorker(workers, "wrk_a", { + capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5, running: 5 } }, + }); + connectWorker(workers, "wrk_b", { + capacity: { ...statusFixture().capacity, ios: noFree }, + }); + + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + clientB.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + workers.refresh("wrk_b", { + capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5, running: 0 } }, + }); + + const grant = await grantPromise; + expect(grant.lease.worker?.id).toBe("wrk_b"); + expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + expect(clientB.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + }); + + it("issues exactly one lease.request for a waiter still in flight, even when another worker's view changes mid-flight", async () => { + const { coordinator, directory, workers } = buildFleet(); + const clientA = new ScriptedWorkerClient(); + const clientB = new ScriptedWorkerClient(); + directory.add("wrk_a", clientA); + directory.add("wrk_b", clientB); + clientA.requestLeaseQueue.push({ kind: "hang" }); + + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + void coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + workers.refresh("wrk_a", { capacity: statusFixture().capacity }); + await tick(); + expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + + connectWorker(workers, "wrk_b", { + capacity: { + ...statusFixture().capacity, + ios: { ...statusFixture().capacity.ios, maxRunning: 10 }, + }, + }); + await tick(); + + expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + expect(clientB.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + }); + + it("dispatches at most one queued waiter per worker per pass, rather than sending every queued waiter at a single over-reporting worker at once (H6, round 3 review)", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + const p1 = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), + ); + const p2 = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), + ); + const p3 = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-3", requesterId: "agent-3" }), + ); + await tick(); + expect(coordinator.queueDepth).toBe(3); + + workers.refresh("wrk_a", { capacity: statusFixture().capacity }); + await tick(); + + const leaseRequests = client.calls.filter((call) => call.startsWith("lease.request")); + expect(leaseRequests).toHaveLength(2); + expect(leaseRequests[1]).toContain("agent-2"); + + await expect(p1).resolves.toBeDefined(); + const outcome = await Promise.race([ + Promise.allSettled([p2, p3]).then(() => "settled" as const), + tick(6).then(() => "still-pending" as const), + ]); + expect(outcome).toBe("still-pending"); + expect(coordinator.queueDepth).toBe(2); + expect(directory.refreshCalls).toEqual(["wrk_a", "wrk_a"]); + }); + + it("gives a waiter passed over by the per-pass cap a guaranteed next look once the claiming attempt settles terminally, even when that failure produces no worker-side event at all (C1, round 3 review)", async () => { + const { clock, coordinator, directory, workers } = buildFleet({ leaseRequestTimeoutMs: 5_000 }); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + client.requestLeaseQueue.push({ kind: "hang" }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); + const w1Request = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), + ); + const w1State = promiseState(w1Request); + const w1 = w1Request.catch((error: unknown) => error); + const w2 = coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), + ); + await tick(); + expect(coordinator.queueDepth).toBe(2); + + workers.refresh("wrk_a", { capacity: statusFixture().capacity }); + await tick(); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + + clock.advance(5_000); + await tick(); + expect(w1State.state).toBe("rejected"); + const w1Error = await w1; + expect(w1Error).toBeInstanceOf(DispatchError); + expect((w1Error as DispatchError).code).toBe("WORKER_UNREACHABLE"); + + const outcome = await Promise.race([ + w2.then( + () => "resolved" as const, + () => "resolved" as const, + ), + tick(6).then(() => "still-pending" as const), + ]); + expect(outcome).toBe("resolved"); + const w2Grant = await w2; + expect(w2Grant.lease.worker?.id).toBe("wrk_a"); + expect(directory.refreshCalls).toEqual(["wrk_a"]); + }); + + it("re-dispatches a waiter whose attempt found a reachable-but-not-yet-connected target, instead of parking it forever", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + directory.reachableButNoClient.add("wrk_a"); + + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + expect(coordinator.queueDepth).toBe(1); + + directory.reachableButNoClient.delete("wrk_a"); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + workers.refresh("wrk_a", {}); // any refresh re-runs dispatch, exactly as a real one would + + const grant = await grantPromise; + expect(grant.lease.worker?.id).toBe("wrk_a"); + expect(coordinator.queueDepth).toBe(0); + }); + + it("leaves a request queued after NO_CAPACITY, refreshes that worker's view, and tries it once more when a refresh changes its capacity", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ error: noCapacityError(), kind: "error" }); + + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + + expect(coordinator.queueDepth).toBe(1); + expect(directory.refreshCalls).toEqual(["wrk_a"]); + + const capacity = statusFixture().capacity; + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + workers.refresh("wrk_a", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 3 } }, + }); + + const grant = await grantPromise; + expect(grant.lease.worker?.id).toBe("wrk_a"); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(2); + }); + + it("sends a worker that keeps answering NO_CAPACITY with an unchanged view one lease.request for that request", async () => { + const { clock, coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + + void coordinator.request(REQUEST, requestOptions()); + await tick(); + for (let refresh = 0; refresh < 3; refresh += 1) { + clock.advance(1_000); // a new lastSeenAt on every refresh, as a real one has + workers.refresh("wrk_a", { capacity: statusFixture().capacity }); + await tick(); + } + + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + expect(coordinator.queueDepth).toBe(1); + }); + + it("tries another eligible worker in the pass that follows a refusal", async () => { + const { coordinator, directory, workers } = buildFleet(); + const preferred = new ScriptedWorkerClient(); + const other = new ScriptedWorkerClient(); + directory.add("wrk_a", preferred); + directory.add("wrk_b", other); + const capacity = statusFixture().capacity; + connectWorker(workers, "wrk_a", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, + }); + connectWorker(workers, "wrk_b"); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grantPromise = coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(other.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + + workers.refresh("wrk_a", {}); + + const grant = await grantPromise; + expect(grant.lease.worker?.id).toBe("wrk_b"); + expect(preferred.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + }); + + it("does not exclude a worker for another request because it refused one", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + + void coordinator.request(REQUEST, requestOptions()); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const grant = await coordinator.request( + REQUEST, + requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), + ); + + expect(grant.lease.worker?.id).toBe("wrk_a"); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toEqual([ + `lease.request:${GATEWAY_PREFIX}agent-1`, + `lease.request:${GATEWAY_PREFIX}agent-2`, + ]); + expect(coordinator.queueDepth).toBe(1); + }); + + it("leaves a request queued when the worker that refused it left the views before the refusal arrived", async () => { + const { coordinator, directory, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + const target = directory.target.bind(directory); + directory.target = (workerId) => { + const found = target(workerId); + if (found === undefined) return undefined; + return { + ...found, + client: () => { + workers.disconnected(workerId); + void workers.remove(workerId); + return found.client(); + }, + }; + }; + + void coordinator.request(REQUEST, requestOptions()); + await tick(); + + expect(workers.views()).toEqual([]); + expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + expect(coordinator.queueDepth).toBe(1); + }); + + it("sends a queued noWait request refused by one worker to only one other worker in the next walk", async () => { + const { coordinator, directory, workers } = buildFleet(); + const first = new ScriptedWorkerClient(); + const second = new ScriptedWorkerClient(); + const third = new ScriptedWorkerClient(); + directory.add("wrk_a", first); + directory.add("wrk_b", second); + directory.add("wrk_c", third); + const capacity = statusFixture().capacity; + connectWorker(workers, "wrk_a", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 6 } }, + }); + connectWorker(workers, "wrk_b", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, + }); + connectWorker(workers, "wrk_c", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, + }); + directory.reachableButNoClient.add("wrk_a"); + second.requestLeaseQueue.push({ kind: "hang" }); + third.requestLeaseQueue.push({ kind: "hang" }); + + void coordinator.request(REQUEST, requestOptions({ noWait: true })); + await tick(); + expect(coordinator.queueDepth).toBe(1); + + directory.reachableButNoClient.delete("wrk_a"); + workers.refresh("wrk_a", {}); + await tick(); + + const leaseRequests = (client: ScriptedWorkerClient) => + client.calls.filter((call) => call.startsWith("lease.request")).length; + expect(leaseRequests(first)).toBe(1); + expect(leaseRequests(second) + leaseRequests(third)).toBe(1); + }); + + it("grants a noWait request refused by one worker from another in the next walk", async () => { + const { coordinator, directory, workers } = buildFleet(); + const preferred = new ScriptedWorkerClient(); + const other = new ScriptedWorkerClient(); + directory.add("wrk_a", preferred); + directory.add("wrk_b", other); + const capacity = statusFixture().capacity; + connectWorker(workers, "wrk_a", { + capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, + }); + connectWorker(workers, "wrk_b"); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const grant = await coordinator.request(REQUEST, requestOptions({ noWait: true })); + + expect(grant.lease.worker?.id).toBe("wrk_b"); + expect(preferred.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); + }); + + it.each([ + { code: "NO_CAPACITY", error: noCapacityError() }, + { + code: "RUNTIME_MISSING", + error: new SimlockError("RUNTIME_MISSING", "domain", "no such runtime", { + downloadable: false, + osVersion: "26.0", + platform: "ios", + }), + }, + ])( + "terminally rejects, rather than re-queuing or asking another worker, a $code that arrives after progress already fired (P1, round 2 review)", + async ({ code, error }) => { + const { coordinator, directory, eventBus, workers } = buildFleet(); + const client = new ScriptedWorkerClient(); + const other = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + directory.add("wrk_b", other); + connectWorker(workers, "wrk_a"); + connectWorker(workers, "wrk_b"); + client.requestLeaseQueue.push({ + error, + kind: "error", + progress: [{ etaMs: 5_000, stage: "reclaiming" }], + }); + other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + const dispatched: unknown[] = []; + eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); + + const rejection = await coordinator + .request(REQUEST, requestOptions()) + .catch((caught: unknown) => caught); + + expect(dispatched).toHaveLength(1); + expect(rejection).toBeInstanceOf(DispatchError); + expect((rejection as DispatchError).code).toBe(code); + expect(coordinator.queueDepth).toBe(0); + expect(directory.refreshCalls).toEqual([]); + expect(other.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); + }, + ); +}); diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index ffdc647da..77d8fba0b 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -1,17 +1,12 @@ import { describe, expect, it } from "vitest"; import type { EventBus, EventMap } from "../bus/index.js"; -import type { LeaseRecord } from "../admin/index.js"; import { SimlockError } from "../contract/index.js"; import { DispatchError } from "../daemon/dispatch.js"; -import { promiseState } from "../test-support/promise-state.js"; -import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; import { RequesterAlreadyLeasedError } from "./queue.js"; -import { createRoutingPolicy } from "./index.js"; import { deviceFixture, grantFixture, leaseFixture, - noCapacityError, ScriptedWorkerClient, statusFixture, } from "./test-support.js"; @@ -152,1468 +147,6 @@ describe("FleetLeaseCoordinator stored requests", () => { }); }); -describe("FleetLeaseCoordinator dispatch", () => { - it("grants a queued request from whichever worker frees first -- never dispatching to the one routing did not prefer", async () => { - const { coordinator, directory, workers } = buildFleet(); - const clientA = new ScriptedWorkerClient(); - const clientB = new ScriptedWorkerClient(); - directory.add("wrk_a", clientA); - directory.add("wrk_b", clientB); - const noFree = { ...statusFixture().capacity.android, maxRunning: 0, running: 0 }; - connectWorker(workers, "wrk_a", { - capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5, running: 5 } }, - }); - connectWorker(workers, "wrk_b", { - capacity: { ...statusFixture().capacity, ios: noFree }, - }); - - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - clientB.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - workers.refresh("wrk_b", { - capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5, running: 0 } }, - }); - - const grant = await grantPromise; - expect(grant.lease.worker?.id).toBe("wrk_b"); - expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - expect(clientB.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - }); - - it("issues exactly one lease.request for a waiter still in flight, even when another worker's view changes mid-flight", async () => { - const { coordinator, directory, workers } = buildFleet(); - const clientA = new ScriptedWorkerClient(); - const clientB = new ScriptedWorkerClient(); - directory.add("wrk_a", clientA); - directory.add("wrk_b", clientB); - clientA.requestLeaseQueue.push({ kind: "hang" }); - - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - void coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - workers.refresh("wrk_a", { capacity: statusFixture().capacity }); - await tick(); - expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - - connectWorker(workers, "wrk_b", { - capacity: { - ...statusFixture().capacity, - ios: { ...statusFixture().capacity.ios, maxRunning: 10 }, - }, - }); - await tick(); - - expect(clientA.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - expect(clientB.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - }); - - it("dispatches at most one queued waiter per worker per pass, rather than sending every queued waiter at a single over-reporting worker at once (H6, round 3 review)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - const p1 = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), - ); - const p2 = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), - ); - const p3 = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-3", requesterId: "agent-3" }), - ); - await tick(); - expect(coordinator.queueDepth).toBe(3); - - workers.refresh("wrk_a", { capacity: statusFixture().capacity }); - await tick(); - - const leaseRequests = client.calls.filter((call) => call.startsWith("lease.request")); - expect(leaseRequests).toHaveLength(2); - expect(leaseRequests[1]).toContain("agent-2"); - - await expect(p1).resolves.toBeDefined(); - const outcome = await Promise.race([ - Promise.allSettled([p2, p3]).then(() => "settled" as const), - tick(6).then(() => "still-pending" as const), - ]); - expect(outcome).toBe("still-pending"); - expect(coordinator.queueDepth).toBe(2); - expect(directory.refreshCalls).toEqual(["wrk_a", "wrk_a"]); - }); - - it("gives a waiter passed over by the per-pass cap a guaranteed next look once the claiming attempt settles terminally, even when that failure produces no worker-side event at all (C1, round 3 review)", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ leaseRequestTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - client.requestLeaseQueue.push({ kind: "hang" }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - const w1Request = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), - ); - const w1State = promiseState(w1Request); - const w1 = w1Request.catch((error: unknown) => error); - const w2 = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), - ); - await tick(); - expect(coordinator.queueDepth).toBe(2); - - workers.refresh("wrk_a", { capacity: statusFixture().capacity }); - await tick(); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - - clock.advance(5_000); - await tick(); - expect(w1State.state).toBe("rejected"); - const w1Error = await w1; - expect(w1Error).toBeInstanceOf(DispatchError); - expect((w1Error as DispatchError).code).toBe("WORKER_UNREACHABLE"); - - const outcome = await Promise.race([ - w2.then( - () => "resolved" as const, - () => "resolved" as const, - ), - tick(6).then(() => "still-pending" as const), - ]); - expect(outcome).toBe("resolved"); - const w2Grant = await w2; - expect(w2Grant.lease.worker?.id).toBe("wrk_a"); - expect(directory.refreshCalls).toEqual(["wrk_a"]); - }); - - it("re-dispatches a waiter whose attempt found a reachable-but-not-yet-connected target, instead of parking it forever", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - directory.reachableButNoClient.add("wrk_a"); - - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - expect(coordinator.queueDepth).toBe(1); - - directory.reachableButNoClient.delete("wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - workers.refresh("wrk_a", {}); // any refresh re-runs dispatch, exactly as a real one would - - const grant = await grantPromise; - expect(grant.lease.worker?.id).toBe("wrk_a"); - expect(coordinator.queueDepth).toBe(0); - }); - - it("leaves a request queued after NO_CAPACITY, refreshes that worker's view, and tries it once more when a refresh changes its capacity", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ error: noCapacityError(), kind: "error" }); - - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - - expect(coordinator.queueDepth).toBe(1); - expect(directory.refreshCalls).toEqual(["wrk_a"]); - - const capacity = statusFixture().capacity; - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - workers.refresh("wrk_a", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 3 } }, - }); - - const grant = await grantPromise; - expect(grant.lease.worker?.id).toBe("wrk_a"); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(2); - }); - - it("sends a worker that keeps answering NO_CAPACITY with an unchanged view one lease.request for that request", async () => { - const { clock, coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - - void coordinator.request(REQUEST, requestOptions()); - await tick(); - for (let refresh = 0; refresh < 3; refresh += 1) { - clock.advance(1_000); // a new lastSeenAt on every refresh, as a real one has - workers.refresh("wrk_a", { capacity: statusFixture().capacity }); - await tick(); - } - - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - expect(coordinator.queueDepth).toBe(1); - }); - - it("tries another eligible worker in the pass that follows a refusal", async () => { - const { coordinator, directory, workers } = buildFleet(); - const preferred = new ScriptedWorkerClient(); - const other = new ScriptedWorkerClient(); - directory.add("wrk_a", preferred); - directory.add("wrk_b", other); - const capacity = statusFixture().capacity; - connectWorker(workers, "wrk_a", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, - }); - connectWorker(workers, "wrk_b"); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(other.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - - workers.refresh("wrk_a", {}); - - const grant = await grantPromise; - expect(grant.lease.worker?.id).toBe("wrk_b"); - expect(preferred.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - }); - - it("does not exclude a worker for another request because it refused one", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - - void coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), - ); - - expect(grant.lease.worker?.id).toBe("wrk_a"); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toEqual([ - `lease.request:${GATEWAY_PREFIX}agent-1`, - `lease.request:${GATEWAY_PREFIX}agent-2`, - ]); - expect(coordinator.queueDepth).toBe(1); - }); - - it("leaves a request queued when the worker that refused it left the views before the refusal arrived", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - const target = directory.target.bind(directory); - directory.target = (workerId) => { - const found = target(workerId); - if (found === undefined) return undefined; - return { - ...found, - client: () => { - workers.disconnected(workerId); - void workers.remove(workerId); - return found.client(); - }, - }; - }; - - void coordinator.request(REQUEST, requestOptions()); - await tick(); - - expect(workers.views()).toEqual([]); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - expect(coordinator.queueDepth).toBe(1); - }); - - it("sends a queued noWait request refused by one worker to only one other worker in the next walk", async () => { - const { coordinator, directory, workers } = buildFleet(); - const first = new ScriptedWorkerClient(); - const second = new ScriptedWorkerClient(); - const third = new ScriptedWorkerClient(); - directory.add("wrk_a", first); - directory.add("wrk_b", second); - directory.add("wrk_c", third); - const capacity = statusFixture().capacity; - connectWorker(workers, "wrk_a", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 6 } }, - }); - connectWorker(workers, "wrk_b", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, - }); - connectWorker(workers, "wrk_c", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, - }); - directory.reachableButNoClient.add("wrk_a"); - second.requestLeaseQueue.push({ kind: "hang" }); - third.requestLeaseQueue.push({ kind: "hang" }); - - void coordinator.request(REQUEST, requestOptions({ noWait: true })); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - directory.reachableButNoClient.delete("wrk_a"); - workers.refresh("wrk_a", {}); - await tick(); - - const leaseRequests = (client: ScriptedWorkerClient) => - client.calls.filter((call) => call.startsWith("lease.request")).length; - expect(leaseRequests(first)).toBe(1); - expect(leaseRequests(second) + leaseRequests(third)).toBe(1); - }); - - it("grants a noWait request refused by one worker from another in the next walk", async () => { - const { coordinator, directory, workers } = buildFleet(); - const preferred = new ScriptedWorkerClient(); - const other = new ScriptedWorkerClient(); - directory.add("wrk_a", preferred); - directory.add("wrk_b", other); - const capacity = statusFixture().capacity; - connectWorker(workers, "wrk_a", { - capacity: { ...capacity, ios: { ...capacity.ios, maxRunning: 4 } }, - }); - connectWorker(workers, "wrk_b"); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await coordinator.request(REQUEST, requestOptions({ noWait: true })); - - expect(grant.lease.worker?.id).toBe("wrk_b"); - expect(preferred.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - }); - - it.each([ - { code: "NO_CAPACITY", error: noCapacityError() }, - { - code: "RUNTIME_MISSING", - error: new SimlockError("RUNTIME_MISSING", "domain", "no such runtime", { - downloadable: false, - osVersion: "26.0", - platform: "ios", - }), - }, - ])( - "terminally rejects, rather than re-queuing or asking another worker, a $code that arrives after progress already fired (P1, round 2 review)", - async ({ code, error }) => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - const other = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - directory.add("wrk_b", other); - connectWorker(workers, "wrk_a"); - connectWorker(workers, "wrk_b"); - client.requestLeaseQueue.push({ - error, - kind: "error", - progress: [{ etaMs: 5_000, stage: "reclaiming" }], - }); - other.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - const rejection = await coordinator - .request(REQUEST, requestOptions()) - .catch((caught: unknown) => caught); - - expect(dispatched).toHaveLength(1); - expect(rejection).toBeInstanceOf(DispatchError); - expect((rejection as DispatchError).code).toBe(code); - expect(coordinator.queueDepth).toBe(0); - expect(directory.refreshCalls).toEqual([]); - expect(other.calls.filter((call) => call.startsWith("lease.request"))).toEqual([]); - }, - ); - - it("bounds a forwarded lease.request against gateway.leaseRequestTimeoutMs when the worker never answers at all (P2, round 2 review)", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ leaseRequestTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ kind: "hang" }); - - const request = coordinator.request(REQUEST, requestOptions()); - const requestState = promiseState(request); - const rejection = request.catch((error: unknown) => error); - await tick(); - expect(coordinator.queueDepth).toBe(0); - expect(directory.refreshCalls).toEqual([]); - - clock.advance(4_999); - await tick(); - expect(requestState.state).toBe("pending"); - - clock.advance(1); - await tick(); - expect(requestState.state).toBe("rejected"); - const error = await rejection; - - expect(error).toBeInstanceOf(DispatchError); - expect((error as DispatchError).code).toBe("WORKER_UNREACHABLE"); - expect(coordinator.queueDepth).toBe(0); - }); - - it("does not emit request.dispatched for a progress push that arrives after gateway.leaseRequestTimeoutMs already rejected the waiter (P4, round 3 review)", async () => { - const { clock, coordinator, directory, eventBus, workers } = buildFleet({ - leaseRequestTimeoutMs: 5_000, - }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ kind: "hang" }); - const events: string[] = []; - eventBus.subscribe("request.dispatched", () => events.push("request.dispatched")); - eventBus.subscribe("lease.rejected", () => events.push("lease.rejected")); - - const request = coordinator.request(REQUEST, requestOptions()); - const requestState = promiseState(request); - const rejection = request.catch((error: unknown) => error); - await tick(); - - clock.advance(5_000); - await tick(); - expect(requestState.state).toBe("rejected"); - const error = await rejection; - expect((error as DispatchError).code).toBe("WORKER_UNREACHABLE"); - expect(events).toEqual(["lease.rejected"]); - - client.lastRequestLeaseOptions?.onProgress?.({ etaMs: 5_000, stage: "provisioning" }); - await tick(); - - expect(events).toEqual(["lease.rejected"]); - }); - - it("passes over a request only a busy worker can serve instead of blocking behind it, and never lets a brand-new admission jump an older, equally-eligible waiter still queued behind it (C2, round 3 review)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_ios", client); - directory.add("wrk_android", new ScriptedWorkerClient()); - client.requestLeaseQueue.push({ kind: "hang" }); - - const capacity = statusFixture().capacity; - connectWorker(workers, "wrk_ios", { capacity: saturatedIos() }); - connectWorker(workers, "wrk_android", { - capacity: { ...capacity, android: { ...capacity.android, maxRunning: 0 } }, - models: ["Pixel 9"], - platform: "android", - }); - const androidPromise = coordinator.request( - { model: "Pixel 9", platform: "android" }, - requestOptions({ ownerId: "agent-android", requesterId: "agent-android" }), - ); - const oldFirstPromise = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-old-1", requesterId: "agent-old-1" }), - ); - const oldSecondPromise = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-old-2", requesterId: "agent-old-2" }), - ); - await tick(); - expect(coordinator.queueDepth).toBe(3); - - workers.refresh("wrk_ios", { capacity: statusFixture().capacity }); - await tick(); - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - - let androidSettled = false; - void androidPromise.then( - () => { - androidSettled = true; - }, - () => { - androidSettled = true; - }, - ); - await tick(); - expect(androidSettled).toBe(false); - - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const newPromise = coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-new", requesterId: "agent-new" }), - ); - await tick(); - - const leaseRequests = client.calls.filter((call) => call.startsWith("lease.request")); - expect(leaseRequests.length).toBeGreaterThanOrEqual(2); - expect(leaseRequests[1]).toContain("agent-old-2"); - - const oldSecondGrant = await oldSecondPromise; - expect(oldSecondGrant.lease.worker?.id).toBe("wrk_ios"); - expect(androidSettled).toBe(false); - void oldFirstPromise; - void newPromise; - }); - - it("indexes a fresh grant under the ownerId it forwarded, not the worker's own echo of it (H6)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { - ...grantFixture().lease, - ownerId: "worker-rewrote-this", - }, - }), - kind: "grant", - }); - - const grant = await coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-1", requesterId: "agent-1" }), - ); - - expect(coordinator.ownerId(grant.lease.id)).toBe("agent-1"); - expect(grant.lease.ownerId).toBe("agent-1"); - }); - - it("emits request.dispatched on the worker's first progress push even when the request never settles at all (§11, C4 round 2 review)", async () => { - const { coordinator, eventBus, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - kind: "hang", - progress: [{ etaMs: 5_000, stage: "provisioning" }], - }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - let settled = false; - void coordinator.request(REQUEST, requestOptions()).then( - () => { - settled = true; - }, - () => { - settled = true; - }, - ); - await tick(); - - expect(settled).toBe(false); - expect(dispatched).toEqual([ - { - model: "iPhone 17", - platform: "ios", - queuedMs: expect.any(Number) as number, - reason: "free-capacity", - requestId: expect.any(String) as string, - requesterId: "agent-1", - stage: "free-capacity", - workerId: "wrk_a", - }, - ]); - }); - - it("emits request.dispatched exactly once, on the worker's first progress push, not again when the grant lands afterward (§11)", async () => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture(), - kind: "grant", - progress: [ - { etaMs: 5_000, stage: "provisioning" }, - { etaMs: 1_000, stage: "booting" }, - ], - }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - const grant = await coordinator.request(REQUEST, requestOptions()); - - expect(dispatched).toEqual([ - { - model: "iPhone 17", - platform: "ios", - queuedMs: expect.any(Number) as number, - reason: "free-capacity", - requestId: expect.any(String) as string, - requesterId: "agent-1", - stage: "free-capacity", - workerId: "wrk_a", - }, - ]); - expect(grant.lease.worker?.id).toBe("wrk_a"); - }); - - it.each(["full", "slim"] as const)( - "request.dispatched carries the requested mode %s", - async (mode) => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture(), - kind: "grant", - progress: [{ etaMs: 5_000, stage: "provisioning" }], - }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - await coordinator.request({ ...REQUEST, mode }, requestOptions()); - - expect(dispatched).toEqual([expect.objectContaining({ mode })]); - }, - ); - - it("request.dispatched carries no mode and no physical key for a plain virtual request", async () => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const dispatched: object[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - await coordinator.request(REQUEST, requestOptions()); - - expect(dispatched).toHaveLength(1); - expect(dispatched[0]).not.toHaveProperty("mode"); - expect(dispatched[0]).not.toHaveProperty("physical"); - }); - - it("relays a worker's downloading pushes to the requester unchanged", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - const pushed = [ - { component: "26.4", stage: "downloading", waiting: true }, - { component: "26.4", stage: "downloading", waiting: false }, - { component: "26.4", percent: 41, stage: "downloading", waiting: false }, - { etaMs: 5_000, stage: "provisioning" }, - ] as const; - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant", progress: pushed }); - const heard: unknown[] = []; - - await coordinator.request( - REQUEST, - requestOptions({ allowDownload: true, onProgress: (progress) => heard.push(progress) }), - ); - - expect(heard).toEqual(pushed); - }); - - it("emits request.dispatched when a grant lands with no progress push at all -- the ADR §11 rule's other half", async () => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - await coordinator.request(REQUEST, requestOptions()); - - expect(dispatched).toEqual([ - { - model: "iPhone 17", - platform: "ios", - queuedMs: expect.any(Number) as number, - reason: "free-capacity", - requestId: expect.any(String) as string, - requesterId: "agent-1", - stage: "free-capacity", - workerId: "wrk_a", - }, - ]); - }); - - it("reports request.dispatched's reason as warm-hit when routing picked the worker for an unleased ready device, and queuedMs as the real wait (C2, round 2 review)", async () => { - const { clock, coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - const noFree = { ...statusFixture().capacity.ios, maxRunning: 0, running: 0 }; - connectWorker(workers, "wrk_a", { capacity: { ...statusFixture().capacity, ios: noFree } }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - clock.advance(2_500); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - workers.refresh("wrk_a", { - capacity: { ...statusFixture().capacity, ios: { ...noFree, maxRunning: 5 } }, - devices: [deviceFixture("dev_1", "ready")], - }); - - await grantPromise; - - expect(dispatched).toEqual([ - { - model: "iPhone 17", - platform: "ios", - queuedMs: 2_500, - reason: "warm-hit", - requestId: expect.any(String) as string, - requesterId: "agent-1", - stage: "warm-hit", - workerId: "wrk_a", - }, - ]); - }); - - it("request.dispatched carries the routing decision's stage name and reason", async () => { - const { coordinator, directory, eventBus, workers } = buildFleet({ - routing: () => ({ - assess: () => ({ kind: "route-or-wait" }), - explain: () => [], - select: (_request, views) => - views[0] === undefined - ? undefined - : { reason: "warm-hit", stage: "test-rank", workerId: views[0].id }, - }), - }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - await coordinator.request(REQUEST, requestOptions()); - - expect(dispatched).toEqual([ - expect.objectContaining({ reason: "warm-hit", stage: "test-rank", workerId: "wrk_a" }), - ]); - }); - - it("does not emit request.dispatched for a stale-view NO_CAPACITY -- the request stayed queued, never actually dispatched", async () => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ error: noCapacityError(), kind: "error" }); - const dispatched: unknown[] = []; - eventBus.subscribe("request.dispatched", (envelope) => dispatched.push(envelope.payload)); - - void coordinator.request(REQUEST, requestOptions()); - await tick(); - - expect(dispatched).toEqual([]); - expect(coordinator.queueDepth).toBe(1); - }); - - it("handles a worker's DAEMON_UPDATING answer as NO_CAPACITY: the view is read again, the request stays queued and is placed once the view changes", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - error: new SimlockError("DAEMON_UPDATING", "domain", "The daemon is updating", {}), - kind: "error", - }); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const outcome = coordinator.request(REQUEST, requestOptions()); - const state = promiseState(outcome); - await tick(); - - expect(state.state).toBe("pending"); - expect(coordinator.queueDepth).toBe(1); - expect(directory.refreshCalls).toEqual(["wrk_a"]); - - workers.refresh("wrk_a", { - update: { forced: false, from: "0.3.0", stage: "installing", startedAt: 1, to: "0.4.0" }, - }); - await tick(); - expect(state.state).toBe("pending"); - - workers.refresh("wrk_a", { update: null }); - await tick(); - - expect((await outcome).lease.worker?.id).toBe("wrk_a"); - }); - - it("forwards device.exec with the namespaced requesterId reaching the scripted worker client", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { - ...grantFixture().lease, - id: "lse_9", - ownerId: "agent-1", - requesterId: `${GATEWAY_PREFIX}agent-1`, - }, - }), - kind: "grant", - }); - - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ exitCode: 0, kind: "ok" }); - - await coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent-1", - role: "agent", - }, - ); - - expect(client.calls).toContain(`device.exec:${GATEWAY_PREFIX}agent-1`); - }); - - it("announces started from the worker's own push, before any output (ADR §19a)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ - exitCode: 0, - kind: "ok", - output: [{ chunk: "hi", stream: "stdout" }], - started: true, - }); - - const order: string[] = []; - await coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onOutput: () => { - order.push("output"); - }, - onStarted: () => { - order.push("started"); - }, - principal: "agent-1", - role: "agent", - }, - ); - - expect(order).toEqual(["started", "output"]); - expect(client.calls).toContain(`device.exec:${GATEWAY_PREFIX}agent-1`); - }); - - it.each([ - ["an empty queue", 0], - ["another request queued", 1], - ] as const)( - "rejects a noWait request refused by its only worker with NO_CAPACITY and lease.rejected no-wait, refreshes that worker's view, and never queues it (no lease.queued, no queued progress), with %s -- the answer does not depend on queue depth (ADR §10, architecture rule 10)", - async (_queue, queued) => { - const { coordinator, directory, eventBus, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - if (queued > 0) { - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - await coordinator.request(REQUEST, requestOptions()); - void coordinator.request( - REQUEST, - requestOptions({ ownerId: "agent-2", requesterId: "agent-2" }), - ); - await tick(); - } - expect(coordinator.queueDepth).toBe(queued); - const rejected: unknown[] = []; - eventBus.subscribe("lease.rejected", (envelope) => rejected.push(envelope.payload)); - const events: string[] = []; - eventBus.subscribe("lease.queued", () => events.push("lease.queued")); - eventBus.subscribe("lease.rejected", () => events.push("lease.rejected")); - const progress: string[] = []; - const refreshesBefore = directory.refreshCalls.length; - - const refused = coordinator.request( - REQUEST, - requestOptions({ - noWait: true, - onProgress: (update) => progress.push(update.stage), - ownerId: "agent-3", - requesterId: "agent-3", - }), - ); - const refusedState = promiseState(refused); - await tick(); - - expect(refusedState.state).toBe("rejected"); - await expect(refused).rejects.toMatchObject({ code: "NO_CAPACITY" }); - expect(client.calls).toContain(`lease.request:${GATEWAY_PREFIX}agent-3`); - expect(rejected).toEqual([expect.objectContaining({ reason: "no-wait" })]); - expect(events).toEqual(["lease.rejected"]); - expect(progress).toEqual([]); - expect(directory.refreshCalls.slice(refreshesBefore)).toEqual(["wrk_a"]); - expect(coordinator.queueDepth).toBe(queued); - }, - ); - - it("relays the worker's own started push for a silent command that never answers (ADR §19a/§19b/§19e)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ kind: "hang", started: true }); - - let started = false; - void coordinator.exec( - { args: ["install", "/path/to.app"], leaseId: grant.lease.id, tool: "simctl" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onStarted: () => { - started = true; - }, - principal: "agent-1", - role: "agent", - }, - ); - await tick(); - - expect(started).toBe(true); - }); - - it("does not announce started for a command the worker refuses before any process exists -- a pre-process refusal keeps its own real status (ADR §19a′)", async () => { - const { clock, coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ - error: new SimlockError("FORBIDDEN", "domain", "Lease belongs to a different requester", {}), - kind: "error", - }); - - let started = false; - const rejection = await coordinator - .exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onStarted: () => { - started = true; - }, - principal: "agent-1", - role: "agent", - }, - ) - .catch((error: unknown) => error); - - expect(rejection).toBeInstanceOf(DispatchError); - expect((rejection as DispatchError).code).toBe("FORBIDDEN"); - expect(started).toBe(false); - - clock.advance(10_000); - expect(started).toBe(false); - }); - - it("times out a forwarded device.exec after gateway.execTimeoutMs when the worker never answers at all (P5)", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { - ...grantFixture().lease, - id: "lse_9", - ownerId: "agent-1", - requesterId: `${GATEWAY_PREFIX}agent-1`, - }, - }), - kind: "grant", - }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ kind: "hang" }); - - const execution = coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent-1", - role: "agent", - }, - ); - const executionState = promiseState(execution); - const rejection = execution.catch((error: unknown) => error); - await tick(); - clock.advance(4_999); - await tick(); - expect(executionState.state).toBe("pending"); - - clock.advance(1); - await tick(); - expect(executionState.state).toBe("rejected"); - const error = await rejection; - expect(error).toBeInstanceOf(DispatchError); - expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); - }); - - it("times out a forwarded device.rotate after gateway.execTimeoutMs when the worker never answers", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.hangingCalls.add("device.rotate"); - - const rotation = coordinator.rotate( - { leaseId: grant.lease.id, orientation: "landscape_left" }, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent-1", - role: "agent", - }, - ); - const rotationState = promiseState(rotation); - const rejection = rotation.catch((error: unknown) => error); - await tick(); - clock.advance(4_999); - await tick(); - expect(rotationState.state).toBe("pending"); - - clock.advance(1); - await tick(); - expect(rotationState.state).toBe("rejected"); - const error = await rejection; - expect(error).toBeInstanceOf(DispatchError); - expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); - }); - - it("stops relaying a worker's device.exec output once gateway.execTimeoutMs has already rejected the call (H4, round 3 review)", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ - grant: grantFixture({ - lease: { - ...grantFixture().lease, - id: "lse_9", - ownerId: "agent-1", - requesterId: `${GATEWAY_PREFIX}agent-1`, - }, - }), - kind: "grant", - }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ kind: "hang" }); - - const relayed: Array<{ stream: string; chunk: string }> = []; - const execution = coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onOutput: (stream, chunk) => { - relayed.push({ chunk, stream }); - }, - principal: "agent-1", - role: "agent", - }, - ); - const executionState = promiseState(execution); - const rejection = execution.catch((error: unknown) => error); - await tick(); - clock.advance(5_000); - await tick(); - expect(executionState.state).toBe("rejected"); - const error = await rejection; - expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); - - client.lastExecOptions?.onOutput?.({ chunk: "late output", stream: "stdout" }); - await tick(); - - expect(relayed).toEqual([]); - }); - - it("does not announce started from a worker frame that arrives after gateway.execTimeoutMs already rejected the call (H4, round 6 review)", async () => { - const { clock, coordinator, directory, workers } = buildFleet({ execTimeoutMs: 5_000 }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ kind: "hang" }); - - let startedCount = 0; - const execution = coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onStarted: () => { - startedCount += 1; - }, - principal: "agent-1", - role: "agent", - }, - ); - const executionState = promiseState(execution); - const rejection = execution.catch((error: unknown) => error); - await tick(); - clock.advance(5_000); - await tick(); - expect(executionState.state).toBe("rejected"); - const error = await rejection; - expect((error as DispatchError).code).toBe("EXEC_TIMEOUT"); - - client.lastExecOptions?.onStarted?.(); - await tick(); - - expect(startedCount).toBe(0); - }); - - it("announces started once even if the worker sends the frame more than once", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const grant = await coordinator.request(REQUEST, requestOptions()); - client.execQueue.push({ exitCode: 0, kind: "ok", started: true }); - - let startedCount = 0; - await coordinator.exec( - { args: ["devices"], leaseId: grant.lease.id, tool: "adb" }, - { - manageEventSubscription: () => undefined, - via: "socket", - onStarted: () => { - startedCount += 1; - }, - principal: "agent-1", - role: "agent", - }, - ); - client.lastExecOptions?.onStarted?.(); - await tick(); - - expect(startedCount).toBe(1); - }); - - it("runs another pass for a view change raised from inside the walk, rather than dropping it (round 4 finding 6, round 6 review)", async () => { - let selectCalls = 0; - const { coordinator, directory, workers } = buildFleet({ - routing: (registry) => { - const real = createRoutingPolicy("warm-then-free"); - return { - assess: real.assess, - explain: real.explain, - select(request, workerViews) { - selectCalls += 1; - if (selectCalls === 2) { - registry.refresh("wrk_a", {}); - return undefined; - } - return real.select(request, workerViews); - }, - }; - }, - }); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - connectWorker(workers, "wrk_a", { capacity: saturatedIos() }); - const grantPromise = coordinator.request(REQUEST, requestOptions()); - await tick(); - expect(coordinator.queueDepth).toBe(1); - - workers.refresh("wrk_a", { capacity: statusFixture().capacity }); - await tick(); - - expect(client.calls.filter((call) => call.startsWith("lease.request"))).toHaveLength(1); - expect(selectCalls).toBe(3); - const grant = await grantPromise; - expect(grant.lease.worker?.id).toBe("wrk_a"); - expect(coordinator.queueDepth).toBe(0); - }); - - it.each([ - { - code: "WORKER_UNREACHABLE", - details: { workerId: "wrk_a" }, - error: new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - "Daemon connection is closed", - {}, - ), - row: "a lost connection", - }, - { - code: "WORKER_UNREACHABLE", - details: { workerId: "wrk_a" }, - error: new SimlockError("DAEMON_STOPPING", "transport", "The daemon is stopping", {}), - row: "a stopping daemon (#514)", - }, - { - code: "INTERNAL", - details: undefined, - error: new TypeError("boom"), - row: "a non-SimlockError (H1, round 2 review)", - }, - ])( - "maps $row on a forwarded lease.request to $code, never the uplink client's own error code verbatim (P4)", - async ({ code, details, error }) => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ error, kind: "error" }); - - const rejection = await coordinator - .request(REQUEST, requestOptions()) - .catch((caught: unknown) => caught); - expect(rejection).toBeInstanceOf(DispatchError); - expect(rejection).toMatchObject({ code, ...(details === undefined ? {} : { details }) }); - }, - ); - - it.each([ - { - code: "WORKER_UNREACHABLE", - error: new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - "Daemon connection is closed", - {}, - ), - row: "a transport failure", - }, - { - code: "INTERNAL", - error: new TypeError("boom"), - row: "a non-SimlockError (H1, round 2 review)", - }, - ])( - "maps $row on a forwarded lease.renew to $code, never the other (P4)", - async ({ code, error }) => { - const { coordinator, directory, workers, leaseIndex } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - leaseIndex.add({ - gatewayLeaseId: "wrk_a.lse_1", - grantedAt: 1, - ownerId: "agent-1", - requesterId: "agent-1", - workerId: "wrk_a", - workerLeaseId: "lse_1", - }); - client.renewLeaseQueue.push({ error, kind: "error" }); - - const rejection = await coordinator - .renew("wrk_a.lse_1", undefined) - .catch((caught: unknown) => caught); - expect(rejection).toBeInstanceOf(DispatchError); - expect((rejection as DispatchError).code).toBe(code); - }, - ); - - it.each([ - { - answer: (client: ScriptedWorkerClient, error: SimlockError<"UNKNOWN_LEASE">) => - client.releaseLeaseQueue.push({ error, kind: "error" }), - call: (coordinator: FleetLeaseCoordinator) => coordinator.release("wrk_a.lse_1"), - verb: "release", - }, - { - answer: (client: ScriptedWorkerClient, error: SimlockError<"UNKNOWN_LEASE">) => - client.renewLeaseQueue.push({ error, kind: "error" }), - call: (coordinator: FleetLeaseCoordinator) => coordinator.renew("wrk_a.lse_1", undefined), - verb: "renew", - }, - ])( - "drops its own index entry when the worker answers UNKNOWN_LEASE to a $verb, instead of leaving a zombie behind (C3)", - async ({ answer, call }) => { - const { coordinator, directory, workers, leaseIndex } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - leaseIndex.add({ - gatewayLeaseId: "wrk_a.lse_1", - grantedAt: 1, - ownerId: "agent-1", - requesterId: "agent-1", - workerId: "wrk_a", - workerLeaseId: "lse_1", - }); - answer( - client, - new SimlockError("UNKNOWN_LEASE", "domain", "Unknown lease: lse_1", { leaseId: "lse_1" }), - ); - - await expect(call(coordinator)).rejects.toMatchObject({ code: "UNKNOWN_LEASE" }); - - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - await expect(coordinator.request(REQUEST, requestOptions())).resolves.toBeDefined(); - }, - ); - - it("release-all treats a worker's UNKNOWN_LEASE as already-released rather than a failure that blocks every future call (C3)", async () => { - const { coordinator, directory, workers, leaseIndex } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - leaseIndex.add({ - gatewayLeaseId: "wrk_a.lse_zombie", - grantedAt: 1, - ownerId: "agent-1", - requesterId: "agent-1", - workerId: "wrk_a", - workerLeaseId: "lse_zombie", - }); - client.releaseLeaseQueue.push({ - error: new SimlockError("UNKNOWN_LEASE", "domain", "Unknown lease: lse_zombie", { - leaseId: "lse_zombie", - }), - kind: "error", - }); - - await expect(coordinator.releaseAll()).resolves.toEqual(["wrk_a.lse_zombie"]); - await expect(coordinator.releaseAll()).resolves.toEqual([]); - }); - - it("does not evict a lease on one worker when only a different worker's view changes repeatedly (C1, round 2 review)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const clientA = new ScriptedWorkerClient(); - const clientB = new ScriptedWorkerClient(); - directory.add("wrk_a", clientA); - directory.add("wrk_b", clientB); - connectWorker(workers, "wrk_a"); - connectWorker(workers, "wrk_b"); - clientA.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - - const grant = await coordinator.request(REQUEST, requestOptions()); - expect(grant.lease.worker?.id).toBe("wrk_a"); - expect(workers.view("wrk_a")?.leases).toEqual([]); - - workers.refresh("wrk_b", {}); - workers.refresh("wrk_b", {}); - - const retry = await coordinator - .request(REQUEST, requestOptions({ noWait: true })) - .catch((error: unknown) => error); - expect(retry).toBeInstanceOf(RequesterAlreadyLeasedError); - expect((retry as RequesterAlreadyLeasedError).existingLeaseId).toBe(grant.lease.id); - }); - - it("grants a requester's next lease when a worker view read before its release lands after it (#898)", async () => { - const { coordinator, directory, workers } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const first = await coordinator.request(REQUEST, requestOptions()); - client.releaseLeaseQueue.push({ kind: "ok" }); - await coordinator.release(first.lease.id); - - workers.refresh("wrk_a", { - leases: [{ ...leaseFixture("lse_1", "dev_1"), requesterId: `${GATEWAY_PREFIX}agent-1` }], - }); - workers.refresh("wrk_a", { leases: [] }); - - client.requestLeaseQueue.push({ - grant: grantFixture({ lease: leaseFixture("lse_2", "dev_1") as LeaseRecord }), - kind: "grant", - }); - const second = await coordinator - .request(REQUEST, requestOptions({ noWait: true })) - .catch((error: unknown) => error); - expect(second).toMatchObject({ lease: { id: "wrk_a.lse_2" } }); - }); - - it.each(["release", "releaseAll"] as const)( - "grants the next lease when %s's relayed event came before the worker's answer and a stale view follows (#898)", - async (how) => { - const { coordinator, directory, workers, leaseIndex } = buildFleet(); - const client = new ScriptedWorkerClient(); - directory.add("wrk_a", client); - connectWorker(workers, "wrk_a"); - client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); - const first = await coordinator.request(REQUEST, requestOptions()); - client.onReleaseLease = (leaseId) => leaseIndex.removeByWorkerLease("wrk_a", leaseId); - if (how === "release") await coordinator.release(first.lease.id); - else await coordinator.releaseAll(); - - workers.refresh("wrk_a", { - leases: [{ ...leaseFixture("lse_1", "dev_1"), requesterId: `${GATEWAY_PREFIX}agent-1` }], - }); - workers.refresh("wrk_a", { leases: [] }); - - client.requestLeaseQueue.push({ - grant: grantFixture({ lease: leaseFixture("lse_2", "dev_1") as LeaseRecord }), - kind: "grant", - }); - const second = await coordinator - .request(REQUEST, requestOptions({ noWait: true })) - .catch((error: unknown) => error); - expect(second).toMatchObject({ lease: { id: "wrk_a.lse_2" } }); - }, - ); - - it("release-all attaches the leases it already released to a later failure's own details (H8)", async () => { - const { coordinator, directory, workers, leaseIndex } = buildFleet(); - const clientA = new ScriptedWorkerClient(); - const clientB = new ScriptedWorkerClient(); - directory.add("wrk_a", clientA); - directory.add("wrk_b", clientB); - connectWorker(workers, "wrk_a"); - connectWorker(workers, "wrk_b"); - leaseIndex.add({ - gatewayLeaseId: "wrk_a.lse_1", - grantedAt: 1, - ownerId: "agent-1", - requesterId: "agent-1", - workerId: "wrk_a", - workerLeaseId: "lse_1", - }); - leaseIndex.add({ - gatewayLeaseId: "wrk_b.lse_1", - grantedAt: 1, - ownerId: "agent-2", - requesterId: "agent-2", - workerId: "wrk_b", - workerLeaseId: "lse_1", - }); - clientA.releaseLeaseQueue.push({ kind: "ok" }); - directory.unreachable.add("wrk_b"); - - const rejection = await coordinator.releaseAll().catch((error: unknown) => error); - expect(rejection).toBeInstanceOf(DispatchError); - expect((rejection as DispatchError).code).toBe("WORKER_UNREACHABLE"); - expect((rejection as DispatchError).details).toMatchObject({ - releasedLeaseIds: ["wrk_a.lse_1"], - }); - await expect(coordinator.renew("wrk_a.lse_1", undefined)).rejects.toMatchObject({ - code: "UNKNOWN_LEASE", - }); - expect(clientA.calls.filter((call) => call.startsWith("lease.renew"))).toEqual([]); - await expect(coordinator.renew("wrk_b.lse_1", undefined)).rejects.toMatchObject({ - code: "WORKER_UNREACHABLE", - }); - }); -}); - describe("drain and unreachable lifecycle (ADR §9/§28/§29, #119)", () => { async function excludedWorkerKeepsLeaseButGetsNoNewDispatch( exclude: (workers: WorkerRegistry) => unknown, From 4eb22755915acff98dd28030bbf31dfbf4cbb13a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:30:29 +0100 Subject: [PATCH 43/83] test(core): split component-installer.test.ts by topic --- ...ent-installer-list-and-in-progress.test.ts | 343 ++++ src/core/component-installer-outcomes.test.ts | 234 +++ src/core/component-installer-remove.test.ts | 580 ++++++ ...onent-installer-timeouts-and-close.test.ts | 397 ++++ src/core/component-installer.test.ts | 1646 +---------------- src/core/test-component-installer.ts | 147 ++ 6 files changed, 1704 insertions(+), 1643 deletions(-) create mode 100644 src/core/component-installer-list-and-in-progress.test.ts create mode 100644 src/core/component-installer-outcomes.test.ts create mode 100644 src/core/component-installer-remove.test.ts create mode 100644 src/core/component-installer-timeouts-and-close.test.ts create mode 100644 src/core/test-component-installer.ts diff --git a/src/core/component-installer-list-and-in-progress.test.ts b/src/core/component-installer-list-and-in-progress.test.ts new file mode 100644 index 000000000..dcce9dce1 --- /dev/null +++ b/src/core/component-installer-list-and-in-progress.test.ts @@ -0,0 +1,343 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock, type Logger } from "../ports/index.js"; +import { ComponentInstallTimeoutError, type DriverComponent, DriverCrashError } from "./driver.js"; +import { FakeDriver } from "./fake-driver.js"; +import type { Registry } from "./registry.js"; +import { + gibibyte, + timeoutMs, + ManualClock, + createHarness, + installs, + flush, + track, + ios, +} from "./test-component-installer.js"; + +describe("ComponentInstaller.inProgress", () => { + it("lists an install at the front of its queue that is still checking whether it is needed as waiting", async () => { + const harness = await createHarness(); + + void harness.installer.install(ios("27.0", { stillNeeded: () => new Promise(() => {}) })); + await flush(); + + expect(harness.installer.inProgress()).toEqual([ + { component: "27.0", platform: "ios", since: 1_000, state: "waiting", waiters: 1 }, + ]); + expect(installs(harness.ios)).toEqual([]); + }); + + it("lists an install whose driver returned and whose record is being written as downloading", async () => { + const harness = await createHarness({ + // A gate that never opens: the record write waits behind it. + decisions: { run: () => new Promise(() => {}) }, + }); + + void harness.installer.install(ios("27.0")); + await flush(); + + expect(installs(harness.ios)).toEqual(["27.0"]); + expect(harness.installer.inProgress()).toEqual([ + { component: "27.0", platform: "ios", since: 1_000, state: "downloading", waiters: 1 }, + ]); + }); + + it("does not list a waiting install once its only call has timed out", async () => { + const harness = await createHarness({ clock: new ManualClock() }); + harness.ios.holdInstalls(); + + void harness.installer.install(ios("27.0")); + const waiting = track(harness.installer.install(ios("28.0"))); + await flush(); + expect(harness.installer.inProgress().map((install) => install.component)).toEqual([ + "27.0", + "28.0", + ]); + // Timer 1 is the budget of the call for 28.0. + (harness.clock as ManualClock).fire(1); + await flush(); + + expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); + expect(harness.installer.inProgress().map((install) => install.component)).toEqual(["27.0"]); + }); + + it("lists the 16 oldest installs across platforms, oldest first, when more are in progress", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }), + new FakeDriver({ availableOsVersions: [], clock, platform: "android" }), + ], + }); + for (const driver of harness.drivers) driver.holdInstalls(); + + // Alternating platforms, so the oldest are not the first ones of any one queue. + for (let index = 0; index < 20; index += 1) { + const platform = index % 2 === 0 ? "ios" : "android"; + void harness.installer.install({ component: `c${String(index)}`, platform }); + harness.clock.advance(1_000); + } + await flush(); + + expect(harness.installer.inProgress().map((install) => install.component)).toEqual( + Array.from({ length: 16 }, (_, index) => `c${String(index)}`), + ); + }); +}); + +/** A fake whose `listComponents` answers with exactly what a test lists. */ +class ListingDriver extends FakeDriver { + constructor( + clock: FakeClock, + platform: "ios" | "android", + private readonly listing: readonly DriverComponent[], + ) { + super({ availableOsVersions: [], clock, platform }); + } + + override listComponents(): Promise { + return Promise.resolve(this.listing); + } +} + +async function addDevice( + registry: Registry, + platform: "ios" | "android", + osVersion: string, +): Promise { + const driverDeviceId = `${platform}-${osVersion}-${String(registry.snapshot.devices.length)}`; + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: driverDeviceId }, + driverDeviceId, + provisionDuration: 0, + spec: { physical: false, model: "Phone", osVersion, platform }, + }); + return device.id; +} + +describe("ComponentInstaller.list", () => { + it("lists a component Simlock installed with installedBySimlock true and the time its record was written", async () => { + const harness = await createHarness({ + drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + harness.clock.advance(4_000); + + await harness.installer.install(ios("27.0")); + + expect(await harness.installer.list()).toEqual([ + { + devices: 0, + foreignDevices: 0, + installedAt: 5_000, + installedBySimlock: true, + platform: "ios", + version: "27.0", + }, + ]); + }); + + it("lists a component that was on the machine before with installedBySimlock false and no installedAt", async () => { + const harness = await createHarness(); + + expect(await harness.installer.list()).toEqual([ + { + devices: 0, + foreignDevices: 0, + installedBySimlock: false, + platform: "ios", + version: "26.5", + }, + ]); + }); + + it("marks nothing as Simlock's for a record of that platform and version whose receipt no longer equals the installed one", async () => { + const harness = await createHarness(); + await harness.registry.recordComponent({ + installedAt: 1_000, + platform: "ios", + receipt: { install: "1", version: "26.5" }, + version: "26.5", + }); + + const [listed] = await harness.installer.list(); + + expect(listed).toMatchObject({ installedBySimlock: false, version: "26.5" }); + expect(listed).not.toHaveProperty("installedAt"); + }); + + it("does not take a record of another platform with an equal receipt as proof", async () => { + const harness = await createHarness(); + await harness.registry.recordComponent({ + installedAt: 1_000, + platform: "android", + receipt: { preinstalled: "26.5" }, + version: "26.5", + }); + + const [listed] = await harness.installer.list(); + + expect(listed).toMatchObject({ installedBySimlock: false, platform: "ios" }); + }); + + it("counts registry devices of that platform and version, and not deleted ones", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: ["26.5", "27.0"], clock, platform: "ios" }), + ], + }); + await addDevice(harness.registry, "ios", "26.5"); + await addDevice(harness.registry, "ios", "26.5"); + const deleted = await addDevice(harness.registry, "ios", "26.5"); + await harness.registry.markDeviceMissing(deleted, "test"); + await addDevice(harness.registry, "ios", "27.0"); + // Same version string on the other platform: not a device of this component. + await addDevice(harness.registry, "android", "26.5"); + + const listed = await harness.installer.list(); + + expect(listed.map(({ devices, version }) => ({ devices, version }))).toEqual([ + { devices: 2, version: "26.5" }, + { devices: 1, version: "27.0" }, + ]); + }); + + it("shows the devices of a version on both of its variants", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new ListingDriver(clock, "android", [ + { + foreignDevices: 0, + receipt: { image: "a" }, + variant: "google_apis/arm64-v8a", + version: "35", + }, + { + foreignDevices: 0, + receipt: { image: "b" }, + variant: "default/arm64-v8a", + version: "35", + }, + ]), + ], + }); + await addDevice(harness.registry, "android", "35"); + await addDevice(harness.registry, "android", "35"); + + const listed = await harness.installer.list(); + + expect(listed.map(({ devices, variant }) => ({ devices, variant }))).toEqual([ + { devices: 2, variant: "default/arm64-v8a" }, + { devices: 2, variant: "google_apis/arm64-v8a" }, + ]); + }); + + it("carries the driver's size, variant and foreign devices through unread", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new ListingDriver(clock, "ios", [ + { + foreignDevices: 3, + receipt: { image: "x" }, + sizeBytes: 9 * gibibyte, + variant: "23A343", + version: "26.0", + }, + ]), + ], + }); + + expect(await harness.installer.list()).toEqual([ + { + devices: 0, + foreignDevices: 3, + installedBySimlock: false, + platform: "ios", + sizeBytes: 9 * gibibyte, + variant: "23A343", + version: "26.0", + }, + ]); + }); + + it("orders by platform, then version as numbers, then variant", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new ListingDriver(clock, "ios", [ + { foreignDevices: 0, receipt: { r: "1" }, version: "26.4" }, + { foreignDevices: 0, receipt: { r: "2" }, version: "9.3" }, + ]), + new ListingDriver(clock, "android", [ + { foreignDevices: 0, receipt: { r: "3" }, variant: "z", version: "35" }, + { foreignDevices: 0, receipt: { r: "4" }, variant: "a", version: "35" }, + { foreignDevices: 0, receipt: { r: "5" }, variant: "a", version: "9" }, + ]), + ], + }); + + const listed = await harness.installer.list(); + + expect(listed.map(({ platform, variant, version }) => [platform, version, variant])).toEqual([ + ["android", "9", "a"], + ["android", "35", "a"], + ["android", "35", "z"], + ["ios", "9.3", undefined], + ["ios", "26.4", undefined], + ]); + }); + + it("asks only the named platform's driver when given a platform", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }), + ], + }); + + const listed = await harness.installer.list("android"); + + expect(listed.map((component) => component.platform)).toEqual(["android"]); + expect(harness.ios.calls.some((call) => call.operation === "listComponents")).toBe(false); + }); + + it("leaves out a driver whose listing rejects, logs why, and still lists the other platform", async () => { + const warnings: { readonly message: string; readonly fields: unknown }[] = []; + const logger: Logger = { + child: () => logger, + debug: () => {}, + error: () => {}, + info: () => {}, + warn: (message, fields) => { + warnings.push({ fields, message }); + }, + }; + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }), + ], + logger, + }); + harness.ios.failOn("listComponents", 1, new DriverCrashError("simctl runtime list failed")); + + const listed = await harness.installer.list(); + + expect(listed.map((component) => [component.platform, component.version])).toEqual([ + ["android", "35"], + ]); + expect(warnings).toEqual([ + { + fields: { error: "DriverCrashError: simctl runtime list failed", platform: "ios" }, + message: "A driver could not list its installed components", + }, + ]); + }); + + it("starts no install, takes no reservation and emits no event", async () => { + const harness = await createHarness(); + + await harness.installer.list(); + + expect(installs(harness.ios)).toEqual([]); + expect(harness.reservations).toEqual([]); + expect(harness.events).toEqual([]); + }); +}); diff --git a/src/core/component-installer-outcomes.test.ts b/src/core/component-installer-outcomes.test.ts new file mode 100644 index 000000000..97adfd4e7 --- /dev/null +++ b/src/core/component-installer-outcomes.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, it } from "vitest"; +import { DriverCrashError, InsufficientDiskSpaceError } from "./driver.js"; +import { FakeDriver } from "./fake-driver.js"; +import { loadTestRegistry } from "./testing.js"; +import { + gibibyte, + createHarness, + installs, + flush, + track, + ios, +} from "./test-component-installer.js"; + +describe("ComponentInstaller", () => { + it("still lists a running install as downloading and one behind it as waiting after a call joined the running one", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: [], clock, installProgress: [41], platform: "ios" }), + ], + }); + harness.ios.holdInstalls(); + + void harness.installer.install(ios("27.0")); + void harness.installer.install(ios("28.0")); + await flush(); + void harness.installer.install(ios("27.0", { onProgress: () => undefined })); + + expect(harness.installer.inProgress()).toEqual([ + { component: "27.0", platform: "ios", since: 1_000, state: "downloading", waiters: 2 }, + { component: "28.0", platform: "ios", since: 1_000, state: "waiting", waiters: 1 }, + ]); + }); + + it("runs an iOS install and an Android install at the same time", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }), + new FakeDriver({ availableOsVersions: [], clock, platform: "android" }), + ], + }); + for (const driver of harness.drivers) driver.holdInstalls(); + + void harness.installer.install(ios("27.0")); + void harness.installer.install({ component: "35", platform: "android" }); + await flush(); + + expect(harness.drivers.map((driver) => installs(driver))).toEqual([["27.0"], ["35"]]); + }); + + it("rejects every call joined to a failed install with that failure, and the next call starts a new install", async () => { + const harness = await createHarness(); + const failure = new DriverCrashError("xcodebuild failed"); + harness.ios.failOn("installComponent", 1, failure); + + const joined = await Promise.allSettled([ + harness.installer.install(ios("27.0")), + harness.installer.install(ios("27.0")), + ]); + const next = await harness.installer.install(ios("27.0")); + + expect(joined).toEqual([ + { reason: failure, status: "rejected" }, + { reason: failure, status: "rejected" }, + ]); + expect(next).toEqual({ outcome: "installed", version: "27.0" }); + expect(installs(harness.ios)).toEqual(["27.0", "27.0"]); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.install-failed", + "component.install-started", + "component.installed", + ]); + }); + + it("ends a call whose stillNeeded answers false as not-needed, with no install and no event", async () => { + const harness = await createHarness(); + + await expect( + harness.installer.install(ios("27.0", { stillNeeded: async () => false })), + ).resolves.toEqual({ outcome: "not-needed" }); + expect(harness.ios.calls.map((call) => call.operation)).toEqual([]); + expect(harness.events).toEqual([]); + }); + + it("runs a waiting call's stillNeeded when it reaches the front, after the install ahead of it ended", async () => { + const harness = await createHarness({ + drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + harness.ios.holdInstalls(); + let asked = 0; + + void harness.installer.install(ios("27.0")); + const waiting = track( + harness.installer.install( + ios("latest", { + stillNeeded: async () => { + asked += 1; + return (await harness.ios.findComponent("27.0")) === undefined; + }, + }), + ), + ); + await flush(); + expect(asked).toBe(0); + harness.ios.releaseInstalls(); + await flush(); + + expect(asked).toBe(1); + expect(waiting.result()).toEqual({ outcome: "not-needed" }); + expect(installs(harness.ios)).toEqual(["27.0"]); + }); + + it("answers already-installed for an installed component with no reservation, no event and no install, even on a full disk", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + componentFootprint: { bytes: 8 * gibibyte, path: "/volume" }, + platform: "ios", + }), + ], + freeDiskBytes: gibibyte, + }); + + await expect(harness.installer.install(ios("26.5"))).resolves.toEqual({ + outcome: "already-installed", + version: "26.5", + }); + expect(installs(harness.ios)).toEqual([]); + expect(harness.reservations).toEqual([]); + expect(harness.events).toEqual([]); + expect(harness.registry.snapshot.components).toEqual([]); + }); + + it("fails with InsufficientDiskSpaceError and never calls installComponent when the footprint does not fit", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + componentFootprint: { bytes: 8 * gibibyte, path: "/volume" }, + platform: "ios", + }), + ], + freeDiskBytes: 4 * gibibyte, + }); + + await expect(harness.installer.install(ios("27.0"))).rejects.toBeInstanceOf( + InsufficientDiskSpaceError, + ); + expect(installs(harness.ios)).toEqual([]); + expect(harness.events).toEqual([]); + }); + + it("refuses the second of two installs on different platforms that fit one at a time but not together, before its driver is called", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, + platform: "ios", + }), + new FakeDriver({ + availableOsVersions: [], + clock, + componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, + platform: "android", + }), + ], + freeDiskBytes: 10 * gibibyte, + }); + const [iosDriver, androidDriver] = harness.drivers as [FakeDriver, FakeDriver]; + iosDriver.holdInstalls(); + + const first = track(harness.installer.install(ios("27.0"))); + await flush(); + await expect( + harness.installer.install({ component: "35", platform: "android" }), + ).rejects.toBeInstanceOf(InsufficientDiskSpaceError); + + expect(installs(androidDriver)).toEqual([]); + expect(installs(iosDriver)).toEqual(["27.0"]); + iosDriver.releaseInstalls(); + await flush(); + expect(first.result()).toEqual({ outcome: "installed", version: "27.0" }); + }); + + it("records an install in the registry with platform, version and the driver's receipt, and the record survives a reload", async () => { + const harness = await createHarness(); + harness.clock.advance(2_000); + + await harness.installer.install(ios("27.0")); + const reloaded = await loadTestRegistry({ + clock: harness.clock, + eventBus: harness.bus, + idGenerator: { generate: () => "id" }, + state: harness.state, + }); + + const expected = [ + { + installedAt: 3_000, + platform: "ios", + receipt: { install: "1", version: "27.0" }, + version: "27.0", + }, + ]; + expect(harness.registry.snapshot.components).toEqual(expected); + expect(reloaded.snapshot.components).toEqual(expected); + }); + + it("stores no record and emits component.installed with alreadyPresent when an install of newest finds the newest runtime already there", async () => { + const harness = await createHarness(); + + await expect(harness.installer.install(ios("latest"))).resolves.toEqual({ + outcome: "already-installed", + version: "26.5", + }); + expect(installs(harness.ios)).toEqual(["latest"]); + expect(harness.registry.snapshot.components).toEqual([]); + expect(harness.events.at(-1)).toEqual({ + event: "component.installed", + payload: { + alreadyPresent: true, + componentId: "latest", + durationMs: 0, + platform: "ios", + version: "26.5", + }, + }); + }); +}); diff --git a/src/core/component-installer-remove.test.ts b/src/core/component-installer-remove.test.ts new file mode 100644 index 000000000..12929dd7e --- /dev/null +++ b/src/core/component-installer-remove.test.ts @@ -0,0 +1,580 @@ +import { describe, expect, it } from "vitest"; +import { type Logger, MemoryFilesystem } from "../ports/index.js"; +import { + ComponentBeingRemovedError, + ComponentInstaller, + ComponentInstallerClosedError, + type ComponentInstallerProgress, +} from "./component-installer.js"; +import { + ComponentBusyError, + ComponentInUseError, + ComponentNotOwnedError, + DiskSpaceGuard, + DriverCrashError, +} from "./driver.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import { FakeDriver } from "./fake-driver.js"; +import type { DeviceRecord } from "./domain.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { + timeoutMs, + createHarness, + installs, + flush, + track, + ios, +} from "./test-component-installer.js"; + +describe("ComponentInstaller.remove", () => { + /** A harness whose iOS driver starts with nothing installed, then has `27.0` installed by Simlock. */ + async function withSimlockInstall( + options: { + readonly componentSizes?: Record; + readonly foreignDevices?: Record; + } = {}, + ) { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: [], clock, platform: "ios", ...options }), + ], + }); + await harness.installer.install(ios("27.0")); + harness.events.length = 0; + return harness; + } + + function removals(driver: FakeDriver): readonly unknown[] { + return driver.calls + .filter((call) => call.operation === "removeComponent") + .map((call) => call.arguments[0]); + } + + const removeIos = (version: string, extra: { readonly dryRun?: boolean } = {}) => ({ + platform: "ios" as const, + requesterId: "tok_admin", + version, + ...extra, + }); + + it("removes a component with a record and no users, deletes its record, and list no longer shows it", async () => { + const harness = await withSimlockInstall({ componentSizes: { "27.0": 7_000_000_000 } }); + const [record] = harness.registry.snapshot.components; + + await expect(harness.installer.remove(removeIos("27.0"))).resolves.toEqual({ + outcome: "removed", + sizeBytes: 7_000_000_000, + }); + + expect(removals(harness.ios)).toEqual([record?.receipt]); + expect(harness.registry.snapshot.components).toEqual([]); + expect(await harness.installer.list()).toEqual([]); + expect(harness.events).toEqual([ + { + event: "component.removed", + payload: { + componentId: "27.0", + platform: "ios", + requesterId: "tok_admin", + sizeBytes: 7_000_000_000, + version: "27.0", + }, + }, + ]); + }); + + it("carries the driver's residue into the answer and the event", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + platform: "ios", + removalResidue: "the download stays", + }), + ], + }); + await harness.installer.install(ios("27.0")); + + await expect(harness.installer.remove(removeIos("27.0"))).resolves.toEqual({ + outcome: "removed", + residue: "the download stays", + }); + expect(harness.events.at(-1)).toMatchObject({ + event: "component.removed", + payload: { residue: "the download stays" }, + }); + }); + + it("refuses a component with no record with ComponentNotOwnedError and never calls removeComponent", async () => { + // 26.5 is on the machine, but Simlock did not install it. + const harness = await createHarness(); + + await expect(harness.installer.remove(removeIos("26.5"))).rejects.toBeInstanceOf( + ComponentNotOwnedError, + ); + expect(removals(harness.ios)).toEqual([]); + expect(await harness.installer.list()).toMatchObject([{ version: "26.5" }]); + }); + + it("refuses a record whose version is no longer installed saying none is installed, and a record whose version was installed again saying that one is not Simlock's, in a removal and a dry run alike", async () => { + // 26.5 is on the machine with another receipt; 25.0 is not on the machine at all. + const harness = await createHarness(); + for (const version of ["25.0", "26.5"]) { + await harness.registry.recordComponent({ + installedAt: 1_000, + platform: "ios", + receipt: { install: "1", version }, + version, + }); + } + + for (const dryRun of [false, true]) { + await expect(harness.installer.remove(removeIos("25.0", { dryRun }))).rejects.toMatchObject({ + message: "No ios 25.0 is installed now, so there is nothing to remove", + name: "ComponentNotOwnedError", + }); + await expect(harness.installer.remove(removeIos("26.5", { dryRun }))).rejects.toMatchObject({ + message: + "The ios 26.5 installed now is not the one Simlock installed, so Simlock will not remove it", + name: "ComponentNotOwnedError", + }); + } + expect(removals(harness.ios)).toEqual([]); + expect(harness.registry.snapshot.components).toContainEqual( + expect.objectContaining({ platform: "ios", version: "26.5" }), + ); + }); + + it("removes a component whose receipt matches the record even when the listing names its version differently", async () => { + // The driver lists 26.5 with the receipt Simlock recorded under 26.5.0. + const harness = await createHarness(); + const receipt = { preinstalled: "26.5" }; + await harness.registry.recordComponent({ + installedAt: 1_000, + platform: "ios", + receipt, + version: "26.5.0", + }); + + await expect(harness.installer.remove(removeIos("26.5.0"))).resolves.toMatchObject({ + outcome: "removed", + }); + expect(removals(harness.ios)).toEqual([receipt]); + }); + + it.each(["provisioning", "ready", "leased", "reclaiming", "quarantined", "shutdown"] as const)( + "refuses a component a %s registry device uses with ComponentInUseError naming the count, and never calls removeComponent", + async (state) => { + const device: DeviceRecord = { + physical: false, + createdAt: 0, + driverData: {}, + driverDeviceId: "udid-1", + id: "dev_1", + mode: "full", + spec: { physical: false, model: "Phone", osVersion: "27.0", platform: "ios" }, + state, + }; + const harness = await withSimlockInstall(); + // The same installer's view, with one device of that version in the registry. + const installer = new ComponentInstaller({ + clock: harness.clock, + decisions: new SerializedDecision(), + diskSpace: new DiskSpaceGuard(), + drivers: new DriverCatalog(harness.drivers), + eventBus: harness.bus, + filesystem: new MemoryFilesystem(), + registry: { + deleteComponent: (platform, version) => + harness.registry.deleteComponent(platform, version), + recordComponent: (record) => harness.registry.recordComponent(record), + get snapshot() { + return { ...harness.registry.snapshot, devices: [device], virtualDevices: [device] }; + }, + }, + timeoutMs, + }); + + const error = await installer.remove(removeIos("27.0")).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(ComponentInUseError); + expect(error).toMatchObject({ devices: 1, foreignDevices: 0 }); + expect(removals(harness.ios)).toEqual([]); + expect(harness.registry.snapshot.components).toHaveLength(1); + }, + ); + + it("does not count a physical device on the runtime's OS as using it, so removal is not refused for it", async () => { + const physical: DeviceRecord = { + createdAt: 0, + driverData: {}, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: [], + id: "dev_physical", + physical: true, + spec: { + class: "phone", + model: "iPhone 15 Pro", + osVersion: "27.0", + physical: true, + platform: "ios", + }, + state: "ready", + }; + const harness = await withSimlockInstall(); + const installer = new ComponentInstaller({ + clock: harness.clock, + decisions: new SerializedDecision(), + diskSpace: new DiskSpaceGuard(), + drivers: new DriverCatalog(harness.drivers), + eventBus: harness.bus, + filesystem: new MemoryFilesystem(), + registry: { + deleteComponent: (platform, version) => harness.registry.deleteComponent(platform, version), + recordComponent: (record) => harness.registry.recordComponent(record), + get snapshot() { + return { ...harness.registry.snapshot, devices: [physical], virtualDevices: [] }; + }, + }, + timeoutMs, + }); + + await expect(installer.remove(removeIos("27.0"))).resolves.toMatchObject({ + outcome: "removed", + }); + expect(removals(harness.ios)).toHaveLength(1); + }); + + it("refuses a component a foreign device uses with ComponentInUseError naming the count, and never calls removeComponent", async () => { + const harness = await withSimlockInstall({ foreignDevices: { "27.0": 2 } }); + + const error = await harness.installer.remove(removeIos("27.0")).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(ComponentInUseError); + expect(error).toMatchObject({ devices: 0, foreignDevices: 2 }); + expect(removals(harness.ios)).toEqual([]); + }); + + it("refuses, and never calls removeComponent, when the driver's listing rejects", async () => { + const harness = await withSimlockInstall(); + harness.ios.failOn("listComponents", 1, new DriverCrashError("default set unreadable")); + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( + "default set unreadable", + ); + expect(removals(harness.ios)).toEqual([]); + expect(harness.registry.snapshot.components).toHaveLength(1); + }); + + it("refuses with ComponentInUseError while a device is being provisioned on it and has no record yet", async () => { + const harness = await withSimlockInstall(); + const release = harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toMatchObject({ + devices: 1, + foreignDevices: 0, + }); + await expect( + harness.installer.remove(removeIos("27.0", { dryRun: true })), + ).rejects.toMatchObject({ devices: 1, foreignDevices: 0, name: "ComponentInUseError" }); + expect(removals(harness.ios)).toEqual([]); + + release(); + await expect(harness.installer.remove(removeIos("27.0"))).resolves.toMatchObject({ + outcome: "removed", + }); + }); + + it("answers a dry run with would-remove and the size, calls no driver verb that changes anything, and keeps the record", async () => { + const harness = await withSimlockInstall({ componentSizes: { "27.0": 7_000_000_000 } }); + const before = harness.ios.calls.length; + + await expect(harness.installer.remove(removeIos("27.0", { dryRun: true }))).resolves.toEqual({ + outcome: "would-remove", + sizeBytes: 7_000_000_000, + }); + + expect(harness.ios.calls.slice(before).map((call) => call.operation)).toEqual([ + "listComponents", + ]); + expect(harness.registry.snapshot.components).toHaveLength(1); + expect(harness.events).toEqual([]); + expect(await harness.installer.list()).toMatchObject([ + { installedBySimlock: true, version: "27.0" }, + ]); + }); + + it.each([false, true])( + "refuses with ComponentBusyError while an install is running on that platform (dry run: %s)", + async (dryRun) => { + const harness = await withSimlockInstall(); + harness.ios.holdInstalls(); + const install = track(harness.installer.install(ios("28.0"))); + await flush(); + expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); + + await expect(harness.installer.remove(removeIos("27.0", { dryRun }))).rejects.toBeInstanceOf( + ComponentBusyError, + ); + expect(removals(harness.ios)).toEqual([]); + + harness.ios.releaseInstalls(); + await flush(); + expect(install.result()).toMatchObject({ outcome: "installed" }); + }, + ); + + it("refuses with ComponentBusyError while an install waits on that platform, before its driver runs", async () => { + const harness = await withSimlockInstall(); + // At the front of the queue, still checking whether it is needed: waiting, not downloading. + void harness.installer + .install(ios("28.0", { stillNeeded: () => new Promise(() => undefined) })) + .catch(() => undefined); + await flush(); + expect(harness.installer.inProgress()).toMatchObject([{ component: "28.0", state: "waiting" }]); + expect(installs(harness.ios)).toEqual(["27.0"]); + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toBeInstanceOf( + ComponentBusyError, + ); + expect(removals(harness.ios)).toEqual([]); + }); + + it("counts Simlock's devices again inside the decision gate, and refuses one claimed after the listing", async () => { + const decisions = new SerializedDecision(); + const harness = await createHarness({ + decisions, + drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + await harness.installer.install(ios("27.0")); + // Something else holds the gate, so the removal's listing and first count pass before it + // gets its turn there. + let releaseGate!: () => void; + void decisions.run( + () => + new Promise((resolve) => { + releaseGate = resolve; + }), + ); + const removal = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + expect(harness.ios.calls.filter((call) => call.operation === "listComponents")).toHaveLength(1); + expect(removal.settled()).toBe(false); + + harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); + releaseGate(); + await flush(); + + expect(removal.error()).toBeInstanceOf(ComponentInUseError); + expect(removal.error()).toMatchObject({ devices: 1 }); + expect(removals(harness.ios)).toEqual([]); + }); + + it("starts an install that arrives during a removal only after the removal has ended", async () => { + const harness = await withSimlockInstall(); + harness.ios.holdRemovals(); + const removal = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + expect(removals(harness.ios)).toHaveLength(1); + + const progress: ComponentInstallerProgress[] = []; + const install = track( + harness.installer.install(ios("28.0", { onProgress: (heard) => progress.push(heard) })), + ); + await flush(); + expect(progress).toEqual([{ stage: "waiting" }]); + expect(installs(harness.ios)).toEqual(["27.0"]); + expect(harness.ios.calls.filter((call) => call.operation === "findComponent")).toHaveLength(1); + expect(harness.installer.inProgress()).toMatchObject([{ component: "28.0", state: "waiting" }]); + + harness.ios.releaseRemovals(); + await flush(); + + expect(removal.result()).toMatchObject({ outcome: "removed" }); + expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); + expect(install.result()).toMatchObject({ outcome: "installed", version: "28.0" }); + }); + + it("refuses a second removal on the platform while one runs with ComponentBusyError", async () => { + const harness = await createHarness({ + drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + await harness.installer.install(ios("27.0")); + await harness.installer.install(ios("28.0")); + harness.ios.holdRemovals(); + const first = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + + await expect(harness.installer.remove(removeIos("28.0"))).rejects.toBeInstanceOf( + ComponentBusyError, + ); + + harness.ios.releaseRemovals(); + await flush(); + expect(first.result()).toMatchObject({ outcome: "removed" }); + }); + + it("refuses every device claimed on the component while it is being removed, and none once the removal has settled", async () => { + const harness = await withSimlockInstall(); + harness.ios.holdRemovals(); + const removal = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + + expect(() => harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" })).toThrow( + ComponentBeingRemovedError, + ); + // Another version is not being removed. + expect(() => + harness.installer.claimProvision({ osVersion: "26.0", platform: "ios" }), + ).not.toThrow(); + + harness.ios.releaseRemovals(); + await flush(); + expect(removal.settled()).toBe(true); + expect(() => + harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }), + ).not.toThrow(); + }); + + it("keeps the record, clears the mark and emits no component.removed when the driver fails the removal", async () => { + const harness = await withSimlockInstall(); + harness.ios.failOn("removeComponent", 1, new DriverCrashError("simctl runtime delete failed")); + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( + "simctl runtime delete failed", + ); + + expect(harness.registry.snapshot.components).toMatchObject([{ version: "27.0" }]); + expect(harness.events).toEqual([]); + expect(() => + harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }), + ).not.toThrow(); + // The turn is back: an install on the platform starts. + await expect(harness.installer.install(ios("28.0"))).resolves.toMatchObject({ + outcome: "installed", + }); + }); + + it("refuses with ComponentInstallerClosedError, removing nothing, when the daemon stops while the listing is read", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + latencyMs: { listComponents: 10 }, + platform: "ios", + }), + ], + }); + await harness.installer.install(ios("27.0")); + const removal = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + expect(harness.ios.calls.filter((call) => call.operation === "listComponents")).toHaveLength(1); + + await harness.installer.close(); + harness.clock.advance(10); + await flush(); + + expect(removal.error()).toBeInstanceOf(ComponentInstallerClosedError); + expect(removals(harness.ios)).toEqual([]); + expect(harness.registry.snapshot.components).toHaveLength(1); + }); + + it("ends one claim per release however often that release is called", async () => { + const harness = await withSimlockInstall(); + const first = harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); + harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); + + first(); + first(); + + expect(await harness.installer.list()).toMatchObject([{ devices: 1, version: "27.0" }]); + }); + + it("refuses with ComponentInstallerClosedError once the daemon has stopped, without reading the listing", async () => { + const harness = await withSimlockInstall(); + await harness.installer.close(); + const before = harness.ios.calls.length; + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toBeInstanceOf( + ComponentInstallerClosedError, + ); + expect(harness.ios.calls.slice(before)).toEqual([]); + }); + + it("resolves close only once a running removal's driver call has returned", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + // A delay the abort does not cut short, as a platform tool that is slow to exit. + latencyMs: { removeComponent: 10 }, + platform: "ios", + }), + ], + }); + await harness.installer.install(ios("27.0")); + void harness.installer.remove(removeIos("27.0")).catch(() => undefined); + await flush(); + expect(removals(harness.ios)).toHaveLength(1); + + const closing = track(harness.installer.close()); + await flush(); + expect(closing.settled()).toBe(false); + + harness.clock.advance(10); + await flush(); + expect(closing.settled()).toBe(true); + }); + + it("logs a failed removal with who asked, after the driver was called", async () => { + const warnings: { readonly message: string; readonly fields: unknown }[] = []; + const logger: Logger = { + child: () => logger, + debug: () => {}, + error: () => {}, + info: () => {}, + warn: (message, fields) => { + warnings.push({ fields, message }); + }, + }; + const harness = await createHarness({ + drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], + logger, + }); + await harness.installer.install(ios("27.0")); + harness.ios.failOn("removeComponent", 1, new DriverCrashError("simctl runtime delete failed")); + + await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( + "simctl runtime delete failed", + ); + + expect(warnings).toEqual([ + { + fields: { + error: "DriverCrashError: simctl runtime delete failed", + platform: "ios", + requesterId: "tok_admin", + version: "27.0", + }, + message: "A component removal failed; Simlock's record of it is kept", + }, + ]); + }); + + it("ends a running removal on close with ComponentInstallerClosedError, and keeps the record", async () => { + const harness = await withSimlockInstall(); + harness.ios.holdRemovals(); + const removal = track(harness.installer.remove(removeIos("27.0"))); + await flush(); + + await harness.installer.close(); + await flush(); + + expect(removal.error()).toBeInstanceOf(ComponentInstallerClosedError); + expect(harness.registry.snapshot.components).toHaveLength(1); + expect(harness.events).toEqual([]); + }); +}); diff --git a/src/core/component-installer-timeouts-and-close.test.ts b/src/core/component-installer-timeouts-and-close.test.ts new file mode 100644 index 000000000..1d50088a2 --- /dev/null +++ b/src/core/component-installer-timeouts-and-close.test.ts @@ -0,0 +1,397 @@ +import { describe, expect, it } from "vitest"; +import { + ComponentInstallerClosedError, + type ComponentInstallerProgress, +} from "./component-installer.js"; +import { ComponentInstallTimeoutError, DriverCrashError } from "./driver.js"; +import { FakeDriver } from "./fake-driver.js"; +import { + gibibyte, + timeoutMs, + ManualClock, + createHarness, + installs, + flush, + track, + ios, +} from "./test-component-installer.js"; + +describe("ComponentInstaller", () => { + it("fails a call still waiting when downloads.timeoutMs has passed since it arrived, without calling its driver", async () => { + const harness = await createHarness(); + harness.ios.holdInstalls(); + + void harness.installer.install(ios("27.0")).catch(() => undefined); + const waiting = track(harness.installer.install(ios("28.0"))); + await flush(); + harness.clock.advance(timeoutMs); + await flush(); + + expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); + // Nothing is ahead of it any more, and its install still never reaches the driver. + harness.ios.releaseInstalls(); + await flush(); + expect(harness.ios.calls.map((call) => call.arguments[0])).not.toContain("28.0"); + }); + + it("aborts the install of a call that waited for half its budget when the other half has passed, not a full budget later", async () => { + const harness = await createHarness(); + harness.ios.holdInstalls(); + + void harness.installer.install(ios("27.0")); + await flush(); + harness.clock.advance(1); + const behind = track(harness.installer.install(ios("28.0"))); + await flush(); + // Half its budget spent waiting; then the install ahead ends and its own starts. + harness.clock.advance(timeoutMs / 2); + harness.ios.releaseInstalls(); + harness.ios.holdInstalls(); + await flush(); + expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); + + harness.clock.advance(timeoutMs / 2 - 1); + await flush(); + expect(behind.settled()).toBe(false); + harness.clock.advance(1); + await flush(); + expect(behind.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.installed", + "component.install-started", + "component.install-failed", + ]); + }); + + it("aborts a running install at the deadline of the oldest call joined to it, and fails a call that joined later at that moment", async () => { + const harness = await createHarness(); + harness.ios.holdInstalls(); + + const oldest = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(10_000); + const later = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(timeoutMs - 10_000); + await flush(); + + const timedOut = new ComponentInstallTimeoutError("ios", "27.0", timeoutMs); + expect(oldest.error()).toEqual(timedOut); + expect(later.error()).toEqual(timedOut); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.install-failed", + ]); + }); + + // Every call has the same budget, so a waiting call's deadline never falls before that of the + // oldest call on the install running ahead of it. Firing only the waiting call's timer is what + // shows its expiry touches nothing but its own call and its own install. + it("leaves the install running ahead of a waiting call alone when that call times out", async () => { + const harness = await createHarness({ clock: new ManualClock() }); + harness.ios.holdInstalls(); + + const ahead = track(harness.installer.install(ios("27.0"))); + const waiting = track(harness.installer.install(ios("28.0"))); + await flush(); + (harness.clock as ManualClock).fire(1); + await flush(); + expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); + harness.ios.releaseInstalls(); + await flush(); + + expect(ahead.result()).toEqual({ outcome: "installed", version: "27.0" }); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.installed", + ]); + }); + + it("starts no install for a waiting install whose only call timed out", async () => { + const harness = await createHarness({ clock: new ManualClock() }); + harness.ios.holdInstalls(); + + void harness.installer.install(ios("27.0")); + void harness.installer.install(ios("28.0")).catch(() => undefined); + await flush(); + (harness.clock as ManualClock).fire(1); + await flush(); + harness.ios.releaseInstalls(); + await flush(); + + expect(installs(harness.ios)).toEqual(["27.0"]); + }); + + it("gives the disk back when an install ends, so the next install that fits alone starts", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, + platform: "ios", + }), + ], + freeDiskBytes: 10 * gibibyte, + }); + harness.ios.failOn("installComponent", 1, new DriverCrashError("xcodebuild failed")); + + await expect(harness.installer.install(ios("27.0"))).rejects.toThrow("xcodebuild failed"); + await expect(harness.installer.install(ios("28.0"))).resolves.toMatchObject({ + outcome: "installed", + }); + await expect(harness.installer.install(ios("29.0"))).resolves.toMatchObject({ + outcome: "installed", + }); + expect(harness.reservations.map((reservation) => reservation.released)).toEqual([ + true, + true, + true, + ]); + }); + + it("keeps the deadline of a call that ended as not-needed from ending the install the other calls joined", async () => { + const harness = await createHarness(); + harness.ios.holdInstalls(); + + const notNeeded = track( + harness.installer.install(ios("27.0", { stillNeeded: async () => false })), + ); + harness.clock.advance(1); + const needed = track(harness.installer.install(ios("27.0"))); + await flush(); + expect(notNeeded.result()).toEqual({ outcome: "not-needed" }); + expect(installs(harness.ios)).toEqual(["27.0"]); + + // The not-needed call's budget runs out here; the install belongs to the other call now. + harness.clock.advance(timeoutMs - 1); + await flush(); + expect(needed.settled()).toBe(false); + harness.clock.advance(1); + await flush(); + expect(needed.error()).toEqual(new ComponentInstallTimeoutError("ios", "27.0", timeoutMs)); + }); + + it("starts a new install for a call that arrives while a timed-out install's driver is still returning, after it returns", async () => { + let finishStubborn: (() => void) | undefined; + class StubbornDriver extends FakeDriver { + readonly started: string[] = []; + + override async installComponent( + component: string, + options: Parameters[1], + ) { + this.started.push(component); + if (this.started.length === 1) { + // Ignores its signal, like an installer that takes a while to die. + await new Promise((resolve) => { + finishStubborn = resolve; + }); + throw new DriverCrashError("ended late"); + } + return super.installComponent(component, options); + } + } + const harness = await createHarness({ + drivers: (clock) => [new StubbornDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + const driver = harness.ios as StubbornDriver; + + const first = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(timeoutMs); + await flush(); + expect(first.error()).toBeInstanceOf(ComponentInstallTimeoutError); + const second = track(harness.installer.install(ios("27.0"))); + await flush(); + expect(driver.started).toEqual(["27.0"]); + + finishStubborn?.(); + await flush(); + expect(driver.started).toEqual(["27.0", "27.0"]); + expect(second.result()).toEqual({ outcome: "installed", version: "27.0" }); + }); + + it("rejects only the call whose stillNeeded throws, and installs for the others joined to it", async () => { + const harness = await createHarness(); + const failure = new Error("resolve failed"); + // An install ahead keeps all three waiting, so each reaches the front with its check. + harness.ios.holdInstalls(); + void harness.installer.install(ios("28.0")); + await flush(); + harness.ios.releaseInstalls(); + + const [plain, throwing, notNeeded] = await Promise.allSettled([ + harness.installer.install(ios("27.0")), + harness.installer.install( + ios("27.0", { + stillNeeded: async () => { + throw failure; + }, + }), + ), + harness.installer.install(ios("27.0", { stillNeeded: async () => false })), + ]); + + expect(plain).toEqual({ + status: "fulfilled", + value: { outcome: "installed", version: "27.0" }, + }); + expect(throwing).toEqual({ reason: failure, status: "rejected" }); + expect(notNeeded).toEqual({ status: "fulfilled", value: { outcome: "not-needed" } }); + expect(installs(harness.ios)).toEqual(["28.0", "27.0"]); + }); + + it("keeps a throwing progress callback from reaching the install or the other calls", async () => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ availableOsVersions: [], clock, installProgress: [50], platform: "ios" }), + ], + }); + const heard: ComponentInstallerProgress[] = []; + + const [throwing, listening] = await Promise.allSettled([ + harness.installer.install( + ios("27.0", { + onProgress: () => { + throw new Error("observer broke"); + }, + }), + ), + harness.installer.install(ios("27.0", { onProgress: (report) => heard.push(report) })), + ]); + + expect(throwing).toMatchObject({ status: "fulfilled" }); + expect(listening).toMatchObject({ status: "fulfilled" }); + expect(heard).toEqual([ + { stage: "downloading" }, + { percent: 50, stage: "downloading" }, + { percent: 100, stage: "downloading" }, + ]); + }); + + it.each([ + ["checking whether the component is installed", "findComponent"], + ["reserving disk", "diskFree"], + ] as const)( + "starts no install when the only call ran out of budget while the installer was %s", + async (_step, slow) => { + const harness = await createHarness({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + latencyMs: slow === "findComponent" ? { findComponent: timeoutMs + 1 } : {}, + platform: "ios", + }), + ], + filesystem: (clock) => ({ + diskFree: async () => { + if (slow === "diskFree") { + await new Promise((resolve) => clock.setTimer(timeoutMs + 1, resolve)); + } + return Number.MAX_SAFE_INTEGER; + }, + }), + }); + + const call = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(timeoutMs); + await flush(); + expect(call.error()).toBeInstanceOf(ComponentInstallTimeoutError); + harness.clock.advance(1); + await flush(); + + expect(installs(harness.ios)).toEqual([]); + expect(harness.events).toEqual([]); + // Nothing reserved once the call is gone; a reservation already taken is given back. + expect(harness.reservations).toEqual( + slow === "findComponent" ? [] : [{ bytes: 0, released: true }], + ); + }, + ); + + it("resolves close only once the running driver call has returned", async () => { + let finishStubborn: (() => void) | undefined; + class StubbornDriver extends FakeDriver { + override async installComponent(): ReturnType { + // Ignores its signal, like an installer that takes a while to die. + await new Promise((resolve) => { + finishStubborn = resolve; + }); + throw new DriverCrashError("ended late"); + } + } + const harness = await createHarness({ + drivers: (clock) => [new StubbornDriver({ availableOsVersions: [], clock, platform: "ios" })], + }); + + void harness.installer.install(ios("27.0")).catch(() => undefined); + await flush(); + const closing = track(harness.installer.close()); + await flush(); + expect(closing.settled()).toBe(false); + + finishStubborn?.(); + await flush(); + expect(closing.settled()).toBe(true); + }); + + it("fails only the call whose budget runs out while a finished install's record is written, and installs for the rest", async () => { + // A gate the test opens by hand, standing in for other registry writes ahead of the record. + let openGate: (() => void) | undefined; + const gate = new Promise((resolve) => { + openGate = resolve; + }); + const harness = await createHarness({ + decisions: { + run: async (operation: () => Result | Promise) => { + await gate; + return operation(); + }, + }, + }); + + const oldest = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(1); + const later = track(harness.installer.install(ios("27.0"))); + await flush(); + harness.clock.advance(timeoutMs - 1); + await flush(); + expect(oldest.error()).toEqual(new ComponentInstallTimeoutError("ios", "27.0", timeoutMs)); + expect(later.settled()).toBe(false); + + openGate?.(); + await flush(); + expect(later.result()).toEqual({ outcome: "installed", version: "27.0" }); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.installed", + ]); + }); + + it("aborts the running install and rejects the waiting calls on close", async () => { + const harness = await createHarness(); + harness.ios.holdInstalls(); + + const running = track(harness.installer.install(ios("27.0"))); + const waiting = track(harness.installer.install(ios("28.0"))); + await flush(); + await harness.installer.close(); + await flush(); + + expect(running.error()).toBeInstanceOf(ComponentInstallerClosedError); + expect(waiting.error()).toBeInstanceOf(ComponentInstallerClosedError); + expect(installs(harness.ios)).toEqual(["27.0"]); + expect(harness.events.map((event) => event.event)).toEqual([ + "component.install-started", + "component.install-failed", + ]); + await expect(harness.installer.install(ios("29.0"))).rejects.toBeInstanceOf( + ComponentInstallerClosedError, + ); + }); +}); diff --git a/src/core/component-installer.test.ts b/src/core/component-installer.test.ts index 5d640261c..ca649639d 100644 --- a/src/core/component-installer.test.ts +++ b/src/core/component-installer.test.ts @@ -1,169 +1,8 @@ import { describe, expect, it, vi } from "vitest"; - -import { EventBus } from "../bus/index.js"; -import { - FakeClock, - type Filesystem, - type Logger, - MemoryFilesystem, - type TimerHandle, -} from "../ports/index.js"; -import { - ComponentBeingRemovedError, - ComponentInstaller, - ComponentInstallerClosedError, - type ComponentInstallerProgress, - type ComponentInstallRequest, -} from "./component-installer.js"; -import { - ComponentBusyError, - ComponentInUseError, - ComponentInstallTimeoutError, - ComponentNotOwnedError, - DiskSpaceGuard, - type DriverComponent, - DriverCrashError, - InsufficientDiskSpaceError, -} from "./driver.js"; -import { DriverCatalog } from "./driver-catalog.js"; +import type { ComponentInstallerProgress } from "./component-installer.js"; +import { DriverCrashError } from "./driver.js"; import { FakeDriver } from "./fake-driver.js"; -import type { DeviceRecord } from "./domain.js"; -import type { Registry } from "./registry.js"; -import { loadTestRegistry, openTestState } from "./testing.js"; -import { SerializedDecision } from "./serialized-decision.js"; - -const gibibyte = 1024 ** 3; -const timeoutMs = 60_000; - -/** A clock whose timers fire only when a test fires them, one at a time, by creation order. */ -class ManualClock extends FakeClock { - readonly #timers: { callback: () => void; cancelled: boolean }[] = []; - - override setTimer(_delayMs: number, callback: () => void): TimerHandle { - const timer = { callback, cancelled: false }; - this.#timers.push(timer); - return timer as unknown as TimerHandle; - } - - override cancel(handle: TimerHandle): void { - (handle as unknown as { cancelled: boolean }).cancelled = true; - } - - fire(index: number): void { - const timer = this.#timers[index]; - if (timer === undefined || timer.cancelled) throw new Error(`No live timer ${String(index)}`); - timer.cancelled = true; - timer.callback(); - } -} - -async function createHarness( - options: { - readonly clock?: FakeClock; - readonly drivers?: (clock: FakeClock) => readonly FakeDriver[]; - readonly decisions?: Pick; - readonly filesystem?: (clock: FakeClock) => Pick; - readonly freeDiskBytes?: number; - readonly logger?: Logger; - } = {}, -) { - const clock = options.clock ?? new FakeClock(1_000); - const bus = new EventBus(clock); - const events: { readonly event: string; readonly payload: unknown }[] = []; - bus.subscribeAll((envelope) => { - if (envelope.event.startsWith("component.")) { - events.push({ event: envelope.event, payload: envelope.payload }); - } - }); - const drivers = options.drivers?.(clock) ?? [ - new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), - ]; - const state = await openTestState(); - let nextId = 0; - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => `id${String((nextId += 1))}` }, - state, - }); - // Every reservation the installer takes, and whether it gave it back. - const guard = new DiskSpaceGuard(); - const reservations: { readonly bytes: number; released: boolean }[] = []; - const installer = new ComponentInstaller({ - clock, - decisions: options.decisions ?? new SerializedDecision(), - diskSpace: { - reserve: async (filesystem, platform, bytes, path) => { - const release = await guard.reserve(filesystem, platform, bytes, path); - const reservation = { bytes, released: false }; - reservations.push(reservation); - return () => { - reservation.released = true; - release(); - }; - }, - }, - drivers: new DriverCatalog(drivers), - eventBus: bus, - filesystem: options.filesystem?.(clock) ?? new MemoryFilesystem(options.freeDiskBytes), - ...(options.logger === undefined ? {} : { logger: options.logger }), - registry, - timeoutMs, - }); - const ios = drivers.find((driver) => driver.platform === "ios") as FakeDriver; - return { - bus, - clock, - drivers, - events, - installer, - ios, - registry, - reservations, - state, - }; -} - -function installs(driver: FakeDriver): readonly unknown[] { - return driver.calls - .filter((call) => call.operation === "installComponent") - .map((call) => call.arguments[0]); -} - -/** Lets every settled promise and queued continuation run. */ -async function flush(): Promise { - for (let count = 0; count < 100; count += 1) { - await Promise.resolve(); - } -} - -/** A promise's outcome without awaiting it, so a test can look while it is still open. */ -function track(promise: Promise): { - readonly settled: () => boolean; - readonly result: () => T | undefined; - readonly error: () => unknown; -} { - let state: { kind: "open" } | { kind: "ok"; value: T } | { kind: "error"; error: unknown } = { - kind: "open", - }; - promise.then( - (value) => { - state = { kind: "ok", value }; - }, - (error: unknown) => { - state = { error, kind: "error" }; - }, - ); - return { - error: () => (state.kind === "error" ? state.error : undefined), - result: () => (state.kind === "ok" ? state.value : undefined), - settled: () => state.kind !== "open", - }; -} - -function ios(component: string, extra: Partial = {}) { - return { component, platform: "ios" as const, ...extra }; -} +import { createHarness, installs, flush, track, ios } from "./test-component-installer.js"; describe("ComponentInstaller", () => { it("admits a call synchronously, before any progress, and a throwing onAdmitted does not stop its install", async () => { @@ -465,1483 +304,4 @@ describe("ComponentInstaller", () => { ]); }); }); - - it("still lists a running install as downloading and one behind it as waiting after a call joined the running one", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: [], clock, installProgress: [41], platform: "ios" }), - ], - }); - harness.ios.holdInstalls(); - - void harness.installer.install(ios("27.0")); - void harness.installer.install(ios("28.0")); - await flush(); - void harness.installer.install(ios("27.0", { onProgress: () => undefined })); - - expect(harness.installer.inProgress()).toEqual([ - { component: "27.0", platform: "ios", since: 1_000, state: "downloading", waiters: 2 }, - { component: "28.0", platform: "ios", since: 1_000, state: "waiting", waiters: 1 }, - ]); - }); - - it("runs an iOS install and an Android install at the same time", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }), - new FakeDriver({ availableOsVersions: [], clock, platform: "android" }), - ], - }); - for (const driver of harness.drivers) driver.holdInstalls(); - - void harness.installer.install(ios("27.0")); - void harness.installer.install({ component: "35", platform: "android" }); - await flush(); - - expect(harness.drivers.map((driver) => installs(driver))).toEqual([["27.0"], ["35"]]); - }); - - it("rejects every call joined to a failed install with that failure, and the next call starts a new install", async () => { - const harness = await createHarness(); - const failure = new DriverCrashError("xcodebuild failed"); - harness.ios.failOn("installComponent", 1, failure); - - const joined = await Promise.allSettled([ - harness.installer.install(ios("27.0")), - harness.installer.install(ios("27.0")), - ]); - const next = await harness.installer.install(ios("27.0")); - - expect(joined).toEqual([ - { reason: failure, status: "rejected" }, - { reason: failure, status: "rejected" }, - ]); - expect(next).toEqual({ outcome: "installed", version: "27.0" }); - expect(installs(harness.ios)).toEqual(["27.0", "27.0"]); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.install-failed", - "component.install-started", - "component.installed", - ]); - }); - - it("ends a call whose stillNeeded answers false as not-needed, with no install and no event", async () => { - const harness = await createHarness(); - - await expect( - harness.installer.install(ios("27.0", { stillNeeded: async () => false })), - ).resolves.toEqual({ outcome: "not-needed" }); - expect(harness.ios.calls.map((call) => call.operation)).toEqual([]); - expect(harness.events).toEqual([]); - }); - - it("runs a waiting call's stillNeeded when it reaches the front, after the install ahead of it ended", async () => { - const harness = await createHarness({ - drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - harness.ios.holdInstalls(); - let asked = 0; - - void harness.installer.install(ios("27.0")); - const waiting = track( - harness.installer.install( - ios("latest", { - stillNeeded: async () => { - asked += 1; - return (await harness.ios.findComponent("27.0")) === undefined; - }, - }), - ), - ); - await flush(); - expect(asked).toBe(0); - harness.ios.releaseInstalls(); - await flush(); - - expect(asked).toBe(1); - expect(waiting.result()).toEqual({ outcome: "not-needed" }); - expect(installs(harness.ios)).toEqual(["27.0"]); - }); - - it("answers already-installed for an installed component with no reservation, no event and no install, even on a full disk", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - componentFootprint: { bytes: 8 * gibibyte, path: "/volume" }, - platform: "ios", - }), - ], - freeDiskBytes: gibibyte, - }); - - await expect(harness.installer.install(ios("26.5"))).resolves.toEqual({ - outcome: "already-installed", - version: "26.5", - }); - expect(installs(harness.ios)).toEqual([]); - expect(harness.reservations).toEqual([]); - expect(harness.events).toEqual([]); - expect(harness.registry.snapshot.components).toEqual([]); - }); - - it("fails with InsufficientDiskSpaceError and never calls installComponent when the footprint does not fit", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - componentFootprint: { bytes: 8 * gibibyte, path: "/volume" }, - platform: "ios", - }), - ], - freeDiskBytes: 4 * gibibyte, - }); - - await expect(harness.installer.install(ios("27.0"))).rejects.toBeInstanceOf( - InsufficientDiskSpaceError, - ); - expect(installs(harness.ios)).toEqual([]); - expect(harness.events).toEqual([]); - }); - - it("refuses the second of two installs on different platforms that fit one at a time but not together, before its driver is called", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, - platform: "ios", - }), - new FakeDriver({ - availableOsVersions: [], - clock, - componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, - platform: "android", - }), - ], - freeDiskBytes: 10 * gibibyte, - }); - const [iosDriver, androidDriver] = harness.drivers as [FakeDriver, FakeDriver]; - iosDriver.holdInstalls(); - - const first = track(harness.installer.install(ios("27.0"))); - await flush(); - await expect( - harness.installer.install({ component: "35", platform: "android" }), - ).rejects.toBeInstanceOf(InsufficientDiskSpaceError); - - expect(installs(androidDriver)).toEqual([]); - expect(installs(iosDriver)).toEqual(["27.0"]); - iosDriver.releaseInstalls(); - await flush(); - expect(first.result()).toEqual({ outcome: "installed", version: "27.0" }); - }); - - it("records an install in the registry with platform, version and the driver's receipt, and the record survives a reload", async () => { - const harness = await createHarness(); - harness.clock.advance(2_000); - - await harness.installer.install(ios("27.0")); - const reloaded = await loadTestRegistry({ - clock: harness.clock, - eventBus: harness.bus, - idGenerator: { generate: () => "id" }, - state: harness.state, - }); - - const expected = [ - { - installedAt: 3_000, - platform: "ios", - receipt: { install: "1", version: "27.0" }, - version: "27.0", - }, - ]; - expect(harness.registry.snapshot.components).toEqual(expected); - expect(reloaded.snapshot.components).toEqual(expected); - }); - - it("stores no record and emits component.installed with alreadyPresent when an install of newest finds the newest runtime already there", async () => { - const harness = await createHarness(); - - await expect(harness.installer.install(ios("latest"))).resolves.toEqual({ - outcome: "already-installed", - version: "26.5", - }); - expect(installs(harness.ios)).toEqual(["latest"]); - expect(harness.registry.snapshot.components).toEqual([]); - expect(harness.events.at(-1)).toEqual({ - event: "component.installed", - payload: { - alreadyPresent: true, - componentId: "latest", - durationMs: 0, - platform: "ios", - version: "26.5", - }, - }); - }); - - it("fails a call still waiting when downloads.timeoutMs has passed since it arrived, without calling its driver", async () => { - const harness = await createHarness(); - harness.ios.holdInstalls(); - - void harness.installer.install(ios("27.0")).catch(() => undefined); - const waiting = track(harness.installer.install(ios("28.0"))); - await flush(); - harness.clock.advance(timeoutMs); - await flush(); - - expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); - // Nothing is ahead of it any more, and its install still never reaches the driver. - harness.ios.releaseInstalls(); - await flush(); - expect(harness.ios.calls.map((call) => call.arguments[0])).not.toContain("28.0"); - }); - - it("aborts the install of a call that waited for half its budget when the other half has passed, not a full budget later", async () => { - const harness = await createHarness(); - harness.ios.holdInstalls(); - - void harness.installer.install(ios("27.0")); - await flush(); - harness.clock.advance(1); - const behind = track(harness.installer.install(ios("28.0"))); - await flush(); - // Half its budget spent waiting; then the install ahead ends and its own starts. - harness.clock.advance(timeoutMs / 2); - harness.ios.releaseInstalls(); - harness.ios.holdInstalls(); - await flush(); - expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); - - harness.clock.advance(timeoutMs / 2 - 1); - await flush(); - expect(behind.settled()).toBe(false); - harness.clock.advance(1); - await flush(); - expect(behind.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.installed", - "component.install-started", - "component.install-failed", - ]); - }); - - it("aborts a running install at the deadline of the oldest call joined to it, and fails a call that joined later at that moment", async () => { - const harness = await createHarness(); - harness.ios.holdInstalls(); - - const oldest = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(10_000); - const later = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(timeoutMs - 10_000); - await flush(); - - const timedOut = new ComponentInstallTimeoutError("ios", "27.0", timeoutMs); - expect(oldest.error()).toEqual(timedOut); - expect(later.error()).toEqual(timedOut); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.install-failed", - ]); - }); - - // Every call has the same budget, so a waiting call's deadline never falls before that of the - // oldest call on the install running ahead of it. Firing only the waiting call's timer is what - // shows its expiry touches nothing but its own call and its own install. - it("leaves the install running ahead of a waiting call alone when that call times out", async () => { - const harness = await createHarness({ clock: new ManualClock() }); - harness.ios.holdInstalls(); - - const ahead = track(harness.installer.install(ios("27.0"))); - const waiting = track(harness.installer.install(ios("28.0"))); - await flush(); - (harness.clock as ManualClock).fire(1); - await flush(); - expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); - harness.ios.releaseInstalls(); - await flush(); - - expect(ahead.result()).toEqual({ outcome: "installed", version: "27.0" }); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.installed", - ]); - }); - - it("starts no install for a waiting install whose only call timed out", async () => { - const harness = await createHarness({ clock: new ManualClock() }); - harness.ios.holdInstalls(); - - void harness.installer.install(ios("27.0")); - void harness.installer.install(ios("28.0")).catch(() => undefined); - await flush(); - (harness.clock as ManualClock).fire(1); - await flush(); - harness.ios.releaseInstalls(); - await flush(); - - expect(installs(harness.ios)).toEqual(["27.0"]); - }); - - it("gives the disk back when an install ends, so the next install that fits alone starts", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - componentFootprint: { bytes: 6 * gibibyte, path: "/volume" }, - platform: "ios", - }), - ], - freeDiskBytes: 10 * gibibyte, - }); - harness.ios.failOn("installComponent", 1, new DriverCrashError("xcodebuild failed")); - - await expect(harness.installer.install(ios("27.0"))).rejects.toThrow("xcodebuild failed"); - await expect(harness.installer.install(ios("28.0"))).resolves.toMatchObject({ - outcome: "installed", - }); - await expect(harness.installer.install(ios("29.0"))).resolves.toMatchObject({ - outcome: "installed", - }); - expect(harness.reservations.map((reservation) => reservation.released)).toEqual([ - true, - true, - true, - ]); - }); - - it("keeps the deadline of a call that ended as not-needed from ending the install the other calls joined", async () => { - const harness = await createHarness(); - harness.ios.holdInstalls(); - - const notNeeded = track( - harness.installer.install(ios("27.0", { stillNeeded: async () => false })), - ); - harness.clock.advance(1); - const needed = track(harness.installer.install(ios("27.0"))); - await flush(); - expect(notNeeded.result()).toEqual({ outcome: "not-needed" }); - expect(installs(harness.ios)).toEqual(["27.0"]); - - // The not-needed call's budget runs out here; the install belongs to the other call now. - harness.clock.advance(timeoutMs - 1); - await flush(); - expect(needed.settled()).toBe(false); - harness.clock.advance(1); - await flush(); - expect(needed.error()).toEqual(new ComponentInstallTimeoutError("ios", "27.0", timeoutMs)); - }); - - it("starts a new install for a call that arrives while a timed-out install's driver is still returning, after it returns", async () => { - let finishStubborn: (() => void) | undefined; - class StubbornDriver extends FakeDriver { - readonly started: string[] = []; - - override async installComponent( - component: string, - options: Parameters[1], - ) { - this.started.push(component); - if (this.started.length === 1) { - // Ignores its signal, like an installer that takes a while to die. - await new Promise((resolve) => { - finishStubborn = resolve; - }); - throw new DriverCrashError("ended late"); - } - return super.installComponent(component, options); - } - } - const harness = await createHarness({ - drivers: (clock) => [new StubbornDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - const driver = harness.ios as StubbornDriver; - - const first = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(timeoutMs); - await flush(); - expect(first.error()).toBeInstanceOf(ComponentInstallTimeoutError); - const second = track(harness.installer.install(ios("27.0"))); - await flush(); - expect(driver.started).toEqual(["27.0"]); - - finishStubborn?.(); - await flush(); - expect(driver.started).toEqual(["27.0", "27.0"]); - expect(second.result()).toEqual({ outcome: "installed", version: "27.0" }); - }); - - it("rejects only the call whose stillNeeded throws, and installs for the others joined to it", async () => { - const harness = await createHarness(); - const failure = new Error("resolve failed"); - // An install ahead keeps all three waiting, so each reaches the front with its check. - harness.ios.holdInstalls(); - void harness.installer.install(ios("28.0")); - await flush(); - harness.ios.releaseInstalls(); - - const [plain, throwing, notNeeded] = await Promise.allSettled([ - harness.installer.install(ios("27.0")), - harness.installer.install( - ios("27.0", { - stillNeeded: async () => { - throw failure; - }, - }), - ), - harness.installer.install(ios("27.0", { stillNeeded: async () => false })), - ]); - - expect(plain).toEqual({ - status: "fulfilled", - value: { outcome: "installed", version: "27.0" }, - }); - expect(throwing).toEqual({ reason: failure, status: "rejected" }); - expect(notNeeded).toEqual({ status: "fulfilled", value: { outcome: "not-needed" } }); - expect(installs(harness.ios)).toEqual(["28.0", "27.0"]); - }); - - it("keeps a throwing progress callback from reaching the install or the other calls", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: [], clock, installProgress: [50], platform: "ios" }), - ], - }); - const heard: ComponentInstallerProgress[] = []; - - const [throwing, listening] = await Promise.allSettled([ - harness.installer.install( - ios("27.0", { - onProgress: () => { - throw new Error("observer broke"); - }, - }), - ), - harness.installer.install(ios("27.0", { onProgress: (report) => heard.push(report) })), - ]); - - expect(throwing).toMatchObject({ status: "fulfilled" }); - expect(listening).toMatchObject({ status: "fulfilled" }); - expect(heard).toEqual([ - { stage: "downloading" }, - { percent: 50, stage: "downloading" }, - { percent: 100, stage: "downloading" }, - ]); - }); - - it.each([ - ["checking whether the component is installed", "findComponent"], - ["reserving disk", "diskFree"], - ] as const)( - "starts no install when the only call ran out of budget while the installer was %s", - async (_step, slow) => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - latencyMs: slow === "findComponent" ? { findComponent: timeoutMs + 1 } : {}, - platform: "ios", - }), - ], - filesystem: (clock) => ({ - diskFree: async () => { - if (slow === "diskFree") { - await new Promise((resolve) => clock.setTimer(timeoutMs + 1, resolve)); - } - return Number.MAX_SAFE_INTEGER; - }, - }), - }); - - const call = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(timeoutMs); - await flush(); - expect(call.error()).toBeInstanceOf(ComponentInstallTimeoutError); - harness.clock.advance(1); - await flush(); - - expect(installs(harness.ios)).toEqual([]); - expect(harness.events).toEqual([]); - // Nothing reserved once the call is gone; a reservation already taken is given back. - expect(harness.reservations).toEqual( - slow === "findComponent" ? [] : [{ bytes: 0, released: true }], - ); - }, - ); - - it("resolves close only once the running driver call has returned", async () => { - let finishStubborn: (() => void) | undefined; - class StubbornDriver extends FakeDriver { - override async installComponent(): ReturnType { - // Ignores its signal, like an installer that takes a while to die. - await new Promise((resolve) => { - finishStubborn = resolve; - }); - throw new DriverCrashError("ended late"); - } - } - const harness = await createHarness({ - drivers: (clock) => [new StubbornDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - - void harness.installer.install(ios("27.0")).catch(() => undefined); - await flush(); - const closing = track(harness.installer.close()); - await flush(); - expect(closing.settled()).toBe(false); - - finishStubborn?.(); - await flush(); - expect(closing.settled()).toBe(true); - }); - - it("fails only the call whose budget runs out while a finished install's record is written, and installs for the rest", async () => { - // A gate the test opens by hand, standing in for other registry writes ahead of the record. - let openGate: (() => void) | undefined; - const gate = new Promise((resolve) => { - openGate = resolve; - }); - const harness = await createHarness({ - decisions: { - run: async (operation: () => Result | Promise) => { - await gate; - return operation(); - }, - }, - }); - - const oldest = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(1); - const later = track(harness.installer.install(ios("27.0"))); - await flush(); - harness.clock.advance(timeoutMs - 1); - await flush(); - expect(oldest.error()).toEqual(new ComponentInstallTimeoutError("ios", "27.0", timeoutMs)); - expect(later.settled()).toBe(false); - - openGate?.(); - await flush(); - expect(later.result()).toEqual({ outcome: "installed", version: "27.0" }); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.installed", - ]); - }); - - it("aborts the running install and rejects the waiting calls on close", async () => { - const harness = await createHarness(); - harness.ios.holdInstalls(); - - const running = track(harness.installer.install(ios("27.0"))); - const waiting = track(harness.installer.install(ios("28.0"))); - await flush(); - await harness.installer.close(); - await flush(); - - expect(running.error()).toBeInstanceOf(ComponentInstallerClosedError); - expect(waiting.error()).toBeInstanceOf(ComponentInstallerClosedError); - expect(installs(harness.ios)).toEqual(["27.0"]); - expect(harness.events.map((event) => event.event)).toEqual([ - "component.install-started", - "component.install-failed", - ]); - await expect(harness.installer.install(ios("29.0"))).rejects.toBeInstanceOf( - ComponentInstallerClosedError, - ); - }); -}); - -describe("ComponentInstaller.inProgress", () => { - it("lists an install at the front of its queue that is still checking whether it is needed as waiting", async () => { - const harness = await createHarness(); - - void harness.installer.install(ios("27.0", { stillNeeded: () => new Promise(() => {}) })); - await flush(); - - expect(harness.installer.inProgress()).toEqual([ - { component: "27.0", platform: "ios", since: 1_000, state: "waiting", waiters: 1 }, - ]); - expect(installs(harness.ios)).toEqual([]); - }); - - it("lists an install whose driver returned and whose record is being written as downloading", async () => { - const harness = await createHarness({ - // A gate that never opens: the record write waits behind it. - decisions: { run: () => new Promise(() => {}) }, - }); - - void harness.installer.install(ios("27.0")); - await flush(); - - expect(installs(harness.ios)).toEqual(["27.0"]); - expect(harness.installer.inProgress()).toEqual([ - { component: "27.0", platform: "ios", since: 1_000, state: "downloading", waiters: 1 }, - ]); - }); - - it("does not list a waiting install once its only call has timed out", async () => { - const harness = await createHarness({ clock: new ManualClock() }); - harness.ios.holdInstalls(); - - void harness.installer.install(ios("27.0")); - const waiting = track(harness.installer.install(ios("28.0"))); - await flush(); - expect(harness.installer.inProgress().map((install) => install.component)).toEqual([ - "27.0", - "28.0", - ]); - // Timer 1 is the budget of the call for 28.0. - (harness.clock as ManualClock).fire(1); - await flush(); - - expect(waiting.error()).toEqual(new ComponentInstallTimeoutError("ios", "28.0", timeoutMs)); - expect(harness.installer.inProgress().map((install) => install.component)).toEqual(["27.0"]); - }); - - it("lists the 16 oldest installs across platforms, oldest first, when more are in progress", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: [], clock, platform: "ios" }), - new FakeDriver({ availableOsVersions: [], clock, platform: "android" }), - ], - }); - for (const driver of harness.drivers) driver.holdInstalls(); - - // Alternating platforms, so the oldest are not the first ones of any one queue. - for (let index = 0; index < 20; index += 1) { - const platform = index % 2 === 0 ? "ios" : "android"; - void harness.installer.install({ component: `c${String(index)}`, platform }); - harness.clock.advance(1_000); - } - await flush(); - - expect(harness.installer.inProgress().map((install) => install.component)).toEqual( - Array.from({ length: 16 }, (_, index) => `c${String(index)}`), - ); - }); -}); - -/** A fake whose `listComponents` answers with exactly what a test lists. */ -class ListingDriver extends FakeDriver { - constructor( - clock: FakeClock, - platform: "ios" | "android", - private readonly listing: readonly DriverComponent[], - ) { - super({ availableOsVersions: [], clock, platform }); - } - - override listComponents(): Promise { - return Promise.resolve(this.listing); - } -} - -async function addDevice( - registry: Registry, - platform: "ios" | "android", - osVersion: string, -): Promise { - const driverDeviceId = `${platform}-${osVersion}-${String(registry.snapshot.devices.length)}`; - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: driverDeviceId }, - driverDeviceId, - provisionDuration: 0, - spec: { physical: false, model: "Phone", osVersion, platform }, - }); - return device.id; -} - -describe("ComponentInstaller.list", () => { - it("lists a component Simlock installed with installedBySimlock true and the time its record was written", async () => { - const harness = await createHarness({ - drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - harness.clock.advance(4_000); - - await harness.installer.install(ios("27.0")); - - expect(await harness.installer.list()).toEqual([ - { - devices: 0, - foreignDevices: 0, - installedAt: 5_000, - installedBySimlock: true, - platform: "ios", - version: "27.0", - }, - ]); - }); - - it("lists a component that was on the machine before with installedBySimlock false and no installedAt", async () => { - const harness = await createHarness(); - - expect(await harness.installer.list()).toEqual([ - { - devices: 0, - foreignDevices: 0, - installedBySimlock: false, - platform: "ios", - version: "26.5", - }, - ]); - }); - - it("marks nothing as Simlock's for a record of that platform and version whose receipt no longer equals the installed one", async () => { - const harness = await createHarness(); - await harness.registry.recordComponent({ - installedAt: 1_000, - platform: "ios", - receipt: { install: "1", version: "26.5" }, - version: "26.5", - }); - - const [listed] = await harness.installer.list(); - - expect(listed).toMatchObject({ installedBySimlock: false, version: "26.5" }); - expect(listed).not.toHaveProperty("installedAt"); - }); - - it("does not take a record of another platform with an equal receipt as proof", async () => { - const harness = await createHarness(); - await harness.registry.recordComponent({ - installedAt: 1_000, - platform: "android", - receipt: { preinstalled: "26.5" }, - version: "26.5", - }); - - const [listed] = await harness.installer.list(); - - expect(listed).toMatchObject({ installedBySimlock: false, platform: "ios" }); - }); - - it("counts registry devices of that platform and version, and not deleted ones", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: ["26.5", "27.0"], clock, platform: "ios" }), - ], - }); - await addDevice(harness.registry, "ios", "26.5"); - await addDevice(harness.registry, "ios", "26.5"); - const deleted = await addDevice(harness.registry, "ios", "26.5"); - await harness.registry.markDeviceMissing(deleted, "test"); - await addDevice(harness.registry, "ios", "27.0"); - // Same version string on the other platform: not a device of this component. - await addDevice(harness.registry, "android", "26.5"); - - const listed = await harness.installer.list(); - - expect(listed.map(({ devices, version }) => ({ devices, version }))).toEqual([ - { devices: 2, version: "26.5" }, - { devices: 1, version: "27.0" }, - ]); - }); - - it("shows the devices of a version on both of its variants", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new ListingDriver(clock, "android", [ - { - foreignDevices: 0, - receipt: { image: "a" }, - variant: "google_apis/arm64-v8a", - version: "35", - }, - { - foreignDevices: 0, - receipt: { image: "b" }, - variant: "default/arm64-v8a", - version: "35", - }, - ]), - ], - }); - await addDevice(harness.registry, "android", "35"); - await addDevice(harness.registry, "android", "35"); - - const listed = await harness.installer.list(); - - expect(listed.map(({ devices, variant }) => ({ devices, variant }))).toEqual([ - { devices: 2, variant: "default/arm64-v8a" }, - { devices: 2, variant: "google_apis/arm64-v8a" }, - ]); - }); - - it("carries the driver's size, variant and foreign devices through unread", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new ListingDriver(clock, "ios", [ - { - foreignDevices: 3, - receipt: { image: "x" }, - sizeBytes: 9 * gibibyte, - variant: "23A343", - version: "26.0", - }, - ]), - ], - }); - - expect(await harness.installer.list()).toEqual([ - { - devices: 0, - foreignDevices: 3, - installedBySimlock: false, - platform: "ios", - sizeBytes: 9 * gibibyte, - variant: "23A343", - version: "26.0", - }, - ]); - }); - - it("orders by platform, then version as numbers, then variant", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new ListingDriver(clock, "ios", [ - { foreignDevices: 0, receipt: { r: "1" }, version: "26.4" }, - { foreignDevices: 0, receipt: { r: "2" }, version: "9.3" }, - ]), - new ListingDriver(clock, "android", [ - { foreignDevices: 0, receipt: { r: "3" }, variant: "z", version: "35" }, - { foreignDevices: 0, receipt: { r: "4" }, variant: "a", version: "35" }, - { foreignDevices: 0, receipt: { r: "5" }, variant: "a", version: "9" }, - ]), - ], - }); - - const listed = await harness.installer.list(); - - expect(listed.map(({ platform, variant, version }) => [platform, version, variant])).toEqual([ - ["android", "9", "a"], - ["android", "35", "a"], - ["android", "35", "z"], - ["ios", "9.3", undefined], - ["ios", "26.4", undefined], - ]); - }); - - it("asks only the named platform's driver when given a platform", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), - new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }), - ], - }); - - const listed = await harness.installer.list("android"); - - expect(listed.map((component) => component.platform)).toEqual(["android"]); - expect(harness.ios.calls.some((call) => call.operation === "listComponents")).toBe(false); - }); - - it("leaves out a driver whose listing rejects, logs why, and still lists the other platform", async () => { - const warnings: { readonly message: string; readonly fields: unknown }[] = []; - const logger: Logger = { - child: () => logger, - debug: () => {}, - error: () => {}, - info: () => {}, - warn: (message, fields) => { - warnings.push({ fields, message }); - }, - }; - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), - new FakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }), - ], - logger, - }); - harness.ios.failOn("listComponents", 1, new DriverCrashError("simctl runtime list failed")); - - const listed = await harness.installer.list(); - - expect(listed.map((component) => [component.platform, component.version])).toEqual([ - ["android", "35"], - ]); - expect(warnings).toEqual([ - { - fields: { error: "DriverCrashError: simctl runtime list failed", platform: "ios" }, - message: "A driver could not list its installed components", - }, - ]); - }); - - it("starts no install, takes no reservation and emits no event", async () => { - const harness = await createHarness(); - - await harness.installer.list(); - - expect(installs(harness.ios)).toEqual([]); - expect(harness.reservations).toEqual([]); - expect(harness.events).toEqual([]); - }); -}); - -describe("ComponentInstaller.remove", () => { - /** A harness whose iOS driver starts with nothing installed, then has `27.0` installed by Simlock. */ - async function withSimlockInstall( - options: { - readonly componentSizes?: Record; - readonly foreignDevices?: Record; - } = {}, - ) { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ availableOsVersions: [], clock, platform: "ios", ...options }), - ], - }); - await harness.installer.install(ios("27.0")); - harness.events.length = 0; - return harness; - } - - function removals(driver: FakeDriver): readonly unknown[] { - return driver.calls - .filter((call) => call.operation === "removeComponent") - .map((call) => call.arguments[0]); - } - - const removeIos = (version: string, extra: { readonly dryRun?: boolean } = {}) => ({ - platform: "ios" as const, - requesterId: "tok_admin", - version, - ...extra, - }); - - it("removes a component with a record and no users, deletes its record, and list no longer shows it", async () => { - const harness = await withSimlockInstall({ componentSizes: { "27.0": 7_000_000_000 } }); - const [record] = harness.registry.snapshot.components; - - await expect(harness.installer.remove(removeIos("27.0"))).resolves.toEqual({ - outcome: "removed", - sizeBytes: 7_000_000_000, - }); - - expect(removals(harness.ios)).toEqual([record?.receipt]); - expect(harness.registry.snapshot.components).toEqual([]); - expect(await harness.installer.list()).toEqual([]); - expect(harness.events).toEqual([ - { - event: "component.removed", - payload: { - componentId: "27.0", - platform: "ios", - requesterId: "tok_admin", - sizeBytes: 7_000_000_000, - version: "27.0", - }, - }, - ]); - }); - - it("carries the driver's residue into the answer and the event", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - platform: "ios", - removalResidue: "the download stays", - }), - ], - }); - await harness.installer.install(ios("27.0")); - - await expect(harness.installer.remove(removeIos("27.0"))).resolves.toEqual({ - outcome: "removed", - residue: "the download stays", - }); - expect(harness.events.at(-1)).toMatchObject({ - event: "component.removed", - payload: { residue: "the download stays" }, - }); - }); - - it("refuses a component with no record with ComponentNotOwnedError and never calls removeComponent", async () => { - // 26.5 is on the machine, but Simlock did not install it. - const harness = await createHarness(); - - await expect(harness.installer.remove(removeIos("26.5"))).rejects.toBeInstanceOf( - ComponentNotOwnedError, - ); - expect(removals(harness.ios)).toEqual([]); - expect(await harness.installer.list()).toMatchObject([{ version: "26.5" }]); - }); - - it("refuses a record whose version is no longer installed saying none is installed, and a record whose version was installed again saying that one is not Simlock's, in a removal and a dry run alike", async () => { - // 26.5 is on the machine with another receipt; 25.0 is not on the machine at all. - const harness = await createHarness(); - for (const version of ["25.0", "26.5"]) { - await harness.registry.recordComponent({ - installedAt: 1_000, - platform: "ios", - receipt: { install: "1", version }, - version, - }); - } - - for (const dryRun of [false, true]) { - await expect(harness.installer.remove(removeIos("25.0", { dryRun }))).rejects.toMatchObject({ - message: "No ios 25.0 is installed now, so there is nothing to remove", - name: "ComponentNotOwnedError", - }); - await expect(harness.installer.remove(removeIos("26.5", { dryRun }))).rejects.toMatchObject({ - message: - "The ios 26.5 installed now is not the one Simlock installed, so Simlock will not remove it", - name: "ComponentNotOwnedError", - }); - } - expect(removals(harness.ios)).toEqual([]); - expect(harness.registry.snapshot.components).toContainEqual( - expect.objectContaining({ platform: "ios", version: "26.5" }), - ); - }); - - it("removes a component whose receipt matches the record even when the listing names its version differently", async () => { - // The driver lists 26.5 with the receipt Simlock recorded under 26.5.0. - const harness = await createHarness(); - const receipt = { preinstalled: "26.5" }; - await harness.registry.recordComponent({ - installedAt: 1_000, - platform: "ios", - receipt, - version: "26.5.0", - }); - - await expect(harness.installer.remove(removeIos("26.5.0"))).resolves.toMatchObject({ - outcome: "removed", - }); - expect(removals(harness.ios)).toEqual([receipt]); - }); - - it.each(["provisioning", "ready", "leased", "reclaiming", "quarantined", "shutdown"] as const)( - "refuses a component a %s registry device uses with ComponentInUseError naming the count, and never calls removeComponent", - async (state) => { - const device: DeviceRecord = { - physical: false, - createdAt: 0, - driverData: {}, - driverDeviceId: "udid-1", - id: "dev_1", - mode: "full", - spec: { physical: false, model: "Phone", osVersion: "27.0", platform: "ios" }, - state, - }; - const harness = await withSimlockInstall(); - // The same installer's view, with one device of that version in the registry. - const installer = new ComponentInstaller({ - clock: harness.clock, - decisions: new SerializedDecision(), - diskSpace: new DiskSpaceGuard(), - drivers: new DriverCatalog(harness.drivers), - eventBus: harness.bus, - filesystem: new MemoryFilesystem(), - registry: { - deleteComponent: (platform, version) => - harness.registry.deleteComponent(platform, version), - recordComponent: (record) => harness.registry.recordComponent(record), - get snapshot() { - return { ...harness.registry.snapshot, devices: [device], virtualDevices: [device] }; - }, - }, - timeoutMs, - }); - - const error = await installer.remove(removeIos("27.0")).catch((e: unknown) => e); - - expect(error).toBeInstanceOf(ComponentInUseError); - expect(error).toMatchObject({ devices: 1, foreignDevices: 0 }); - expect(removals(harness.ios)).toEqual([]); - expect(harness.registry.snapshot.components).toHaveLength(1); - }, - ); - - it("does not count a physical device on the runtime's OS as using it, so removal is not refused for it", async () => { - const physical: DeviceRecord = { - createdAt: 0, - driverData: {}, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: [], - id: "dev_physical", - physical: true, - spec: { - class: "phone", - model: "iPhone 15 Pro", - osVersion: "27.0", - physical: true, - platform: "ios", - }, - state: "ready", - }; - const harness = await withSimlockInstall(); - const installer = new ComponentInstaller({ - clock: harness.clock, - decisions: new SerializedDecision(), - diskSpace: new DiskSpaceGuard(), - drivers: new DriverCatalog(harness.drivers), - eventBus: harness.bus, - filesystem: new MemoryFilesystem(), - registry: { - deleteComponent: (platform, version) => harness.registry.deleteComponent(platform, version), - recordComponent: (record) => harness.registry.recordComponent(record), - get snapshot() { - return { ...harness.registry.snapshot, devices: [physical], virtualDevices: [] }; - }, - }, - timeoutMs, - }); - - await expect(installer.remove(removeIos("27.0"))).resolves.toMatchObject({ - outcome: "removed", - }); - expect(removals(harness.ios)).toHaveLength(1); - }); - - it("refuses a component a foreign device uses with ComponentInUseError naming the count, and never calls removeComponent", async () => { - const harness = await withSimlockInstall({ foreignDevices: { "27.0": 2 } }); - - const error = await harness.installer.remove(removeIos("27.0")).catch((e: unknown) => e); - - expect(error).toBeInstanceOf(ComponentInUseError); - expect(error).toMatchObject({ devices: 0, foreignDevices: 2 }); - expect(removals(harness.ios)).toEqual([]); - }); - - it("refuses, and never calls removeComponent, when the driver's listing rejects", async () => { - const harness = await withSimlockInstall(); - harness.ios.failOn("listComponents", 1, new DriverCrashError("default set unreadable")); - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( - "default set unreadable", - ); - expect(removals(harness.ios)).toEqual([]); - expect(harness.registry.snapshot.components).toHaveLength(1); - }); - - it("refuses with ComponentInUseError while a device is being provisioned on it and has no record yet", async () => { - const harness = await withSimlockInstall(); - const release = harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toMatchObject({ - devices: 1, - foreignDevices: 0, - }); - await expect( - harness.installer.remove(removeIos("27.0", { dryRun: true })), - ).rejects.toMatchObject({ devices: 1, foreignDevices: 0, name: "ComponentInUseError" }); - expect(removals(harness.ios)).toEqual([]); - - release(); - await expect(harness.installer.remove(removeIos("27.0"))).resolves.toMatchObject({ - outcome: "removed", - }); - }); - - it("answers a dry run with would-remove and the size, calls no driver verb that changes anything, and keeps the record", async () => { - const harness = await withSimlockInstall({ componentSizes: { "27.0": 7_000_000_000 } }); - const before = harness.ios.calls.length; - - await expect(harness.installer.remove(removeIos("27.0", { dryRun: true }))).resolves.toEqual({ - outcome: "would-remove", - sizeBytes: 7_000_000_000, - }); - - expect(harness.ios.calls.slice(before).map((call) => call.operation)).toEqual([ - "listComponents", - ]); - expect(harness.registry.snapshot.components).toHaveLength(1); - expect(harness.events).toEqual([]); - expect(await harness.installer.list()).toMatchObject([ - { installedBySimlock: true, version: "27.0" }, - ]); - }); - - it.each([false, true])( - "refuses with ComponentBusyError while an install is running on that platform (dry run: %s)", - async (dryRun) => { - const harness = await withSimlockInstall(); - harness.ios.holdInstalls(); - const install = track(harness.installer.install(ios("28.0"))); - await flush(); - expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); - - await expect(harness.installer.remove(removeIos("27.0", { dryRun }))).rejects.toBeInstanceOf( - ComponentBusyError, - ); - expect(removals(harness.ios)).toEqual([]); - - harness.ios.releaseInstalls(); - await flush(); - expect(install.result()).toMatchObject({ outcome: "installed" }); - }, - ); - - it("refuses with ComponentBusyError while an install waits on that platform, before its driver runs", async () => { - const harness = await withSimlockInstall(); - // At the front of the queue, still checking whether it is needed: waiting, not downloading. - void harness.installer - .install(ios("28.0", { stillNeeded: () => new Promise(() => undefined) })) - .catch(() => undefined); - await flush(); - expect(harness.installer.inProgress()).toMatchObject([{ component: "28.0", state: "waiting" }]); - expect(installs(harness.ios)).toEqual(["27.0"]); - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toBeInstanceOf( - ComponentBusyError, - ); - expect(removals(harness.ios)).toEqual([]); - }); - - it("counts Simlock's devices again inside the decision gate, and refuses one claimed after the listing", async () => { - const decisions = new SerializedDecision(); - const harness = await createHarness({ - decisions, - drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - await harness.installer.install(ios("27.0")); - // Something else holds the gate, so the removal's listing and first count pass before it - // gets its turn there. - let releaseGate!: () => void; - void decisions.run( - () => - new Promise((resolve) => { - releaseGate = resolve; - }), - ); - const removal = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - expect(harness.ios.calls.filter((call) => call.operation === "listComponents")).toHaveLength(1); - expect(removal.settled()).toBe(false); - - harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); - releaseGate(); - await flush(); - - expect(removal.error()).toBeInstanceOf(ComponentInUseError); - expect(removal.error()).toMatchObject({ devices: 1 }); - expect(removals(harness.ios)).toEqual([]); - }); - - it("starts an install that arrives during a removal only after the removal has ended", async () => { - const harness = await withSimlockInstall(); - harness.ios.holdRemovals(); - const removal = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - expect(removals(harness.ios)).toHaveLength(1); - - const progress: ComponentInstallerProgress[] = []; - const install = track( - harness.installer.install(ios("28.0", { onProgress: (heard) => progress.push(heard) })), - ); - await flush(); - expect(progress).toEqual([{ stage: "waiting" }]); - expect(installs(harness.ios)).toEqual(["27.0"]); - expect(harness.ios.calls.filter((call) => call.operation === "findComponent")).toHaveLength(1); - expect(harness.installer.inProgress()).toMatchObject([{ component: "28.0", state: "waiting" }]); - - harness.ios.releaseRemovals(); - await flush(); - - expect(removal.result()).toMatchObject({ outcome: "removed" }); - expect(installs(harness.ios)).toEqual(["27.0", "28.0"]); - expect(install.result()).toMatchObject({ outcome: "installed", version: "28.0" }); - }); - - it("refuses a second removal on the platform while one runs with ComponentBusyError", async () => { - const harness = await createHarness({ - drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], - }); - await harness.installer.install(ios("27.0")); - await harness.installer.install(ios("28.0")); - harness.ios.holdRemovals(); - const first = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - - await expect(harness.installer.remove(removeIos("28.0"))).rejects.toBeInstanceOf( - ComponentBusyError, - ); - - harness.ios.releaseRemovals(); - await flush(); - expect(first.result()).toMatchObject({ outcome: "removed" }); - }); - - it("refuses every device claimed on the component while it is being removed, and none once the removal has settled", async () => { - const harness = await withSimlockInstall(); - harness.ios.holdRemovals(); - const removal = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - - expect(() => harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" })).toThrow( - ComponentBeingRemovedError, - ); - // Another version is not being removed. - expect(() => - harness.installer.claimProvision({ osVersion: "26.0", platform: "ios" }), - ).not.toThrow(); - - harness.ios.releaseRemovals(); - await flush(); - expect(removal.settled()).toBe(true); - expect(() => - harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }), - ).not.toThrow(); - }); - - it("keeps the record, clears the mark and emits no component.removed when the driver fails the removal", async () => { - const harness = await withSimlockInstall(); - harness.ios.failOn("removeComponent", 1, new DriverCrashError("simctl runtime delete failed")); - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( - "simctl runtime delete failed", - ); - - expect(harness.registry.snapshot.components).toMatchObject([{ version: "27.0" }]); - expect(harness.events).toEqual([]); - expect(() => - harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }), - ).not.toThrow(); - // The turn is back: an install on the platform starts. - await expect(harness.installer.install(ios("28.0"))).resolves.toMatchObject({ - outcome: "installed", - }); - }); - - it("refuses with ComponentInstallerClosedError, removing nothing, when the daemon stops while the listing is read", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - latencyMs: { listComponents: 10 }, - platform: "ios", - }), - ], - }); - await harness.installer.install(ios("27.0")); - const removal = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - expect(harness.ios.calls.filter((call) => call.operation === "listComponents")).toHaveLength(1); - - await harness.installer.close(); - harness.clock.advance(10); - await flush(); - - expect(removal.error()).toBeInstanceOf(ComponentInstallerClosedError); - expect(removals(harness.ios)).toEqual([]); - expect(harness.registry.snapshot.components).toHaveLength(1); - }); - - it("ends one claim per release however often that release is called", async () => { - const harness = await withSimlockInstall(); - const first = harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); - harness.installer.claimProvision({ osVersion: "27.0", platform: "ios" }); - - first(); - first(); - - expect(await harness.installer.list()).toMatchObject([{ devices: 1, version: "27.0" }]); - }); - - it("refuses with ComponentInstallerClosedError once the daemon has stopped, without reading the listing", async () => { - const harness = await withSimlockInstall(); - await harness.installer.close(); - const before = harness.ios.calls.length; - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toBeInstanceOf( - ComponentInstallerClosedError, - ); - expect(harness.ios.calls.slice(before)).toEqual([]); - }); - - it("resolves close only once a running removal's driver call has returned", async () => { - const harness = await createHarness({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - // A delay the abort does not cut short, as a platform tool that is slow to exit. - latencyMs: { removeComponent: 10 }, - platform: "ios", - }), - ], - }); - await harness.installer.install(ios("27.0")); - void harness.installer.remove(removeIos("27.0")).catch(() => undefined); - await flush(); - expect(removals(harness.ios)).toHaveLength(1); - - const closing = track(harness.installer.close()); - await flush(); - expect(closing.settled()).toBe(false); - - harness.clock.advance(10); - await flush(); - expect(closing.settled()).toBe(true); - }); - - it("logs a failed removal with who asked, after the driver was called", async () => { - const warnings: { readonly message: string; readonly fields: unknown }[] = []; - const logger: Logger = { - child: () => logger, - debug: () => {}, - error: () => {}, - info: () => {}, - warn: (message, fields) => { - warnings.push({ fields, message }); - }, - }; - const harness = await createHarness({ - drivers: (clock) => [new FakeDriver({ availableOsVersions: [], clock, platform: "ios" })], - logger, - }); - await harness.installer.install(ios("27.0")); - harness.ios.failOn("removeComponent", 1, new DriverCrashError("simctl runtime delete failed")); - - await expect(harness.installer.remove(removeIos("27.0"))).rejects.toThrow( - "simctl runtime delete failed", - ); - - expect(warnings).toEqual([ - { - fields: { - error: "DriverCrashError: simctl runtime delete failed", - platform: "ios", - requesterId: "tok_admin", - version: "27.0", - }, - message: "A component removal failed; Simlock's record of it is kept", - }, - ]); - }); - - it("ends a running removal on close with ComponentInstallerClosedError, and keeps the record", async () => { - const harness = await withSimlockInstall(); - harness.ios.holdRemovals(); - const removal = track(harness.installer.remove(removeIos("27.0"))); - await flush(); - - await harness.installer.close(); - await flush(); - - expect(removal.error()).toBeInstanceOf(ComponentInstallerClosedError); - expect(harness.registry.snapshot.components).toHaveLength(1); - expect(harness.events).toEqual([]); - }); }); diff --git a/src/core/test-component-installer.ts b/src/core/test-component-installer.ts new file mode 100644 index 000000000..cac0fb79a --- /dev/null +++ b/src/core/test-component-installer.ts @@ -0,0 +1,147 @@ +import { EventBus } from "../bus/index.js"; +import { + FakeClock, + type Filesystem, + type Logger, + MemoryFilesystem, + type TimerHandle, +} from "../ports/index.js"; +import { ComponentInstaller, type ComponentInstallRequest } from "./component-installer.js"; +import { DiskSpaceGuard } from "./driver.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import { FakeDriver } from "./fake-driver.js"; +import { loadTestRegistry, openTestState } from "./testing.js"; +import { SerializedDecision } from "./serialized-decision.js"; + +export const gibibyte = 1024 ** 3; +export const timeoutMs = 60_000; + +/** A clock whose timers fire only when a test fires them, one at a time, by creation order. */ +export class ManualClock extends FakeClock { + readonly #timers: { callback: () => void; cancelled: boolean }[] = []; + + override setTimer(_delayMs: number, callback: () => void): TimerHandle { + const timer = { callback, cancelled: false }; + this.#timers.push(timer); + return timer as unknown as TimerHandle; + } + + override cancel(handle: TimerHandle): void { + (handle as unknown as { cancelled: boolean }).cancelled = true; + } + + fire(index: number): void { + const timer = this.#timers[index]; + if (timer === undefined || timer.cancelled) throw new Error(`No live timer ${String(index)}`); + timer.cancelled = true; + timer.callback(); + } +} + +export async function createHarness( + options: { + readonly clock?: FakeClock; + readonly drivers?: (clock: FakeClock) => readonly FakeDriver[]; + readonly decisions?: Pick; + readonly filesystem?: (clock: FakeClock) => Pick; + readonly freeDiskBytes?: number; + readonly logger?: Logger; + } = {}, +) { + const clock = options.clock ?? new FakeClock(1_000); + const bus = new EventBus(clock); + const events: { readonly event: string; readonly payload: unknown }[] = []; + bus.subscribeAll((envelope) => { + if (envelope.event.startsWith("component.")) { + events.push({ event: envelope.event, payload: envelope.payload }); + } + }); + const drivers = options.drivers?.(clock) ?? [ + new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }), + ]; + const state = await openTestState(); + let nextId = 0; + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => `id${String((nextId += 1))}` }, + state, + }); + // Every reservation the installer takes, and whether it gave it back. + const guard = new DiskSpaceGuard(); + const reservations: { readonly bytes: number; released: boolean }[] = []; + const installer = new ComponentInstaller({ + clock, + decisions: options.decisions ?? new SerializedDecision(), + diskSpace: { + reserve: async (filesystem, platform, bytes, path) => { + const release = await guard.reserve(filesystem, platform, bytes, path); + const reservation = { bytes, released: false }; + reservations.push(reservation); + return () => { + reservation.released = true; + release(); + }; + }, + }, + drivers: new DriverCatalog(drivers), + eventBus: bus, + filesystem: options.filesystem?.(clock) ?? new MemoryFilesystem(options.freeDiskBytes), + ...(options.logger === undefined ? {} : { logger: options.logger }), + registry, + timeoutMs, + }); + const ios = drivers.find((driver) => driver.platform === "ios") as FakeDriver; + return { + bus, + clock, + drivers, + events, + installer, + ios, + registry, + reservations, + state, + }; +} + +export function installs(driver: FakeDriver): readonly unknown[] { + return driver.calls + .filter((call) => call.operation === "installComponent") + .map((call) => call.arguments[0]); +} + +/** Lets every settled promise and queued continuation run. */ +export async function flush(): Promise { + for (let count = 0; count < 100; count += 1) { + await Promise.resolve(); + } +} + +/** A promise's outcome without awaiting it, so a test can look while it is still open. */ +export function track(promise: Promise): { + readonly settled: () => boolean; + readonly result: () => T | undefined; + readonly error: () => unknown; +} { + let state: { kind: "open" } | { kind: "ok"; value: T } | { kind: "error"; error: unknown } = { + kind: "open", + }; + promise.then( + (value) => { + state = { kind: "ok", value }; + }, + (error: unknown) => { + state = { error, kind: "error" }; + }, + ); + return { + error: () => (state.kind === "error" ? state.error : undefined), + result: () => (state.kind === "ok" ? state.value : undefined), + settled: () => state.kind !== "open", + }; +} + +export function ios(component: string, extra: Partial = {}) { + return { component, platform: "ios" as const, ...extra }; +} From 1f1c9d75061b4e16d5a7e0ed7ad514e6dc6ef0f8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:31:48 +0100 Subject: [PATCH 44/83] test(daemon): split main.test.ts by topic --- src/daemon/main-history.test.ts | 130 ++ src/daemon/main-leasing.test.ts | 538 +++++++ src/daemon/main-start-daemon.test.ts | 663 ++++++++ src/daemon/main-state-db.test.ts | 353 +++++ src/daemon/main-worker-lock.test.ts | 245 +++ src/daemon/main.test.ts | 2174 ++------------------------ src/daemon/test-main-history.ts | 23 + src/daemon/test-main-state.ts | 16 + 8 files changed, 2121 insertions(+), 2021 deletions(-) create mode 100644 src/daemon/main-history.test.ts create mode 100644 src/daemon/main-leasing.test.ts create mode 100644 src/daemon/main-start-daemon.test.ts create mode 100644 src/daemon/main-state-db.test.ts create mode 100644 src/daemon/main-worker-lock.test.ts create mode 100644 src/daemon/test-main-history.ts create mode 100644 src/daemon/test-main-state.ts diff --git a/src/daemon/main-history.test.ts b/src/daemon/main-history.test.ts new file mode 100644 index 000000000..6034f28b1 --- /dev/null +++ b/src/daemon/main-history.test.ts @@ -0,0 +1,130 @@ +import { hasEvent, historyEventsWhen } from "./test-main-history.js"; +import { mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { seedHistory } from "../history/testing.js"; +import { start, stopDaemons, temporaryDirectories, writeModule } from "./test-main.js"; + +afterEach(stopDaemons); + +describe("the event history", () => { + it("prunes at start by the configured eventLog.retention, so an event older than it on the injected clock is gone", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); + temporaryDirectories.push(directory); + // why: the fake clock reads 1_000; with a retention of 100 the cutoff is 900, past 500 and before 950. + await seedHistory(directory, [500, 950]); + + await start({ configOverrides: { eventLog: { retention: 100 } }, dataDirectory: directory }); + + const events = await historyEventsWhen(directory, hasEvent("daemon.started")); + expect(events.map((event) => event.id)).not.toContain("evt_1"); + expect(events.map((event) => event.id)).toContain("evt_2"); + }); +}); + +describe("startDaemon event history", () => { + const previousModule = process.env.SIMLOCK_DRIVERS_MODULE; + + afterEach(() => { + if (previousModule === undefined) { + delete process.env.SIMLOCK_DRIVERS_MODULE; + } else { + process.env.SIMLOCK_DRIVERS_MODULE = previousModule; + } + }); + + it("starts a daemon whose history path is a directory, sets it aside and writes a fresh file", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); + temporaryDirectories.push(directory); + await mkdir(join(directory, "history.db")); + + const { daemon } = await start({ dataDirectory: directory }); + + const events = await historyEventsWhen(directory, hasEvent("daemon.started")); + const names = await readdir(directory); + const status = (await daemon.dispatch( + "status.get", + {}, + { + manageEventSubscription: () => undefined, + principal: "operator", + role: "admin", + via: "socket", + }, + )) as { readonly history?: unknown }; + expect(daemon.health).toBe("running"); + expect(names).toContain("history.db.damaged-1000"); + expect((await stat(join(directory, "history.db.damaged-1000"))).isDirectory()).toBe(true); + expect((await stat(join(directory, "history.db"))).isFile()).toBe(true); + expect(events.length).toBeGreaterThan(0); + expect(status.history).toEqual({ opened: true, setAsideAt: 1_000, writing: true }); + }); + + it("a daemon that loses the socket claim leaves a damaged history.db byte-identical and in place", async () => { + const winner = await mkdtemp(join(tmpdir(), "simlock-main-claim-")); + const loser = await mkdtemp(join(tmpdir(), "simlock-main-claim-")); + temporaryDirectories.push(winner, loser); + const junk = "this is not a sqlite database ".repeat(20); + await writeFile(join(loser, "history.db"), junk); + const socketPath = join(winner, "daemon.sock"); + await start({ dataDirectory: winner, socketPath }); + + const outcome = await start({ dataDirectory: loser, socketPath }).then( + () => "resolved" as const, + () => "rejected" as const, + ); + + expect(outcome).toBe("rejected"); + expect( + await readdir(loser).then((names) => names.filter((name) => name.startsWith("history"))), + ).toEqual(["history.db"]); + expect(await readFile(join(loser, "history.db"), "utf8")).toBe(junk); + }); + + it("mints event ids from the idGenerator the daemon was given", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); + temporaryDirectories.push(directory); + process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( + `export function createDrivers({ eventBus }) { + eventBus.emit("component.installed", { componentId: "18.6", durationMs: 1, platform: "ios" }, "test"); + return []; + }`, + ); + + await start({ + dataDirectory: directory, + drivers: undefined, + idGenerator: { generate: () => "from-the-given-generator" }, + }); + + const ids = ( + await historyEventsWhen(directory, (events) => + events.some((event) => event.id === "evt_from-the-given-generator"), + ) + ).map((event) => event.id); + expect(ids).toContain("evt_from-the-given-generator"); + }); + + it("keeps component.installed and device.slimmed in the event history", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); + temporaryDirectories.push(directory); + process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( + `export function createDrivers({ eventBus }) { + eventBus.emit("component.installed", { componentId: "18.6", durationMs: 1, platform: "ios" }, "test"); + eventBus.emit("device.slimmed", { + address: "a", categories: [], deviceId: "d", durationMs: 1, labelCount: 0, + platform: "ios", signature: "s", unknownLabels: [], + }, "test"); + return []; + }`, + ); + + await start({ dataDirectory: directory, drivers: undefined }); + + const written = (await historyEventsWhen(directory, hasEvent("device.slimmed"))).map( + (event) => event.event, + ); + expect(written).toEqual(expect.arrayContaining(["component.installed", "device.slimmed"])); + }); +}); diff --git a/src/daemon/main-leasing.test.ts b/src/daemon/main-leasing.test.ts new file mode 100644 index 000000000..15171694d --- /dev/null +++ b/src/daemon/main-leasing.test.ts @@ -0,0 +1,538 @@ +import { hasEvent, historyEventsWhen } from "./test-main-history.js"; +import { queryState } from "./test-main-state.js"; +import { existsSync } from "node:fs"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DriverCrashError, registryOwner, baselinesOwner } from "../core/index.js"; +import { + FakeDriver, + fakeDeviceDataTable, + fakeDeviceDataTables, + loadTestRegistry, +} from "../core/testing.js"; +import { openStateDatabase } from "../state/index.js"; +import { EventBus } from "../bus/index.js"; +import { + FakeClock, + FakeHostInfo, + FakeSystemStats, + JsonLinesLogger, + MemoryFilesystem, + MemoryLogSink, + NodeSqliteOpener, + NoopLogger, + ScriptedProcessRunner, +} from "../ports/index.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; +import { + STANDALONE_OFF, + runningDaemons, + start, + stopDaemons, + temporaryDirectories, +} from "./test-main.js"; + +afterEach(stopDaemons); + +describe("startDaemon wires core and leasing together", () => { + const agent = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + } as const; + const admin = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + } as const; + const ios = { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + requesterId: "agent-1", + } as const; + type Grant = { + readonly lease: { readonly id: string }; + readonly device: { readonly id: string }; + }; + + /** why: Lets the work a mutated daemon would still be doing finish, so "nothing happened" is a + * statement about the daemon and not about how early the test looked. Only sound when that + * work is all in memory: a timer fires after it however loaded the machine is, but real I/O + * (a process spawn, a real file) can outlast it. */ + const settleRealTime = async () => new Promise((resolve) => setTimeout(resolve, 25)); + + async function leasedIos(options: ConstructorParameters[0]) { + const clock = options.clock as FakeClock; + const driver = new FakeDriver(options); + const filesystem = new MemoryFilesystem(); + const started = await start({ clock, drivers: [driver], filesystem }); + const grant = (await started.daemon.dispatch("lease.request", ios, agent)) as Grant; + return { ...started, clock, driver, filesystem, grant }; + } + + it("runs the operator reset through core's nuke, which releases the lease through leasing", async () => { + const clock = new FakeClock(1_000); + const { daemon, grant } = await leasedIos({ + availableOsVersions: ["26.5"], + clock, + platform: "ios", + }); + + const report = await daemon.dispatch("nuke.run", { deleteDevices: true }, admin); + + expect(report).toEqual({ + deletedDevices: [grant.device.id], + releasedLeaseIds: [grant.lease.id], + removedBaselines: [], + }); + }); + + it("runs the operator's warm.forget through core's warm pool, which answers the kinds it forgot", async () => { + const { daemon } = await start(); + + const report = await daemon.dispatch( + "warm.forget", + { model: "iPhone 17 Pro", platform: "ios" }, + admin, + ); + + expect(report).toEqual({ forgotten: [] }); + }); + + it("grants a waiting physical request once device.add enrolls a second device of the model", async () => { + const clock = new FakeClock(1_000); + const phone = { apps: [], class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" } as const; + const driver = new FakeDriver({ + clock, + physicalDevices: [ + { ...phone, ids: ["UDID-A"] }, + { ...phone, ids: ["UDID-B"] }, + ], + platform: "ios", + }); + driver.setPhysicalPresence("UDID-A", true); + driver.setPhysicalPresence("UDID-B", true); + const { daemon } = await start({ clock, drivers: [driver] }); + const physical = { model: "iPhone 15 Pro", physical: true, platform: "ios" } as const; + await daemon.dispatch("device.add", { id: "UDID-A", platform: "ios" }, admin); + await daemon.dispatch("lease.request", { ...physical, requesterId: "agent-1" }, agent); + const waiting = daemon.dispatch( + "lease.request", + { ...physical, requesterId: "agent-2" }, + agent, + ) as Promise; + await settleRealTime(); + + await daemon.dispatch("device.add", { id: "UDID-B", platform: "ios" }, admin); + + const outcome = await Promise.race([waiting, settleRealTime().then(() => "still waiting")]); + expect(outcome).toMatchObject({ device: { physical: true } }); + }); + + it("status.get carries the presence the grant's read found on a physical device, through the daemon's own wiring", async () => { + const clock = new FakeClock(1_000); + const phone = { apps: [], class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" } as const; + const driver = new FakeDriver({ + clock, + physicalDevices: [{ ...phone, ids: ["UDID-WIRED-PRESENCE"] }], + platform: "ios", + }); + driver.setPhysicalPresence("UDID-WIRED-PRESENCE", true); + const { daemon } = await start({ clock, drivers: [driver] }); + await daemon.dispatch("device.add", { id: "UDID-WIRED-PRESENCE", platform: "ios" }, admin); + await daemon.dispatch( + "lease.request", + { model: "iPhone 15 Pro", physical: true, platform: "ios", requesterId: "agent-1" }, + agent, + ); + + const status = (await daemon.dispatch("status.get", {}, agent)) as { + devices?: { driverDeviceId?: string; presence?: unknown }[]; + }; + + const entry = status.devices?.find((device) => device.driverDeviceId === "UDID-WIRED-PRESENCE"); + expect(entry).toMatchObject({ presence: { present: true } }); + }); + + it("starts a device.exec child from the allowlisted keys of the daemon's environment plus the command's own env, without a secret the daemon inherited", async () => { + const clock = new FakeClock(1_000); + const runner = new ScriptedProcessRunner([ + { match: { args: ["list", "devices"], command: "fake-simctl" } }, + ]); + const driver = new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + passthrough: () => ({ + args: ["list", "devices"], + command: "fake-simctl", + env: { SIMLOCK_SCOPED: "1" }, + }), + passthroughTools: ["simctl"], + platform: "ios", + }); + vi.stubEnv("AWS_SECRET_ACCESS_KEY", "aws-secret"); + vi.stubEnv("LC_CTYPE", "simlock-test-locale"); + try { + const { daemon } = await start({ + clock, + drivers: [driver], + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + processRunner: runner, + }); + const grant = (await daemon.dispatch("lease.request", ios, agent)) as Grant; + + await daemon.dispatch( + "device.exec", + { args: ["list", "devices"], leaseId: grant.lease.id, tool: "simctl" }, + agent, + ); + + const env = runner.calls.find((call) => call.command === "fake-simctl")?.options.env; + expect(env).toMatchObject({ LC_CTYPE: "simlock-test-locale", SIMLOCK_SCOPED: "1" }); + expect(env?.PATH).toBe(process.env.PATH); + expect(env).not.toHaveProperty("AWS_SECRET_ACCESS_KEY"); + } finally { + vi.unstubAllEnvs(); + } + }); + + it("announces the capacity figures and then the queue depth once startup is done", async () => { + const { directory } = await start(); + + const events = await historyEventsWhen(directory, hasEvent("queue.changed")); + const names = events.map((event) => event.event); + + expect(names.indexOf("capacity.changed")).toBeGreaterThan(-1); + expect(names.indexOf("queue.changed")).toBeGreaterThan(names.indexOf("capacity.changed")); + expect(events.find((event) => event.event === "queue.changed")?.module).toBe("wait-queue"); + }); + + it("runs doctor's startup pass on the startup read before core's device steps announce the capacity figures", async () => { + const { directory } = await start(); + + const names = (await historyEventsWhen(directory, hasEvent("capacity.changed"))).map( + (event) => event.event, + ); + + expect(names.indexOf("doctor.reconciled")).toBeGreaterThan(-1); + expect(names.indexOf("doctor.reconciled")).toBeLessThan(names.indexOf("capacity.changed")); + }); + + it("marks a saved baseline whose runtime is gone as stale at start, before anything lists baselines", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); + temporaryDirectories.push(directory); + const state = await openStateDatabase({ + dataDirectory: directory, + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + oldFiles: [], + owners: [registryOwner, fakeDeviceDataTable, baselinesOwner], + sqlite: new NodeSqliteOpener(), + upgradeStep: "unused", + }); + state.database + .prepare( + "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES ('login', 1, 'ios', 'iPhone 17 Pro', '26.5', 'full', '2026-11-02T13:40:00.000Z', 'src-login')", + ) + .run(); + state.close(); + const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.setSources([ + { + id: "src-login", + kind: "baseline", + model: "iPhone 17 Pro", + osVersion: "26.5", + staleReason: reason, + }, + ]); + + const daemon = await startDaemon({ + clock, + configOverrides: STANDALONE_OFF, + dataDirectory: directory, + deviceDataTables: fakeDeviceDataTables, + drivers: [driver], + filesystem: new MemoryFilesystem(), + logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), + version: "1.2.3", + }); + runningDaemons.push(daemon); + + const stale = await historyEventsWhen(directory, hasEvent("baseline.stale")); + expect( + stale.filter((event) => event.event === "baseline.stale").map((event) => event.payload), + ).toEqual([{ name: "login", reason, version: 1 }]); + }); + + it("settles the requests a restart left open before core's device steps announce the capacity figures", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); + temporaryDirectories.push(directory); + const state = await openStateDatabase({ + dataDirectory: directory, + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + oldFiles: [], + owners: [registryOwner, fakeDeviceDataTable], + sqlite: new NodeSqliteOpener(), + upgradeStep: "unused", + }); + const previous = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "1" }, + state, + }); + await previous.createLeaseRequest({ + ownerId: "agent-1", + request: ios, + requesterId: "agent-1", + }); + state.close(); + + const daemon = await startDaemon({ + clock, + configOverrides: STANDALONE_OFF, + dataDirectory: directory, + deviceDataTables: fakeDeviceDataTables, + drivers: [new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" })], + filesystem: new MemoryFilesystem(), + logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), + version: "1.2.3", + }); + runningDaemons.push(daemon); + + const names = (await historyEventsWhen(directory, hasEvent("capacity.changed"))).map( + (event) => event.event, + ); + expect(names.indexOf("lease.rejected")).toBeGreaterThan(-1); + expect(names.indexOf("capacity.changed")).toBeGreaterThan(names.indexOf("lease.rejected")); + }); + + it("ends the lease whose device is not running before core's device steps announce capacity, so a restart no longer holds it", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); + temporaryDirectories.push(directory); + const options = () => + ({ + clock, + configOverrides: STANDALONE_OFF, + dataDirectory: directory, + deviceDataTables: fakeDeviceDataTables, + drivers: [new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" })], + filesystem: new MemoryFilesystem(), + logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), + // why: capacity follows the host's cpus and memory, and a small runner has too few slots for three leases at once, so a later request waits for ever on the fake clock (30 s timeout in runs 39459769408 and 39457371326). + systemStats: new FakeSystemStats({ cpuCount: 16, totalRamBytes: 64 * 1024 ** 3 }), + version: "1.2.3", + }) as StartDaemonOptions; + const first = await startDaemon(options()); + // why: several leases, so a reconcile that runs beside core's steps lands some releases after the announcement. + const grants: Grant[] = []; + for (const requesterId of ["agent-1", "agent-2", "agent-3"]) { + grants.push((await first.dispatch("lease.request", { ...ios, requesterId }, agent)) as Grant); + } + await first.stop("test"); + // why: the history orders events by their time, and the second start's must follow the first's. + clock.advance(1); + const secondRunStart = clock.now(); + + const daemon = await startDaemon(options()); + runningDaemons.push(daemon); + + const leases = (await daemon.dispatch("list.get", { kind: "leases" }, admin)) as readonly { + readonly id: string; + }[]; + for (const grant of grants) { + expect(leases.map((lease) => lease.id)).not.toContain(grant.lease.id); + } + const secondRun = ( + await historyEventsWhen(directory, (all) => + all.some( + (event) => event.event === "capacity.changed" && event.timestamp >= secondRunStart, + ), + ) + ) + .filter((event) => event.timestamp >= secondRunStart) + .sort((left, right) => left.seq - right.seq) + .map((event) => event.event); + expect(secondRun.filter((name) => name === "lease.released")).toHaveLength(grants.length); + expect(secondRun.lastIndexOf("lease.released")).toBeLessThan( + secondRun.indexOf("capacity.changed"), + ); + // why: two daemon starts and a history wait of up to 4 s outrun the 5 s default on a loaded CI runner (run 39454292501 timed out at 5015 ms, 59 ms locally). + }, 30_000); + + it("closes state.db on stop, and lists the same device and lease after a worker is stopped and started again", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); + temporaryDirectories.push(directory); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const options = () => + ({ + clock, + configOverrides: STANDALONE_OFF, + dataDirectory: directory, + deviceDataTables: fakeDeviceDataTables, + drivers: [driver], + filesystem: new MemoryFilesystem(), + logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), + version: "1.2.3", + }) as StartDaemonOptions; + const first = await startDaemon(options()); + const grant = (await first.dispatch("lease.request", ios, agent)) as Grant; + const list = async (daemon: Awaited>, kind: string) => + ( + (await daemon.dispatch("list.get", { kind }, admin)) as readonly { readonly id: string }[] + ).map((row) => row.id); + const devicesBefore = await list(first, "devices"); + await first.stop("test"); + // why: closing the last connection checkpoints and removes the WAL, so a stop that left state.db open leaves it. + expect(existsSync(join(directory, "state.db-wal"))).toBe(false); + + const daemon = await startDaemon(options()); + runningDaemons.push(daemon); + + expect(devicesBefore).toContain(grant.device.id); + expect(await list(daemon, "devices")).toEqual(devicesBefore); + expect(await list(daemon, "leases")).toEqual([grant.lease.id]); + }); + + it("holds shutdown open while a release's erase runs, and does not read that erase as a stall", async () => { + const clock = new FakeClock(1_000); + const { daemon, driver, grant } = await leasedIos({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { destroy: 600_000 }, + platform: "ios", + }); + await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); + await settleRealTime(); + + clock.advance(300_000); + const devices = (await daemon.dispatch("list.get", { kind: "devices" }, admin)) as readonly { + readonly id: string; + readonly state: string; + readonly stalled?: true; + }[]; + expect(devices.map(({ id, state, stalled }) => ({ id, state, stalled }))).toEqual([ + { id: grant.device.id, state: "shutdown", stalled: undefined }, + ]); + + let stopped = false; + const stopping = daemon.stop("test").then(() => { + stopped = true; + }); + await settleRealTime(); + expect(stopped).toBe(false); + clock.advance(300_000); + await stopping; + expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); + }); + + it("cancels the expiry timer of an outstanding lease on shutdown, so nothing expires after it", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const { daemon, directory } = await start({ clock, drivers: [driver] }); + await daemon.dispatch("lease.request", { ...ios, ttlMs: 10_000 }, agent); + const leaseIds = async () => + (await queryState(directory, "SELECT id FROM leases")).map((row) => row.id); + const before = await leaseIds(); + expect(before).toHaveLength(1); + + await daemon.stop("test"); + clock.advance(10_000); + await settleRealTime(); + + expect(await leaseIds()).toEqual(before); + }); + + it("cancels the quarantine retry timer on shutdown, so no retry runs after it", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + const started = await start({ clock, drivers: [driver] }); + const grant = (await started.daemon.dispatch("lease.request", ios, agent)) as Grant; + await started.daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); + + await started.daemon.stop("test"); + clock.advance(30_000); + await settleRealTime(); + + expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); + }); + + describe("a device quarantined by a failed delete", () => { + type CauseView = { + readonly id: string; + readonly quarantineCause?: { readonly kind: string; readonly message?: string }; + readonly state: string; + }; + + async function quarantinedAfterFailedPurge() { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + const { daemon } = await start({ + clock, + configOverrides: { warmPool: { enabled: false } }, + drivers: [driver], + }); + const grant = (await daemon.dispatch("lease.request", ios, agent)) as Grant; + await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); + await settleRealTime(); + return daemon; + } + + const devicesOf = async ( + daemon: Awaited>, + session: typeof agent | typeof admin, + ) => + ( + (await daemon.dispatch("status.get", {}, session)) as { + readonly devices: readonly CauseView[]; + } + ).devices; + + it("shows quarantineCause delete-failed with the driver's error in status.get and list.get", async () => { + const daemon = await quarantinedAfterFailedPurge(); + + const status = await devicesOf(daemon, admin); + const listed = (await daemon.dispatch( + "list.get", + { kind: "devices" }, + admin, + )) as readonly CauseView[]; + + for (const devices of [status, listed]) { + expect(devices).toHaveLength(1); + expect(devices[0]).toMatchObject({ state: "quarantined" }); + expect(devices[0]?.quarantineCause).toEqual({ + kind: "delete-failed", + message: "DriverCrashError: purge exploded", + }); + } + }); + }); + + it("probes a leased device again as time passes once startup is done", async () => { + const clock = new FakeClock(1_000); + const { driver } = await leasedIos({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const probesBefore = driver.calls.filter((call) => call.operation === "listManaged").length; + + clock.advance(30_000); + await settleRealTime(); + + expect(driver.calls.filter((call) => call.operation === "listManaged").length).toBeGreaterThan( + probesBefore, + ); + }); +}); diff --git a/src/daemon/main-start-daemon.test.ts b/src/daemon/main-start-daemon.test.ts new file mode 100644 index 000000000..bb804bf81 --- /dev/null +++ b/src/daemon/main-start-daemon.test.ts @@ -0,0 +1,663 @@ +import { hasEvent, historyEventsWhen } from "./test-main-history.js"; +import { queryState } from "./test-main-state.js"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createServer } from "node:net"; +import { createServer as createHttpServer, type IncomingHttpHeaders } from "node:http"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { type Config, type DriverToolVersion, REDACTED_VALUE } from "../core/index.js"; +import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; +import { + FakeClock, + FakeHostInfo, + daemonDataDirectory, + FakeServiceManager, + serviceDefinition, + type ServiceDefinition, + type ServiceManager, + type ServiceState, + MemoryFilesystem, +} from "../ports/index.js"; +import { productionServiceCheck } from "./main.js"; +import type { DaemonServer } from "./server.js"; +import { start, stopDaemons, temporaryDirectories } from "./test-main.js"; + +afterEach(stopDaemons); + +/** why: A gateway URL on a port just released. Nothing listens there, so a worker's uplink dial + * fails fast, and the fake clock never fires its retry. */ +async function unreachableGatewayUrl(): Promise { + const probe = createServer(); + await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); + const { port } = probe.address() as { readonly port: number }; + await new Promise((resolve) => probe.close(() => resolve())); + return `ws://127.0.0.1:${port}`; +} + +class DisposableFakeDriver extends FakeDriver { + constructor( + private readonly disposed: string[], + options: ConstructorParameters[0], + private readonly failure?: Error, + ) { + super(options); + } + + async dispose(): Promise { + this.disposed.push(this.platform); + if (this.failure !== undefined) { + throw this.failure; + } + } +} + +async function agentStatus(daemon: DaemonServer) { + return (await daemon.dispatch( + "status.get", + {}, + { manageEventSubscription: () => undefined, via: "socket", principal: "agent", role: "agent" }, + )) as { readonly host: { readonly tools: readonly unknown[] } }; +} + +class ToolsFakeDriver extends FakeDriver { + constructor( + options: ConstructorParameters[0], + private readonly tools: () => Promise, + ) { + super(options); + } + + toolVersions(): Promise { + return this.tools(); + } +} + +describe("startDaemon", () => { + it.each([ + { name: "not set", headers: undefined, label: undefined, expected: {} }, + { + name: "two headers", + headers: { "CF-Access-Client-Id": "abc.access", "CF-Access-Client-Secret": "s3cr3t" }, + label: undefined, + expected: { "cf-access-client-id": "abc.access", "cf-access-client-secret": "s3cr3t" }, + }, + { + name: "an empty value", + headers: { "x-empty": "" }, + label: undefined, + expected: { "x-empty": "" }, + }, + { + name: "a label set too", + headers: { "x-proxy-key": "k" }, + label: "mac-1", + expected: { "x-proxy-key": "k" }, + }, + ])( + "a worker started with gateway.headers sends every entry on its uplink dial, beside the join token and worker id: $name", + async ({ headers, label, expected }) => { + let received: IncomingHttpHeaders | undefined; + const gateway = createHttpServer(); + gateway.on("upgrade", (request, socket) => { + received = request.headers; + socket.destroy(); + }); + await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve)); + try { + const { port } = gateway.address() as { readonly port: number }; + await start({ + configOverrides: { + gateway: { + token: "join-token", + url: `ws://127.0.0.1:${port}`, + ...(headers === undefined ? {} : { headers }), + ...(label === undefined ? {} : { label }), + }, + }, + }); + await vi.waitFor(() => expect(received).toBeDefined()); + const seen = received ?? {}; + expect({ + authorization: seen["authorization"], + workerId: typeof seen["x-simlock-worker-id"], + label: seen["x-simlock-worker-label"], + extra: Object.fromEntries(Object.entries(seen).filter(([name]) => name in expected)), + }).toEqual({ + authorization: "Bearer join-token", + workerId: "string", + label, + extra: expected, + }); + } finally { + await new Promise((resolve) => gateway.close(() => resolve())); + } + }, + ); + + it.each([ + { role: "worker", withToken: true }, + { role: "gateway", withToken: false }, + ] as const)( + "keeps gateway.token and every gateway.headers value out of the daemon log, the events.replay answer, and history.db, while config.get returns them only as the redaction marker unless showSecrets is asked: $role", + async ({ role, withToken }) => { + const secret = "secret-join-token-170"; + const headerSecret = "s3cr3t-proxy-credential"; + const secondHeaderSecret = "second-proxy-credential"; + const { daemon, directory, sink } = await start({ + role, + configOverrides: { + gateway: { + headers: { + "CF-Access-Client-Id": "client-id-170", + "CF-Access-Client-Secret": headerSecret, + "X-Proxy-Key": secondHeaderSecret, + }, + ...(withToken ? { token: secret, url: await unreachableGatewayUrl() } : {}), + }, + }, + }); + const admin = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + } as const; + type Answer = { + readonly gateway: { + readonly token?: string; + readonly headers?: Readonly>; + }; + }; + + const replayed = await daemon.dispatch("events.replay", {}, admin); + const eventFile = JSON.stringify( + await historyEventsWhen(directory, hasEvent("daemon.started")), + ); + const config = (await daemon.dispatch("config.get", {}, admin)) as Answer; + + // why: The fixture is real: the daemon did log its start and emit `daemon.started` to both + // the ring and the file, so "absent" below means redacted, not never written. + expect(sink.records.some((record) => record.message === "Daemon started")).toBe(true); + expect(eventFile).toContain('"daemon.started"'); + expect(JSON.stringify(replayed)).toContain('"daemon.started"'); + const leaks = (value: string) => ({ + daemonLog: JSON.stringify(sink.records).includes(value), + eventsReplay: JSON.stringify(replayed).includes(value), + historyDb: eventFile.includes(value), + }); + const clean = { daemonLog: false, eventsReplay: false, historyDb: false }; + expect({ + token: leaks(secret), + header: leaks(headerSecret), + second: leaks(secondHeaderSecret), + first: leaks("client-id-170"), + }).toEqual({ + token: clean, + header: clean, + second: clean, + first: clean, + }); + const shown = (await daemon.dispatch("config.get", { showSecrets: true }, admin)) as Answer; + expect({ + default: config.gateway.headers, + shown: shown.gateway.headers, + token: [config.gateway.token, shown.gateway.token], + }).toEqual({ + default: { + "CF-Access-Client-Id": "[redacted]", + "CF-Access-Client-Secret": "[redacted]", + "X-Proxy-Key": "[redacted]", + }, + shown: { + "CF-Access-Client-Id": "client-id-170", + "CF-Access-Client-Secret": headerSecret, + "X-Proxy-Key": secondHeaderSecret, + }, + token: withToken ? ["[redacted]", secret] : [undefined, undefined], + }); + }, + ); + + it.each([ + { + name: "nothing set: each driver's own defaults", + drivers: {}, + expected: (home: string) => ({ + android: { adbServerPort: 5038, deviceRoot: join(home, "devices", "android") }, + ios: { deviceRoot: join(home, "devices", "ios"), deviceSet: "simlock" }, + }), + }, + { + name: "set keys win over the defaults, and a driver's unknown key is carried", + drivers: { android: { adbServerPort: 5039 }, ios: { deviceSet: "simlock", retries: 3 } }, + expected: (home: string) => ({ + android: { adbServerPort: 5039, deviceRoot: join(home, "devices", "android") }, + ios: { deviceRoot: join(home, "devices", "ios"), deviceSet: "simlock", retries: 3 }, + }), + }, + { + name: "deviceSet default: no iOS deviceRoot, which that set does not use", + drivers: { ios: { deviceSet: "default" } }, + expected: (home: string) => ({ + android: { adbServerPort: 5038, deviceRoot: join(home, "devices", "android") }, + ios: { deviceSet: "default" }, + }), + }, + ])( + "answers config.get with every drivers.* key the drivers use, so config get reads it back ($name)", + async ({ drivers, expected }) => { + const { daemon, directory } = await start({ configOverrides: { drivers } }); + + const config = (await daemon.dispatch( + "config.get", + {}, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + }, + )) as { readonly drivers?: unknown }; + + expect(config.drivers).toEqual(expected(directory)); + }, + ); + + it("keeps gateway.token in the daemon.started payload and the Daemon started log line as the redaction marker, so a set token still reads as set", async () => { + const { daemon, sink } = await start({ + configOverrides: { + gateway: { token: "secret-join-token-170", url: await unreachableGatewayUrl() }, + }, + }); + const replayed = (await daemon.dispatch( + "events.replay", + {}, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + }, + )) as readonly { readonly event: string; readonly payload: unknown }[]; + + const started = replayed.find((envelope) => envelope.event === "daemon.started"); + const logged = sink.records.find((record) => record.message === "Daemon started"); + const snapshot = (started?.payload as { readonly configSnapshot?: Config } | undefined) + ?.configSnapshot; + const loggedConfig = logged?.fields?.config as Config | undefined; + expect({ + eventToken: snapshot?.gateway.token, + logToken: loggedConfig?.gateway.token, + }).toEqual({ eventToken: REDACTED_VALUE, logToken: REDACTED_VALUE }); + }); + + it("runs no prerequisite check at startup convergence, and runs every check on each doctor.run", async () => { + const runs = { android: 0, ios: 0 }; + const missing = { message: "gone", prerequisite: "android-emulator", remedy: "install it" }; + const { daemon } = await start({ + prerequisiteChecks: [ + { + check: () => { + runs.ios += 1; + return Promise.resolve([]); + }, + platform: "ios", + }, + { + check: () => { + runs.android += 1; + return Promise.resolve([missing]); + }, + platform: "android", + }, + ], + }); + const admin = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + } as const; + + // why: doctor.run parks until startup convergence has finished, so by the time it answers, + // convergence's own reconcile has run -- and must not have counted. + const first = (await daemon.dispatch("doctor.run", {}, admin)) as { + readonly findings: readonly { readonly kind: string; readonly prerequisite?: string }[]; + }; + expect(runs).toEqual({ android: 1, ios: 1 }); + expect(first.findings).toContainEqual( + expect.objectContaining({ kind: "prerequisite-missing", platform: "android" }), + ); + + await daemon.dispatch("doctor.run", {}, admin); + expect(runs).toEqual({ android: 2, ios: 2 }); + }); + + it("a worker's doctor reports service-broken for each service whose paths are gone", async () => { + const gone = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; + const mainGone = "/Users/me/simlock/dist/daemon/main.js"; + const name = "com.callstack.simlock.worker"; + const remedy = "Run `simlock daemon install` again."; + const admin = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "operator", + role: "admin", + } as const; + for (const testCase of [ + { label: "not installed", missing: [], expected: [] }, + { label: "installed and whole", missing: [], expected: [] }, + { + label: "one path gone", + missing: [gone], + expected: [{ kind: "service-broken", manager: "launchd", name, missing: [gone], remedy }], + }, + { + label: "both gone", + missing: [gone, mainGone], + expected: [ + { kind: "service-broken", manager: "launchd", name, missing: [gone, mainGone], remedy }, + ], + }, + ]) { + const { daemon } = await start({ + serviceCheck: { manager: "launchd", name, check: async () => testCase.missing }, + }); + const report = (await daemon.dispatch("doctor.run", {}, admin)) as { + readonly findings: readonly { readonly kind: string }[]; + }; + expect( + report.findings.filter((finding) => finding.kind === "service-broken"), + testCase.label, + ).toEqual(testCase.expected); + } + }); + + it("productionServiceCheck builds the service definition from the process facts and reads its state: the missing paths when broken, none otherwise, and no check on a gateway or without a manager", async () => { + const seen: ServiceDefinition[] = []; + const manager: ServiceManager = { + manager: "launchd", + state: async (definition) => { + seen.push(definition); + return { state: "broken", path: definition.path, missing: [definition.command] }; + }, + install: async () => undefined, + uninstall: async () => undefined, + start: async () => undefined, + }; + const host = { + manager, + root: "/Users/me/.simlock", + home: "/Users/me", + nodePath: "/gone/node", + mainPath: "/gone/main.js", + env: { PATH: "/usr/bin" }, + }; + + const { serviceCheck } = productionServiceCheck("worker", host); + + expect(serviceCheck?.name).toBe("com.callstack.simlock.worker"); + expect(await serviceCheck?.check()).toEqual(["/gone/node"]); + expect(seen).toEqual([ + serviceDefinition({ + role: "worker", + root: "/Users/me/.simlock", + dataDirectory: daemonDataDirectory("/Users/me/.simlock", "worker"), + home: "/Users/me", + nodePath: "/gone/node", + mainPath: "/gone/main.js", + env: { PATH: "/usr/bin" }, + }), + ]); + expect(productionServiceCheck("gateway", host)).toEqual({}); + expect(productionServiceCheck("worker", { ...host, manager: undefined })).toEqual({}); + + const states: readonly { readonly state: ServiceState; readonly missing: readonly string[] }[] = + [ + { state: { state: "not-installed" }, missing: [] }, + { state: { state: "installed", path: "/p" }, missing: [] }, + { state: { state: "not-loaded", path: "/p" }, missing: [] }, + { + state: { state: "broken", path: "/p", missing: ["/gone/node", "/gone/main.js"] }, + missing: ["/gone/node", "/gone/main.js"], + }, + ]; + for (const testCase of states) { + const checked = productionServiceCheck("worker", { + ...host, + manager: new FakeServiceManager({ state: testCase.state }), + }); + expect(checked.serviceCheck?.manager).toBe("launchd"); + expect(checked.serviceCheck?.name).toBe("com.callstack.simlock.worker"); + expect(await checked.serviceCheck?.check(), testCase.state.state).toEqual(testCase.missing); + } + expect( + productionServiceCheck("worker", { + ...host, + manager: new FakeServiceManager({ manager: "systemd" }), + }).serviceCheck?.manager, + ).toBe("systemd"); + + const systemd = productionServiceCheck("worker", { + ...host, + manager: { ...manager, manager: "systemd" }, + home: "/home/me", + root: "/home/me/.simlock", + }); + await systemd.serviceCheck?.check(); + expect(seen.at(-1)?.path).toBe( + "/home/me/.config/systemd/user/com.callstack.simlock.worker.service", + ); + }); + + it("stores lease requests under the configured limits, with failures classified by contract code", async () => { + const { daemon, directory } = await start({ + configOverrides: { lease: { maxRequestRecords: 1 } }, + }); + const missingRuntime = { + model: "iPhone 16", + osVersion: "99.0", + platform: "ios", + } as const; + const session = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + } as const; + + for (const requesterId of ["first", "second"]) { + await daemon + .dispatch("lease.request", { ...missingRuntime, requesterId }, session) + .catch(() => undefined); + } + + await expect + .poll(() => queryState(directory, "SELECT requester_id, failure_code FROM lease_requests")) + .toEqual([{ failure_code: "RUNTIME_MISSING", requester_id: "second" }]); + }); + + it("establishes the instance identity once, and reuses it on the next start", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-identity-")); + temporaryDirectories.push(directory); + const filesystem = new MemoryFilesystem(); + const identityPath = join(directory, "instance.json"); + + const first = await start({ dataDirectory: directory, filesystem }); + const written = JSON.parse(await filesystem.readFile(identityPath)) as { + readonly instanceId: string; + }; + await first.daemon.stop("test"); + await start({ dataDirectory: directory, filesystem }); + + // why: Never regenerated: a second id would strand every device already sitting in a root + // marked with the first one (ADR 0001, decision 2). + expect(written.instanceId).not.toBe(""); + expect(JSON.parse(await filesystem.readFile(identityPath))).toEqual(written); + }); + + it("disposes every driver on shutdown, and reports the one that failed without stopping", async () => { + // why: The wiring, not the halves: `DaemonServer` awaiting its `dispose` option is tested in + // `server.test.ts` and a driver reaping its adb server in the Android driver's own, and + // between them sat the composition that connects the two. Losing it silently abandons + // Simlock's adb server on every shutdown, and only `Driver.dispose` can stop one. + const disposed: string[] = []; + const clock = new FakeClock(1_000); + const { daemon, sink } = await start({ + drivers: [ + new DisposableFakeDriver(disposed, { + availableOsVersions: ["26.5"], + clock, + platform: "ios", + }), + new DisposableFakeDriver( + disposed, + { availableOsVersions: ["35"], clock, platform: "android" }, + new Error("adb server would not die"), + ), + ], + deviceDataTables: fakeDeviceDataTables, + }); + + await daemon.stop("test"); + + expect(disposed).toEqual(["ios", "android"]); + expect(sink.records).toContainEqual( + expect.objectContaining({ + level: "error", + message: "Driver disposal failed", + fields: expect.objectContaining({ platform: "android" }), + }), + ); + }); + + it("ends a running component install on shutdown, before the drivers are disposed", async () => { + const clock = new FakeClock(1_000); + const events: string[] = []; + class InstallingDriver extends FakeDriver { + override async installComponent( + component: string, + options: Parameters[1], + ) { + events.push("install-started"); + options.signal.addEventListener("abort", () => events.push("install-ended")); + return super.installComponent(component, options); + } + + async dispose(): Promise { + events.push("disposed"); + } + } + const driver = new InstallingDriver({ availableOsVersions: [], clock, platform: "ios" }); + driver.holdInstalls(); + const { daemon } = await start({ drivers: [driver] }); + + const lease = daemon + .dispatch( + "lease.request", + { + allowDownload: true, + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + }, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + }, + ) + .catch((error: unknown) => error); + await expect.poll(() => events).toEqual(["install-started"]); + await daemon.stop("test"); + + expect(events).toEqual(["install-started", "install-ended", "disposed"]); + await expect(lease).resolves.toBeInstanceOf(Error); + }); + + it("starts and reports no entry for a driver whose tool read fails, and every other tool", async () => { + const clock = new FakeClock(1_000); + const { daemon } = await start({ + drivers: [ + new ToolsFakeDriver( + { availableOsVersions: ["26.5"], clock, platform: "ios" }, + // What an iOS driver whose `xcodebuild` fails looks like from here. + () => Promise.reject(new Error("xcodebuild -version exited with 1")), + ), + new ToolsFakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }, () => + Promise.resolve([{ name: "emulator", version: "35.4.9" }]), + ), + ], + deviceDataTables: fakeDeviceDataTables, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + }); + + await expect + .poll(async () => (await agentStatus(daemon)).host) + .toEqual({ + arch: "arm64", + os: "macOS", + osVersion: "15.5", + tools: [{ name: "emulator", platform: "android", version: "35.4.9" }], + }); + }); + + it("answers status.get when a driver reports a tool version past the contract's bounds", async () => { + const clock = new FakeClock(1_000); + const { daemon } = await start({ + drivers: [ + new ToolsFakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }, () => + Promise.resolve([ + { name: "emulator", version: "9".repeat(500) }, + { name: "platform-tools", version: "36.0.0" }, + ]), + ), + ], + deviceDataTables: fakeDeviceDataTables, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + }); + + await expect + .poll(async () => (await agentStatus(daemon)).host.tools) + .toEqual([{ name: "platform-tools", platform: "android", version: "36.0.0" }]); + }); + + it("reads tool versions at start, before anything asks for status", async () => { + const clock = new FakeClock(1_000); + let reads = 0; + await start({ + drivers: [ + new ToolsFakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }, () => { + reads += 1; + return Promise.resolve([]); + }), + ], + deviceDataTables: fakeDeviceDataTables, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + }); + + expect(reads).toBe(1); + }); + + it("still answers component.list when a driver's component listing fails", async () => { + const clock = new FakeClock(1_000); + const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + ios.failOn("listComponents", 1, new Error("simctl runtime list failed")); + const { daemon } = await start({ drivers: [ios] }); + + const listed = await daemon.dispatch( + "component.list", + {}, + { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + }, + ); + + expect(listed).toEqual({ components: [] }); + }); +}); diff --git a/src/daemon/main-state-db.test.ts b/src/daemon/main-state-db.test.ts new file mode 100644 index 000000000..aed6cb8c4 --- /dev/null +++ b/src/daemon/main-state-db.test.ts @@ -0,0 +1,353 @@ +import { queryState } from "./test-main-state.js"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { registryOwner } from "../core/index.js"; +import { + FakeDriver, + fakeDeviceDataTable, + fakeDeviceDataTables, + loadTestRegistry, +} from "../core/testing.js"; +import { openStateDatabase } from "../state/index.js"; +import { androidDeviceDataTable } from "../drivers/android/index.js"; +import { iosDeviceDataTable } from "../drivers/ios/index.js"; +import { EventBus } from "../bus/index.js"; +import { + FakeClock, + IpcError, + MemoryFilesystem, + MemoryIpcTransport, + NodeSqliteOpener, + NoopLogger, +} from "../ports/index.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; +import { productionDeviceDataTables, resolveDeviceDataTables } from "./driver-discovery.js"; +import { + STANDALONE_OFF, + runningDaemons, + start, + stopDaemons, + temporaryDirectories, + writeModule, +} from "./test-main.js"; +import { startOnFreePort } from "../test-support/loopback-port.js"; + +afterEach(stopDaemons); + +describe("startDaemon state.db", () => { + const previousModule = process.env.SIMLOCK_DRIVERS_MODULE; + + afterEach(() => { + if (previousModule === undefined) { + delete process.env.SIMLOCK_DRIVERS_MODULE; + } else { + process.env.SIMLOCK_DRIVERS_MODULE = previousModule; + } + }); + + it.each([ + [ + "worker", + "state.json", + "with the older Simlock, release every lease and run `simlock nuke --delete-devices`; then remove and start again.", + ], + [ + "worker", + "tokens.json", + "with the older Simlock, release every lease and run `simlock nuke --delete-devices`; then remove and start again.", + ], + [ + "gateway", + "tokens.json", + "remove and start again; then create each worker's join token again with `simlock --gateway token create --role worker`, set it as that worker's `gateway.token`, and drain workers again.", + ], + [ + "gateway", + "workers.json", + "remove and start again; then create each worker's join token again with `simlock --gateway token create --role worker`, set it as that worker's `gateway.token`, and drain workers again.", + ], + ] as const)( + "a %s started with %s in its folder refuses with the old-file message and its step", + async (role, file, step) => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-old-file-")); + temporaryDirectories.push(directory); + const oldFile = join(directory, file); + await writeFile(oldFile, "[]", "utf8"); + + await expect(start({ dataDirectory: directory, role })).rejects.toMatchObject({ + message: `Found ${oldFile}, which this Simlock no longer reads. To upgrade: ${step.replace("", oldFile)} See https://callstackincubator.github.io/simlock/docs/worker/running-the-daemon#upgrade-simlock.`, + }); + expect(existsSync(join(directory, "state.db"))).toBe(false); + }, + ); + + it("a token created, then the daemon restarted on the same state.db, verifies and lists with every field", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-token-restart-")); + temporaryDirectories.push(directory); + const admin = { + manageEventSubscription: () => undefined, + principal: "admin", + role: "admin", + via: "socket", + } as const; + let port = 0; + const launch = async (now: number) => { + const launched = await startOnFreePort((candidate) => + startDaemon({ + clock: new FakeClock(now), + configOverrides: { + ...STANDALONE_OFF, + http: { enabled: true, host: "127.0.0.1", port: candidate }, + }, + dataDirectory: directory, + deviceDataTables: fakeDeviceDataTables, + drivers: [], + hostLoadProbe: null, + version: "1.2.3", + } as StartDaemonOptions), + ); + port = launched.port; + return launched.started; + }; + type Created = { readonly secret: string; readonly token: Record }; + const first = await launch(1_000); + const labelled = (await first.dispatch( + "token.create", + { label: "ci", role: "agent" }, + admin, + )) as Created; + const bare = (await first.dispatch("token.create", { role: "operator" }, admin)) as Created; + await first.stop(); + + const second = await launch(2_000); + runningDaemons.push(second); + + const listed = (await second.dispatch("token.list", {}, admin)) as { + readonly tokens: readonly Record[]; + }; + expect(listed.tokens).toEqual([labelled.token, bare.token]); + expect("label" in (listed.tokens[1] ?? {})).toBe(false); + + const statusWith = async (secret: string) => + ( + await fetch(`http://127.0.0.1:${port}/v1/status`, { + headers: { authorization: `Bearer ${secret}` }, + }) + ).status; + expect(await statusWith(labelled.secret)).toBe(200); + expect(await statusWith(bare.secret)).toBe(200); + expect(await statusWith("slk_not-a-created-secret")).toBe(401); + }); + + it("a start that fails after state.db opened closes it before it throws", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-closes-state-")); + temporaryDirectories.push(directory); + const state = await openStateDatabase({ + dataDirectory: directory, + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + oldFiles: [], + owners: [registryOwner, fakeDeviceDataTable], + sqlite: new NodeSqliteOpener(), + upgradeStep: "unused", + }); + const previous = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "1" }, + state, + }); + await previous.registerDevice({ + driverData: { fakeDeviceId: "fk-1" }, + driverDeviceId: "drv-1", + provisionDuration: 0, + spec: { model: "iPhone 16", osVersion: "26.5", physical: false, platform: "ios" }, + }); + state.database.exec("DELETE FROM fake_device_data"); + state.close(); + expect(existsSync(join(directory, "state.db-wal"))).toBe(false); + + await expect(start({ dataDirectory: directory })).rejects.toMatchObject({ + name: "RegistryLoadError", + }); + + expect(existsSync(join(directory, "state.db-wal"))).toBe(false); + }); + + it("a worker drops a deleted device record older than the configured devices.deletedRetentionMs at start", async () => { + const clock = new FakeClock(1_000); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-deleted-retention-")); + temporaryDirectories.push(directory); + const state = await openStateDatabase({ + dataDirectory: directory, + filesystem: new MemoryFilesystem(), + logger: new NoopLogger(), + oldFiles: [], + owners: [registryOwner, fakeDeviceDataTable], + sqlite: new NodeSqliteOpener(), + upgradeStep: "unused", + }); + const previous = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "1" }, + state, + }); + const device = await previous.registerDevice({ + driverData: { fakeDeviceId: "fk-1" }, + driverDeviceId: "drv-1", + provisionDuration: 0, + spec: { model: "iPhone 16", osVersion: "26.5", physical: false, platform: "ios" }, + }); + await previous.markDeviceMissing(device.id, "doctor"); + state.close(); + // why: 30 minutes is inside the default hour and outside the ten minutes configured below. + clock.advance(30 * 60_000); + + await start({ + clock, + configOverrides: { devices: { deletedRetentionMs: 600_000 } }, + dataDirectory: directory, + }); + + expect(await queryState(directory, "SELECT id FROM devices")).toEqual([]); + }); + + it("a worker refuses a re-sent chosen lease id inside the configured lease.endedIdRetentionMs and grants it after", async () => { + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const { daemon } = await start({ + clock, + configOverrides: { lease: { endedIdRetentionMs: 600_000 } }, + drivers: [driver], + }); + const agent = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent", + role: "agent", + } as const; + const chosen = { + leaseId: "retained", + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + requesterId: "agent-1", + } as const; + await daemon.dispatch("lease.request", chosen, agent); + await daemon.dispatch("lease.release", { leaseId: "retained" }, agent); + + // why: 5 minutes is inside the ten minutes configured and far inside the default day. + clock.advance(5 * 60_000); + await expect(daemon.dispatch("lease.request", chosen, agent)).rejects.toMatchObject({ + code: "LEASE_ENDED", + }); + // why: 30 minutes is outside the ten minutes configured and inside the default day. + clock.advance(30 * 60_000); + await expect(daemon.dispatch("lease.request", chosen, agent)).resolves.toMatchObject({ + lease: { id: "retained" }, + }); + }); + + it("a worker's state.db holds both platforms' source-data tables, a gateway's holds neither", async () => { + const worker = await mkdtemp(join(tmpdir(), "simlock-main-source-owners-")); + const gateway = await mkdtemp(join(tmpdir(), "simlock-main-source-owners-")); + temporaryDirectories.push(worker, gateway); + await start({ dataDirectory: worker }); + await start({ dataDirectory: gateway, role: "gateway" }); + const owners = async (directory: string) => + (await queryState(directory, "SELECT owner FROM schema_versions")) + .map((row) => row["owner"]) + .filter((owner) => owner === "ios-source-data" || owner === "android-source-data") + .sort(); + + expect(await owners(worker)).toEqual(["android-source-data", "ios-source-data"]); + expect(await owners(gateway)).toEqual([]); + }); + + it("a second startDaemon on a folder whose daemon runs leaves state.db's schema_versions and bytes unchanged", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-live-state-")); + temporaryDirectories.push(directory); + await start({ dataDirectory: directory }); + const versions = () => queryState(directory, "SELECT owner, version FROM schema_versions"); + const before = await versions(); + const bytesBefore = await readFile(join(directory, "state.db")); + // why: a newer binary brings a newer version of an owner the live daemon's state.db has; opening would migrate it in. + const newerTable = { + ...fakeDeviceDataTable, + migrations: [ + ...fakeDeviceDataTable.migrations, + "CREATE TABLE newer_fake_data (id TEXT) STRICT", + ], + version: 2, + }; + + await expect( + start({ + dataDirectory: directory, + deviceDataTables: new Map([["ios", newerTable]]), + }), + ).rejects.toMatchObject({ name: "DaemonAlreadyRunningError" }); + + expect(await versions()).toEqual(before); + expect(await readFile(join(directory, "state.db"))).toEqual(bytesBefore); + }); + + it("refuses to start as already running when connecting to its endpoint times out, as a Windows pipe whose every instance is busy does", async () => { + const memory = new MemoryIpcTransport(); + const ipc = { + connect: async () => { + throw new IpcError("connection-timeout", "timed out", undefined); + }, + listen: memory.listen.bind(memory), + }; + + await expect(start({ ipc })).rejects.toMatchObject({ name: "DaemonAlreadyRunningError" }); + }); + + it("a SIMLOCK_DRIVERS_MODULE without a deviceDataTables export fails start naming it", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-no-tables-")); + temporaryDirectories.push(directory); + const modulePath = join(directory, "drivers.mjs"); + await writeFile(modulePath, "export function createDrivers() { return []; }\n", "utf8"); + process.env.SIMLOCK_DRIVERS_MODULE = modulePath; + + await expect( + start({ dataDirectory: directory, deviceDataTables: undefined, drivers: undefined }), + ).rejects.toMatchObject({ + message: `SIMLOCK_DRIVERS_MODULE ${modulePath} does not export deviceDataTables, a Map from platform to its device-data table`, + }); + expect(existsSync(join(directory, "state.db"))).toBe(false); + }); + + it("resolveDeviceDataTables answers the option's map when given, else the drivers module's export, else the production map", async () => { + expect(await resolveDeviceDataTables({ deviceDataTables: fakeDeviceDataTables })).toBe( + fakeDeviceDataTables, + ); + + // why: the module exports tables of its own, so a module checked first would answer a different map. + process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( + "export function createDrivers() { return []; }", + ); + const optionMap = new Map(fakeDeviceDataTables); + expect(await resolveDeviceDataTables({ deviceDataTables: optionMap })).toBe(optionMap); + + delete process.env.SIMLOCK_DRIVERS_MODULE; + const production = await resolveDeviceDataTables({}); + expect(production).toEqual(productionDeviceDataTables()); + expect(production.get("ios")).toBe(iosDeviceDataTable); + expect(production.get("android")).toBe(androidDeviceDataTable); + + process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( + "export function createDrivers() { return []; }", + ); + const fromModule = await resolveDeviceDataTables({}); + expect([...fromModule.keys()].sort()).toEqual(["android", "ios"]); + expect(fromModule.get("ios")?.name).toBe("fake-device-data"); + expect(fromModule).not.toBe(fakeDeviceDataTables); + }); +}); diff --git a/src/daemon/main-worker-lock.test.ts b/src/daemon/main-worker-lock.test.ts new file mode 100644 index 000000000..77972972c --- /dev/null +++ b/src/daemon/main-worker-lock.test.ts @@ -0,0 +1,245 @@ +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + FakeHostInfo, + FakeNpmCli, + FakeProcessSupervisor, + FakeTcpProbe, + MemoryFilesystem, + ScriptedProcessRunner, +} from "../ports/index.js"; +import { type StartDaemonOptions } from "./main.js"; +import { + start, + startGateway, + stopDaemons, + temporaryDirectories, + writeModule, +} from "./test-main.js"; + +afterEach(stopDaemons); + +describe("startDaemon worker lock", () => { + const LOCK_PATH = "/tmp/simlock-test/worker.lock"; + const OTHER_FOLDER = "/home/other/.simlock"; + + async function filesystemWithLockHeldBy( + owner: { readonly pid: number; readonly folder: string }, + path = LOCK_PATH, + ): Promise { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp(path); + await filesystem.writeFileAtomic( + join(path, "owner.json"), + JSON.stringify({ ...owner, startedAt: "2026-11-02T08:00:00.000Z" }), + ); + return filesystem; + } + + async function installAndroidSdk(filesystem: MemoryFilesystem): Promise { + vi.stubEnv("ANDROID_HOME", "/android-sdk"); + for (const binary of [ + "platform-tools/adb", + "emulator/emulator", + "cmdline-tools/latest/bin/avdmanager", + "cmdline-tools/latest/bin/sdkmanager", + ]) { + await filesystem.mkdirp(dirname(join("/android-sdk", binary))); + await filesystem.writeFileAtomic(join("/android-sdk", binary), "binary"); + } + } + + // why: real drivers are the ones discovered, so neither the drivers option nor the substitute module may be set. + async function startWithRealDrivers( + filesystem: MemoryFilesystem, + livePids: readonly number[] | FakeProcessSupervisor, + processRunner: ScriptedProcessRunner, + dataDirectory?: string, + extra: Partial = {}, + ) { + vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); + return start({ + ...extra, + drivers: undefined, + filesystem, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + npm: new FakeNpmCli(), + processRunner, + processSupervisor: Array.isArray(livePids) + ? new FakeProcessSupervisor(livePids) + : (livePids as FakeProcessSupervisor), + workerLockPath: LOCK_PATH, + ...(dataDirectory === undefined ? {} : { dataDirectory }), + }); + } + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("a worker whose lock is held by a live pid in another folder fails to start with the refusal message, before any driver is discovered", async () => { + const filesystem = await filesystemWithLockHeldBy({ folder: OTHER_FOLDER, pid: 4242 }); + // why: without an SDK discovery does nothing a test can see, so a lock taken after it would pass; with one, discovery creates the Android device root. + await installAndroidSdk(filesystem); + const processRunner = new ScriptedProcessRunner([]); + const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); + temporaryDirectories.push(dataDirectory); + + await expect( + startWithRealDrivers(filesystem, [4242], processRunner, dataDirectory), + ).rejects.toThrow( + `another Simlock worker runs for this user (pid 4242, folder ${OTHER_FOLDER}, since 2026-11-02T08:00:00.000Z); stop it first: SIMLOCK_HOME=${OTHER_FOLDER} simlock daemon stop`, + ); + expect(processRunner.calls).toEqual([]); + expect(await filesystem.exists(join(dataDirectory, "devices", "android"))).toBe(false); + }); + + it.each([ + { name: "real drivers", substitute: "none" }, + { name: "drivers passed to startDaemon", substitute: "passed" }, + { name: "SIMLOCK_DRIVERS_MODULE set", substitute: "module" }, + ] as const)( + "takes the worker lock when drivers are discovered, writing its pid, folder and start time, and reads a held one never otherwise: $name", + async ({ substitute }) => { + const held = { folder: OTHER_FOLDER, pid: 4242 }; + const filesystem = await filesystemWithLockHeldBy( + substitute === "none" ? { folder: "/gone", pid: 4343 } : held, + ); + vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); + if (substitute === "module") { + vi.stubEnv( + "SIMLOCK_DRIVERS_MODULE", + await writeModule(`export function createDrivers() { return []; }`), + ); + } + + const { directory } = await start({ + ...(substitute === "passed" ? {} : { drivers: undefined }), + filesystem, + hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), + npm: new FakeNpmCli(), + processRunner: new ScriptedProcessRunner([]), + processSupervisor: new FakeProcessSupervisor([4242]), + workerLockPath: LOCK_PATH, + }); + + const owner = JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json"))); + expect(owner).toEqual( + substitute === "none" + ? { folder: directory, pid: process.pid, startedAt: "1970-01-01T00:00:01.000Z" } + : { ...held, startedAt: "2026-11-02T08:00:00.000Z" }, + ); + }, + ); + + it("a worker that wins the socket claim while the lock names its folder and another live pid rewrites owner.json with its own pid", async () => { + const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); + temporaryDirectories.push(dataDirectory); + const filesystem = await filesystemWithLockHeldBy({ folder: dataDirectory, pid: 4242 }); + + await startWithRealDrivers(filesystem, [4242], new ScriptedProcessRunner([]), dataDirectory); + + expect(JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json")))).toEqual({ + folder: dataDirectory, + pid: process.pid, + startedAt: "1970-01-01T00:00:01.000Z", + }); + }); + + it("a worker that wins the socket claim while a live pid in another folder holds the lock fails its start with the refusal message", async () => { + const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); + temporaryDirectories.push(dataDirectory); + const filesystem = await filesystemWithLockHeldBy({ folder: dataDirectory, pid: 4242 }); + const ownerFile = join(LOCK_PATH, "owner.json"); + const reads: string[] = []; + // why: the other worker takes the lock over between this worker's take and its check after the claim, which a start cannot be made to wait for. + const readFile = filesystem.readFile.bind(filesystem); + filesystem.readFile = async (path: string) => { + if (path === ownerFile) reads.push(path); + return path === ownerFile && reads.length > 1 + ? JSON.stringify({ folder: OTHER_FOLDER, pid: 4242, startedAt: "t0" }) + : readFile(path); + }; + + await expect( + startWithRealDrivers(filesystem, [4242], new ScriptedProcessRunner([]), dataDirectory), + ).rejects.toThrow( + `another Simlock worker runs for this user (pid 4242, folder ${OTHER_FOLDER}, since t0)`, + ); + }); + + it.each([ + { name: "a stop removes the lock it wrote", takenOver: false, failedStart: false }, + { name: "a stop leaves a lock another worker took over", takenOver: true, failedStart: false }, + { + name: "a start that fails after taking the lock removes it", + takenOver: false, + failedStart: true, + }, + ])("$name", async ({ failedStart, takenOver }) => { + const filesystem = new MemoryFilesystem(); + const ownerFile = join(LOCK_PATH, "owner.json"); + const taken = { folder: OTHER_FOLDER, pid: 4242, startedAt: "t0" }; + // why: the Android driver's disposal ends the adb server this record names, which is the one thing a test can see a driver do at disposal; the lock must still stand then (ADR 0059 §3). + await installAndroidSdk(filesystem); + const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); + temporaryDirectories.push(dataDirectory); + await filesystem.mkdirp(dataDirectory); + await filesystem.writeFileAtomic( + join(dataDirectory, "adb-server.json"), + JSON.stringify({ pid: 4343, port: 5038, startedAt: 1 }), + ); + const supervisor = new FakeProcessSupervisor([4242, 4343]); + supervisor.answerCommandLine(4343, "/android-sdk/platform-tools/adb -P 5038 nodaemon server"); + const lockAtDisposal: Promise[] = []; + const signal = supervisor.signal.bind(supervisor); + supervisor.signal = (pid, kill) => { + lockAtDisposal.push(filesystem.exists(LOCK_PATH)); + signal(pid, kill); + supervisor.markDead(pid); + }; + const starting = startWithRealDrivers( + filesystem, + supervisor, + new ScriptedProcessRunner([]), + dataDirectory, + // why: a module named like a core route is refused by the module host, which runs after the lock is taken. + { + tcpProbe: new FakeTcpProbe([5038]), + ...(failedStart ? { optionalModules: [{ name: "status" } as never] } : {}), + }, + ); + + if (failedStart) { + await expect(starting).rejects.toThrow("status"); + } else { + const { daemon } = await starting; + if (takenOver) await filesystem.writeFileAtomic(ownerFile, JSON.stringify(taken)); + await daemon.stop("test"); + } + + if (!failedStart) expect(await Promise.all(lockAtDisposal)).toEqual([true]); + expect(await filesystem.exists(LOCK_PATH)).toBe(takenOver); + if (takenOver) expect(JSON.parse(await filesystem.readFile(ownerFile))).toEqual(taken); + }); + + it("a gateway never takes the worker lock, even when it is held", async () => { + const filesystem = await filesystemWithLockHeldBy({ folder: OTHER_FOLDER, pid: 4242 }); + vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); + + const { daemon } = await startGateway({ + // why: a gateway given no drivers is the case where a worker would discover them and take the lock. + drivers: undefined as never, + filesystem, + processSupervisor: new FakeProcessSupervisor([4242]), + workerLockPath: LOCK_PATH, + }); + + expect(daemon.health).toBe("running"); + expect(JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json")))).toMatchObject({ + pid: 4242, + }); + }); +}); diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 32226abfd..9a8f5b242 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -1,57 +1,22 @@ import { existsSync } from "node:fs"; -import { chmod, mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, mkdtemp, readFile, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { connect, createServer } from "node:net"; -import { createServer as createHttpServer, type IncomingHttpHeaders } from "node:http"; +import { join } from "node:path"; +import { connect } from "node:net"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - type Config, - DriverCrashError, - type DriverToolVersion, - REDACTED_VALUE, - registryOwner, - baselinesOwner, -} from "../core/index.js"; -import { - FakeDriver, - fakeDeviceDataTable, - fakeDeviceDataTables, - loadTestRegistry, -} from "../core/testing.js"; +import { DriverCrashError } from "../core/index.js"; +import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; import { seedTokensDatabase } from "../auth/testing.js"; -import { openStateDatabase } from "../state/index.js"; -import { androidDeviceDataTable } from "../drivers/android/index.js"; -import { iosDeviceDataTable } from "../drivers/ios/index.js"; -import { EventBus, type EventEnvelope } from "../bus/index.js"; -import { readHistoryEvents, seedHistory } from "../history/testing.js"; import { DAEMON_PROTOCOL_VERSION } from "../daemon-protocol/index.js"; import { FakeClock, - FakeHostInfo, - FakeNpmCli, FakeProcessSupervisor, - daemonDataDirectory, - FakeServiceManager, - serviceDefinition, - type ServiceDefinition, - type ServiceManager, - type ServiceState, - FakeSystemStats, - FakeTcpProbe, - IpcError, JsonLinesLogger, MemoryFilesystem, - MemoryIpcTransport, MemoryLogSink, NodeFilesystem, - NodeSqliteOpener, - NoopLogger, - ScriptedProcessRunner, } from "../ports/index.js"; -import { productionServiceCheck, startDaemon, type StartDaemonOptions } from "./main.js"; -import type { DaemonServer } from "./server.js"; -import { productionDeviceDataTables, resolveDeviceDataTables } from "./driver-discovery.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; import { parseDaemonArgv } from "./entry-support.js"; import { SIMLOCK_HOME, @@ -61,10 +26,8 @@ import { pollUntilListening, runningDaemons, start, - startGateway, stopDaemons, temporaryDirectories, - writeModule, } from "./test-main.js"; import { startOnFreePort } from "../test-support/loopback-port.js"; @@ -219,26 +182,6 @@ describe("interrupted root assembly", () => { }); }); -async function historyEventsWhen( - directory: string, - done: (events: readonly EventEnvelope[]) => boolean, -): Promise { - let events: EventEnvelope[] = []; - await vi.waitFor( - async () => { - events = await readHistoryEvents(directory); - expect(done(events)).toBe(true); - }, - { timeout: 4_000 }, - ); - return events; -} - -const hasEvent = - (name: string) => - (events: readonly EventEnvelope[]): boolean => - events.some((event) => event.event === name); - describe("the data directory", () => { it("creates a missing data directory owner-only, and for one other users can reach logs one warning at start and leaves its mode", async () => { // why: under the umask every real host starts from, a directory made without a mode comes @@ -343,1451 +286,178 @@ describe("the version a daemon reports", () => { }); }); -describe("the event history", () => { - it("prunes at start by the configured eventLog.retention, so an event older than it on the injected clock is gone", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); +describe("startDaemon startup readiness", () => { + it("claims the socket and answers hello/status.get while doctor.reconcile is still in flight", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-slow-")); temporaryDirectories.push(directory); - // why: the fake clock reads 1_000; with a retention of 100 the cutoff is 900, past 500 and before 950. - await seedHistory(directory, [500, 950]); - - await start({ configOverrides: { eventLog: { retention: 100 } }, dataDirectory: directory }); - - const events = await historyEventsWhen(directory, hasEvent("daemon.started")); - expect(events.map((event) => event.id)).not.toContain("evt_1"); - expect(events.map((event) => event.id)).toContain("evt_2"); - }); -}); - -async function queryState( - directory: string, - sql: string, -): Promise[]> { - const database = await new NodeSqliteOpener().open(join(directory, "state.db"), { - readOnly: true, - }); - try { - return database.prepare(sql).all(); - } finally { - database.close(); - } -} - -/** why: A gateway URL on a port just released. Nothing listens there, so a worker's uplink dial - * fails fast, and the fake clock never fires its retry. */ -async function unreachableGatewayUrl(): Promise { - const probe = createServer(); - await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); - const { port } = probe.address() as { readonly port: number }; - await new Promise((resolve) => probe.close(() => resolve())); - return `ws://127.0.0.1:${port}`; -} - -class DisposableFakeDriver extends FakeDriver { - constructor( - private readonly disposed: string[], - options: ConstructorParameters[0], - private readonly failure?: Error, - ) { - super(options); - } - - async dispose(): Promise { - this.disposed.push(this.platform); - if (this.failure !== undefined) { - throw this.failure; - } - } -} - -async function agentStatus(daemon: DaemonServer) { - return (await daemon.dispatch( - "status.get", - {}, - { manageEventSubscription: () => undefined, via: "socket", principal: "agent", role: "agent" }, - )) as { readonly host: { readonly tools: readonly unknown[] } }; -} - -class ToolsFakeDriver extends FakeDriver { - constructor( - options: ConstructorParameters[0], - private readonly tools: () => Promise, - ) { - super(options); - } - - toolVersions(): Promise { - return this.tools(); - } -} - -describe("startDaemon", () => { - it.each([ - { name: "not set", headers: undefined, label: undefined, expected: {} }, - { - name: "two headers", - headers: { "CF-Access-Client-Id": "abc.access", "CF-Access-Client-Secret": "s3cr3t" }, - label: undefined, - expected: { "cf-access-client-id": "abc.access", "cf-access-client-secret": "s3cr3t" }, - }, - { - name: "an empty value", - headers: { "x-empty": "" }, - label: undefined, - expected: { "x-empty": "" }, - }, - { - name: "a label set too", - headers: { "x-proxy-key": "k" }, - label: "mac-1", - expected: { "x-proxy-key": "k" }, - }, - ])( - "a worker started with gateway.headers sends every entry on its uplink dial, beside the join token and worker id: $name", - async ({ headers, label, expected }) => { - let received: IncomingHttpHeaders | undefined; - const gateway = createHttpServer(); - gateway.on("upgrade", (request, socket) => { - received = request.headers; - socket.destroy(); - }); - await new Promise((resolve) => gateway.listen(0, "127.0.0.1", resolve)); - try { - const { port } = gateway.address() as { readonly port: number }; - await start({ - configOverrides: { - gateway: { - token: "join-token", - url: `ws://127.0.0.1:${port}`, - ...(headers === undefined ? {} : { headers }), - ...(label === undefined ? {} : { label }), - }, - }, - }); - await vi.waitFor(() => expect(received).toBeDefined()); - const seen = received ?? {}; - expect({ - authorization: seen["authorization"], - workerId: typeof seen["x-simlock-worker-id"], - label: seen["x-simlock-worker-label"], - extra: Object.fromEntries(Object.entries(seen).filter(([name]) => name in expected)), - }).toEqual({ - authorization: "Bearer join-token", - workerId: "string", - label, - extra: expected, - }); - } finally { - await new Promise((resolve) => gateway.close(() => resolve())); - } - }, - ); + const clock = new FakeClock(1_000); + const socketPath = join(directory, "daemon.sock"); + const startPromise = startDaemon({ + clock, + configOverrides: STANDALONE_OFF, + dataDirectory: directory, + drivers: [ + new FakeDriver({ + availableOsVersions: ["26.5"], + clock, + latencyMs: { listManaged: 30_000 }, + platform: "ios", + }), + ], + deviceDataTables: fakeDeviceDataTables, + filesystem: new MemoryFilesystem(), + socketPath, + version: "1.2.3", + } as StartDaemonOptions).then((daemon) => { + runningDaemons.push(daemon); + return daemon; + }); - it.each([ - { role: "worker", withToken: true }, - { role: "gateway", withToken: false }, - ] as const)( - "keeps gateway.token and every gateway.headers value out of the daemon log, the events.replay answer, and history.db, while config.get returns them only as the redaction marker unless showSecrets is asked: $role", - async ({ role, withToken }) => { - const secret = "secret-join-token-170"; - const headerSecret = "s3cr3t-proxy-credential"; - const secondHeaderSecret = "second-proxy-credential"; - const { daemon, directory, sink } = await start({ - role, - configOverrides: { - gateway: { - headers: { - "CF-Access-Client-Id": "client-id-170", - "CF-Access-Client-Secret": headerSecret, - "X-Proxy-Key": secondHeaderSecret, - }, - ...(withToken ? { token: secret, url: await unreachableGatewayUrl() } : {}), - }, - }, + const client = await connectRetrying(socketPath); + try { + await client.request("hello", { + clientVersion: "test", + protocolVersion: DAEMON_PROTOCOL_VERSION, }); - const admin = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "operator", - role: "admin", - } as const; - type Answer = { - readonly gateway: { - readonly token?: string; - readonly headers?: Readonly>; - }; - }; - - const replayed = await daemon.dispatch("events.replay", {}, admin); - const eventFile = JSON.stringify( - await historyEventsWhen(directory, hasEvent("daemon.started")), - ); - const config = (await daemon.dispatch("config.get", {}, admin)) as Answer; + const starting = await client.request("status.get", {}); + expect(starting.payload).toMatchObject({ daemon: { health: "starting" } }); - // why: The fixture is real: the daemon did log its start and emit `daemon.started` to both - // the ring and the file, so "absent" below means redacted, not never written. - expect(sink.records.some((record) => record.message === "Daemon started")).toBe(true); - expect(eventFile).toContain('"daemon.started"'); - expect(JSON.stringify(replayed)).toContain('"daemon.started"'); - const leaks = (value: string) => ({ - daemonLog: JSON.stringify(sink.records).includes(value), - eventsReplay: JSON.stringify(replayed).includes(value), - historyDb: eventFile.includes(value), - }); - const clean = { daemonLog: false, eventsReplay: false, historyDb: false }; - expect({ - token: leaks(secret), - header: leaks(headerSecret), - second: leaks(secondHeaderSecret), - first: leaks("client-id-170"), - }).toEqual({ - token: clean, - header: clean, - second: clean, - first: clean, + const parkedLease = client.request("lease.request", { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", }); - const shown = (await daemon.dispatch("config.get", { showSecrets: true }, admin)) as Answer; - expect({ - default: config.gateway.headers, - shown: shown.gateway.headers, - token: [config.gateway.token, shown.gateway.token], - }).toEqual({ - default: { - "CF-Access-Client-Id": "[redacted]", - "CF-Access-Client-Secret": "[redacted]", - "X-Proxy-Key": "[redacted]", - }, - shown: { - "CF-Access-Client-Id": "client-id-170", - "CF-Access-Client-Secret": headerSecret, - "X-Proxy-Key": secondHeaderSecret, - }, - token: withToken ? ["[redacted]", secret] : [undefined, undefined], + let leaseSettled = false; + void parkedLease.then(() => { + leaseSettled = true; }); - }, - ); - - it.each([ - { - name: "nothing set: each driver's own defaults", - drivers: {}, - expected: (home: string) => ({ - android: { adbServerPort: 5038, deviceRoot: join(home, "devices", "android") }, - ios: { deviceRoot: join(home, "devices", "ios"), deviceSet: "simlock" }, - }), - }, - { - name: "set keys win over the defaults, and a driver's unknown key is carried", - drivers: { android: { adbServerPort: 5039 }, ios: { deviceSet: "simlock", retries: 3 } }, - expected: (home: string) => ({ - android: { adbServerPort: 5039, deviceRoot: join(home, "devices", "android") }, - ios: { deviceRoot: join(home, "devices", "ios"), deviceSet: "simlock", retries: 3 }, - }), - }, - { - name: "deviceSet default: no iOS deviceRoot, which that set does not use", - drivers: { ios: { deviceSet: "default" } }, - expected: (home: string) => ({ - android: { adbServerPort: 5038, deviceRoot: join(home, "devices", "android") }, - ios: { deviceSet: "default" }, - }), - }, - ])( - "answers config.get with every drivers.* key the drivers use, so config get reads it back ($name)", - async ({ drivers, expected }) => { - const { daemon, directory } = await start({ configOverrides: { drivers } }); + // why: Round-tripping another status.get proves the parked request was already + // dispatched (and is awaiting the readiness gate) without relying on a timer. + await client.request("status.get", {}); + expect(leaseSettled).toBe(false); - const config = (await daemon.dispatch( - "config.get", - {}, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "operator", - role: "admin", - }, - )) as { readonly drivers?: unknown }; + clock.advance(30_000); + const daemon = await startPromise; + expect(daemon).toBeDefined(); - expect(config.drivers).toEqual(expected(directory)); - }, - ); + await expect(parkedLease).resolves.toMatchObject({ ok: true }); + const running = await client.request("status.get", {}); + expect(running.payload).toMatchObject({ daemon: { health: "running" } }); + } finally { + client.socket.end(); + } + }); +}); - it("keeps gateway.token in the daemon.started payload and the Daemon started log line as the redaction marker, so a set token still reads as set", async () => { - const { daemon, sink } = await start({ - configOverrides: { - gateway: { token: "secret-join-token-170", url: await unreachableGatewayUrl() }, - }, +describe("startDaemon with HTTP off", () => { + it("starts no HTTP gateway when http.enabled is false", async () => { + const { port } = await startOnFreePort((port) => + start({ configOverrides: { http: { enabled: false, host: "127.0.0.1", port } } }), + ); + await new Promise((resolve) => setTimeout(resolve, 200)); + const outcome = await new Promise<"connected" | "refused">((resolve) => { + const socket = connect(port, "127.0.0.1"); + socket.once("connect", () => { + socket.destroy(); + resolve("connected"); + }); + socket.once("error", () => resolve("refused")); }); - const replayed = (await daemon.dispatch( - "events.replay", - {}, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "operator", - role: "admin", - }, - )) as readonly { readonly event: string; readonly payload: unknown }[]; - - const started = replayed.find((envelope) => envelope.event === "daemon.started"); - const logged = sink.records.find((record) => record.message === "Daemon started"); - const snapshot = (started?.payload as { readonly configSnapshot?: Config } | undefined) - ?.configSnapshot; - const loggedConfig = logged?.fields?.config as Config | undefined; - expect({ - eventToken: snapshot?.gateway.token, - logToken: loggedConfig?.gateway.token, - }).toEqual({ eventToken: REDACTED_VALUE, logToken: REDACTED_VALUE }); + expect(outcome).toBe("refused"); }); +}); - it("runs no prerequisite check at startup convergence, and runs every check on each doctor.run", async () => { - const runs = { android: 0, ios: 0 }; - const missing = { message: "gone", prerequisite: "android-emulator", remedy: "install it" }; - const { daemon } = await start({ - prerequisiteChecks: [ - { - check: () => { - runs.ios += 1; - return Promise.resolve([]); - }, - platform: "ios", - }, - { - check: () => { - runs.android += 1; - return Promise.resolve([missing]); - }, - platform: "android", - }, - ], - }); - const admin = { +describe("startDaemon HTTP status for a quarantined device", () => { + it("gives an operator token the cause's message and an agent token its kind alone", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-main-http-cause-")); + temporaryDirectories.push(directory); + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + await seedTokensDatabase(directory, [ + ["tok_agent", "slk_cause_agent", "agent"], + ["tok_op", "slk_cause_operator", "operator"], + ]); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); + const { port, started: daemon } = await startOnFreePort((port) => + startDaemon({ + deviceDataTables: fakeDeviceDataTables, + clock, + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + dataDirectory: directory, + drivers: [driver], + filesystem, + hostLoadProbe: null, + version: "1.2.3", + } as StartDaemonOptions), + ); + runningDaemons.push(daemon); + const session = { manageEventSubscription: () => undefined, + principal: "agent-1", + role: "agent", via: "socket", - principal: "operator", - role: "admin", } as const; + const grant = (await daemon.dispatch( + "lease.request", + { model: "iPhone 16", osVersion: "26.5", platform: "ios", requesterId: "agent-1" }, + session, + )) as { readonly lease: { readonly id: string } }; + await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, session); + await new Promise((resolve) => setTimeout(resolve, 25)); - // why: doctor.run parks until startup convergence has finished, so by the time it answers, - // convergence's own reconcile has run -- and must not have counted. - const first = (await daemon.dispatch("doctor.run", {}, admin)) as { - readonly findings: readonly { readonly kind: string; readonly prerequisite?: string }[]; + const causeFor = async (secret: string) => { + const status = (await pollUntilListening(port, secret)) as { + readonly devices?: readonly { + readonly quarantineCause?: unknown; + readonly state: string; + }[]; + }; + expect(status.devices?.[0]?.state).toBe("quarantined"); + return status.devices?.[0]?.quarantineCause; }; - expect(runs).toEqual({ android: 1, ios: 1 }); - expect(first.findings).toContainEqual( - expect.objectContaining({ kind: "prerequisite-missing", platform: "android" }), - ); - await daemon.dispatch("doctor.run", {}, admin); - expect(runs).toEqual({ android: 2, ios: 2 }); + expect(await causeFor("slk_cause_operator")).toEqual({ + kind: "delete-failed", + message: "DriverCrashError: purge exploded", + }); + expect(await causeFor("slk_cause_agent")).toEqual({ kind: "delete-failed" }); }); +}); - it("a worker's doctor reports service-broken for each service whose paths are gone", async () => { - const gone = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; - const mainGone = "/Users/me/simlock/dist/daemon/main.js"; - const name = "com.callstack.simlock.worker"; - const remedy = "Run `simlock daemon install` again."; - const admin = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "operator", - role: "admin", - } as const; - for (const testCase of [ - { label: "not installed", missing: [], expected: [] }, - { label: "installed and whole", missing: [], expected: [] }, - { - label: "one path gone", - missing: [gone], - expected: [{ kind: "service-broken", manager: "launchd", name, missing: [gone], remedy }], - }, +class UnmeasurableDiskFilesystem extends MemoryFilesystem { + async diskFree(_path: string): Promise { + throw Object.assign(new Error("statfs failed"), { code: "EIO" }); + } +} + +describe("startDaemon logger wiring", () => { + it("startDaemon hands its logger to the cleanup reaper: a scheduled cleanup run that fails is logged", async () => { + const { sink } = await start({ filesystem: new UnmeasurableDiskFilesystem() }); + + // why: `daemon.started` triggers a run nobody awaits; its failure has no caller to reach. + await pollUntil(() => + sink.records.some((record) => record.message === "scheduled cleanup run failed"), + ); + expect( + sink.records.filter((record) => record.message === "scheduled cleanup run failed"), + ).toMatchObject([ { - label: "both gone", - missing: [gone, mainGone], - expected: [ - { kind: "service-broken", manager: "launchd", name, missing: [gone, mainGone], remedy }, - ], + level: "error", + module: "daemon.reaper", + fields: { step: "cleanup", error: expect.stringContaining("statfs failed") }, }, - ]) { - const { daemon } = await start({ - serviceCheck: { manager: "launchd", name, check: async () => testCase.missing }, - }); - const report = (await daemon.dispatch("doctor.run", {}, admin)) as { - readonly findings: readonly { readonly kind: string }[]; - }; - expect( - report.findings.filter((finding) => finding.kind === "service-broken"), - testCase.label, - ).toEqual(testCase.expected); - } + ]); }); - it("productionServiceCheck builds the service definition from the process facts and reads its state: the missing paths when broken, none otherwise, and no check on a gateway or without a manager", async () => { - const seen: ServiceDefinition[] = []; - const manager: ServiceManager = { - manager: "launchd", - state: async (definition) => { - seen.push(definition); - return { state: "broken", path: definition.path, missing: [definition.command] }; - }, - install: async () => undefined, - uninstall: async () => undefined, - start: async () => undefined, - }; - const host = { - manager, - root: "/Users/me/.simlock", - home: "/Users/me", - nodePath: "/gone/node", - mainPath: "/gone/main.js", - env: { PATH: "/usr/bin" }, - }; - - const { serviceCheck } = productionServiceCheck("worker", host); - - expect(serviceCheck?.name).toBe("com.callstack.simlock.worker"); - expect(await serviceCheck?.check()).toEqual(["/gone/node"]); - expect(seen).toEqual([ - serviceDefinition({ - role: "worker", - root: "/Users/me/.simlock", - dataDirectory: daemonDataDirectory("/Users/me/.simlock", "worker"), - home: "/Users/me", - nodePath: "/gone/node", - mainPath: "/gone/main.js", - env: { PATH: "/usr/bin" }, - }), - ]); - expect(productionServiceCheck("gateway", host)).toEqual({}); - expect(productionServiceCheck("worker", { ...host, manager: undefined })).toEqual({}); + it("a gateway rejects worker.drain for an unknown worker with UNKNOWN_WORKER", async () => { + // why: A gateway serves HTTP; on its default port two overlapping runs of this file collide. + const { daemon } = await start({ role: "gateway" }); - const states: readonly { readonly state: ServiceState; readonly missing: readonly string[] }[] = - [ - { state: { state: "not-installed" }, missing: [] }, - { state: { state: "installed", path: "/p" }, missing: [] }, - { state: { state: "not-loaded", path: "/p" }, missing: [] }, - { - state: { state: "broken", path: "/p", missing: ["/gone/node", "/gone/main.js"] }, - missing: ["/gone/node", "/gone/main.js"], - }, - ]; - for (const testCase of states) { - const checked = productionServiceCheck("worker", { - ...host, - manager: new FakeServiceManager({ state: testCase.state }), - }); - expect(checked.serviceCheck?.manager).toBe("launchd"); - expect(checked.serviceCheck?.name).toBe("com.callstack.simlock.worker"); - expect(await checked.serviceCheck?.check(), testCase.state.state).toEqual(testCase.missing); - } - expect( - productionServiceCheck("worker", { - ...host, - manager: new FakeServiceManager({ manager: "systemd" }), - }).serviceCheck?.manager, - ).toBe("systemd"); - - const systemd = productionServiceCheck("worker", { - ...host, - manager: { ...manager, manager: "systemd" }, - home: "/home/me", - root: "/home/me/.simlock", - }); - await systemd.serviceCheck?.check(); - expect(seen.at(-1)?.path).toBe( - "/home/me/.config/systemd/user/com.callstack.simlock.worker.service", - ); - }); - - it("stores lease requests under the configured limits, with failures classified by contract code", async () => { - const { daemon, directory } = await start({ - configOverrides: { lease: { maxRequestRecords: 1 } }, - }); - const missingRuntime = { - model: "iPhone 16", - osVersion: "99.0", - platform: "ios", - } as const; - const session = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent", - role: "agent", - } as const; - - for (const requesterId of ["first", "second"]) { - await daemon - .dispatch("lease.request", { ...missingRuntime, requesterId }, session) - .catch(() => undefined); - } - - await expect - .poll(() => queryState(directory, "SELECT requester_id, failure_code FROM lease_requests")) - .toEqual([{ failure_code: "RUNTIME_MISSING", requester_id: "second" }]); - }); - - it("establishes the instance identity once, and reuses it on the next start", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-identity-")); - temporaryDirectories.push(directory); - const filesystem = new MemoryFilesystem(); - const identityPath = join(directory, "instance.json"); - - const first = await start({ dataDirectory: directory, filesystem }); - const written = JSON.parse(await filesystem.readFile(identityPath)) as { - readonly instanceId: string; - }; - await first.daemon.stop("test"); - await start({ dataDirectory: directory, filesystem }); - - // why: Never regenerated: a second id would strand every device already sitting in a root - // marked with the first one (ADR 0001, decision 2). - expect(written.instanceId).not.toBe(""); - expect(JSON.parse(await filesystem.readFile(identityPath))).toEqual(written); - }); - - it("disposes every driver on shutdown, and reports the one that failed without stopping", async () => { - // why: The wiring, not the halves: `DaemonServer` awaiting its `dispose` option is tested in - // `server.test.ts` and a driver reaping its adb server in the Android driver's own, and - // between them sat the composition that connects the two. Losing it silently abandons - // Simlock's adb server on every shutdown, and only `Driver.dispose` can stop one. - const disposed: string[] = []; - const clock = new FakeClock(1_000); - const { daemon, sink } = await start({ - drivers: [ - new DisposableFakeDriver(disposed, { - availableOsVersions: ["26.5"], - clock, - platform: "ios", - }), - new DisposableFakeDriver( - disposed, - { availableOsVersions: ["35"], clock, platform: "android" }, - new Error("adb server would not die"), - ), - ], - deviceDataTables: fakeDeviceDataTables, - }); - - await daemon.stop("test"); - - expect(disposed).toEqual(["ios", "android"]); - expect(sink.records).toContainEqual( - expect.objectContaining({ - level: "error", - message: "Driver disposal failed", - fields: expect.objectContaining({ platform: "android" }), - }), - ); - }); - - it("ends a running component install on shutdown, before the drivers are disposed", async () => { - const clock = new FakeClock(1_000); - const events: string[] = []; - class InstallingDriver extends FakeDriver { - override async installComponent( - component: string, - options: Parameters[1], - ) { - events.push("install-started"); - options.signal.addEventListener("abort", () => events.push("install-ended")); - return super.installComponent(component, options); - } - - async dispose(): Promise { - events.push("disposed"); - } - } - const driver = new InstallingDriver({ availableOsVersions: [], clock, platform: "ios" }); - driver.holdInstalls(); - const { daemon } = await start({ drivers: [driver] }); - - const lease = daemon - .dispatch( - "lease.request", - { - allowDownload: true, - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent", - role: "agent", - }, - ) - .catch((error: unknown) => error); - await expect.poll(() => events).toEqual(["install-started"]); - await daemon.stop("test"); - - expect(events).toEqual(["install-started", "install-ended", "disposed"]); - await expect(lease).resolves.toBeInstanceOf(Error); - }); - - it("starts and reports no entry for a driver whose tool read fails, and every other tool", async () => { - const clock = new FakeClock(1_000); - const { daemon } = await start({ - drivers: [ - new ToolsFakeDriver( - { availableOsVersions: ["26.5"], clock, platform: "ios" }, - // What an iOS driver whose `xcodebuild` fails looks like from here. - () => Promise.reject(new Error("xcodebuild -version exited with 1")), - ), - new ToolsFakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }, () => - Promise.resolve([{ name: "emulator", version: "35.4.9" }]), - ), - ], - deviceDataTables: fakeDeviceDataTables, - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - }); - - await expect - .poll(async () => (await agentStatus(daemon)).host) - .toEqual({ - arch: "arm64", - os: "macOS", - osVersion: "15.5", - tools: [{ name: "emulator", platform: "android", version: "35.4.9" }], - }); - }); - - it("answers status.get when a driver reports a tool version past the contract's bounds", async () => { - const clock = new FakeClock(1_000); - const { daemon } = await start({ - drivers: [ - new ToolsFakeDriver({ availableOsVersions: ["35"], clock, platform: "android" }, () => - Promise.resolve([ - { name: "emulator", version: "9".repeat(500) }, - { name: "platform-tools", version: "36.0.0" }, - ]), - ), - ], - deviceDataTables: fakeDeviceDataTables, - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - }); - - await expect - .poll(async () => (await agentStatus(daemon)).host.tools) - .toEqual([{ name: "platform-tools", platform: "android", version: "36.0.0" }]); - }); - - it("reads tool versions at start, before anything asks for status", async () => { - const clock = new FakeClock(1_000); - let reads = 0; - await start({ - drivers: [ - new ToolsFakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }, () => { - reads += 1; - return Promise.resolve([]); - }), - ], - deviceDataTables: fakeDeviceDataTables, - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - }); - - expect(reads).toBe(1); - }); - - it("still answers component.list when a driver's component listing fails", async () => { - const clock = new FakeClock(1_000); - const ios = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - ios.failOn("listComponents", 1, new Error("simctl runtime list failed")); - const { daemon } = await start({ drivers: [ios] }); - - const listed = await daemon.dispatch( - "component.list", - {}, - { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent", - role: "agent", - }, - ); - - expect(listed).toEqual({ components: [] }); - }); -}); - -describe("startDaemon wires core and leasing together", () => { - const agent = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent", - role: "agent", - } as const; - const admin = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "operator", - role: "admin", - } as const; - const ios = { - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - requesterId: "agent-1", - } as const; - type Grant = { - readonly lease: { readonly id: string }; - readonly device: { readonly id: string }; - }; - - /** why: Lets the work a mutated daemon would still be doing finish, so "nothing happened" is a - * statement about the daemon and not about how early the test looked. Only sound when that - * work is all in memory: a timer fires after it however loaded the machine is, but real I/O - * (a process spawn, a real file) can outlast it. */ - const settleRealTime = async () => new Promise((resolve) => setTimeout(resolve, 25)); - - async function leasedIos(options: ConstructorParameters[0]) { - const clock = options.clock as FakeClock; - const driver = new FakeDriver(options); - const filesystem = new MemoryFilesystem(); - const started = await start({ clock, drivers: [driver], filesystem }); - const grant = (await started.daemon.dispatch("lease.request", ios, agent)) as Grant; - return { ...started, clock, driver, filesystem, grant }; - } - - it("runs the operator reset through core's nuke, which releases the lease through leasing", async () => { - const clock = new FakeClock(1_000); - const { daemon, grant } = await leasedIos({ - availableOsVersions: ["26.5"], - clock, - platform: "ios", - }); - - const report = await daemon.dispatch("nuke.run", { deleteDevices: true }, admin); - - expect(report).toEqual({ - deletedDevices: [grant.device.id], - releasedLeaseIds: [grant.lease.id], - removedBaselines: [], - }); - }); - - it("runs the operator's warm.forget through core's warm pool, which answers the kinds it forgot", async () => { - const { daemon } = await start(); - - const report = await daemon.dispatch( - "warm.forget", - { model: "iPhone 17 Pro", platform: "ios" }, - admin, - ); - - expect(report).toEqual({ forgotten: [] }); - }); - - it("grants a waiting physical request once device.add enrolls a second device of the model", async () => { - const clock = new FakeClock(1_000); - const phone = { apps: [], class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" } as const; - const driver = new FakeDriver({ - clock, - physicalDevices: [ - { ...phone, ids: ["UDID-A"] }, - { ...phone, ids: ["UDID-B"] }, - ], - platform: "ios", - }); - driver.setPhysicalPresence("UDID-A", true); - driver.setPhysicalPresence("UDID-B", true); - const { daemon } = await start({ clock, drivers: [driver] }); - const physical = { model: "iPhone 15 Pro", physical: true, platform: "ios" } as const; - await daemon.dispatch("device.add", { id: "UDID-A", platform: "ios" }, admin); - await daemon.dispatch("lease.request", { ...physical, requesterId: "agent-1" }, agent); - const waiting = daemon.dispatch( - "lease.request", - { ...physical, requesterId: "agent-2" }, - agent, - ) as Promise; - await settleRealTime(); - - await daemon.dispatch("device.add", { id: "UDID-B", platform: "ios" }, admin); - - const outcome = await Promise.race([waiting, settleRealTime().then(() => "still waiting")]); - expect(outcome).toMatchObject({ device: { physical: true } }); - }); - - it("status.get carries the presence the grant's read found on a physical device, through the daemon's own wiring", async () => { - const clock = new FakeClock(1_000); - const phone = { apps: [], class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" } as const; - const driver = new FakeDriver({ - clock, - physicalDevices: [{ ...phone, ids: ["UDID-WIRED-PRESENCE"] }], - platform: "ios", - }); - driver.setPhysicalPresence("UDID-WIRED-PRESENCE", true); - const { daemon } = await start({ clock, drivers: [driver] }); - await daemon.dispatch("device.add", { id: "UDID-WIRED-PRESENCE", platform: "ios" }, admin); - await daemon.dispatch( - "lease.request", - { model: "iPhone 15 Pro", physical: true, platform: "ios", requesterId: "agent-1" }, - agent, - ); - - const status = (await daemon.dispatch("status.get", {}, agent)) as { - devices?: { driverDeviceId?: string; presence?: unknown }[]; - }; - - const entry = status.devices?.find((device) => device.driverDeviceId === "UDID-WIRED-PRESENCE"); - expect(entry).toMatchObject({ presence: { present: true } }); - }); - - it("starts a device.exec child from the allowlisted keys of the daemon's environment plus the command's own env, without a secret the daemon inherited", async () => { - const clock = new FakeClock(1_000); - const runner = new ScriptedProcessRunner([ - { match: { args: ["list", "devices"], command: "fake-simctl" } }, - ]); - const driver = new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - passthrough: () => ({ - args: ["list", "devices"], - command: "fake-simctl", - env: { SIMLOCK_SCOPED: "1" }, - }), - passthroughTools: ["simctl"], - platform: "ios", - }); - vi.stubEnv("AWS_SECRET_ACCESS_KEY", "aws-secret"); - vi.stubEnv("LC_CTYPE", "simlock-test-locale"); - try { - const { daemon } = await start({ - clock, - drivers: [driver], - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - processRunner: runner, - }); - const grant = (await daemon.dispatch("lease.request", ios, agent)) as Grant; - - await daemon.dispatch( - "device.exec", - { args: ["list", "devices"], leaseId: grant.lease.id, tool: "simctl" }, - agent, - ); - - const env = runner.calls.find((call) => call.command === "fake-simctl")?.options.env; - expect(env).toMatchObject({ LC_CTYPE: "simlock-test-locale", SIMLOCK_SCOPED: "1" }); - expect(env?.PATH).toBe(process.env.PATH); - expect(env).not.toHaveProperty("AWS_SECRET_ACCESS_KEY"); - } finally { - vi.unstubAllEnvs(); - } - }); - - it("announces the capacity figures and then the queue depth once startup is done", async () => { - const { directory } = await start(); - - const events = await historyEventsWhen(directory, hasEvent("queue.changed")); - const names = events.map((event) => event.event); - - expect(names.indexOf("capacity.changed")).toBeGreaterThan(-1); - expect(names.indexOf("queue.changed")).toBeGreaterThan(names.indexOf("capacity.changed")); - expect(events.find((event) => event.event === "queue.changed")?.module).toBe("wait-queue"); - }); - - it("runs doctor's startup pass on the startup read before core's device steps announce the capacity figures", async () => { - const { directory } = await start(); - - const names = (await historyEventsWhen(directory, hasEvent("capacity.changed"))).map( - (event) => event.event, - ); - - expect(names.indexOf("doctor.reconciled")).toBeGreaterThan(-1); - expect(names.indexOf("doctor.reconciled")).toBeLessThan(names.indexOf("capacity.changed")); - }); - - it("marks a saved baseline whose runtime is gone as stale at start, before anything lists baselines", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); - temporaryDirectories.push(directory); - const state = await openStateDatabase({ - dataDirectory: directory, - filesystem: new MemoryFilesystem(), - logger: new NoopLogger(), - oldFiles: [], - owners: [registryOwner, fakeDeviceDataTable, baselinesOwner], - sqlite: new NodeSqliteOpener(), - upgradeStep: "unused", - }); - state.database - .prepare( - "INSERT INTO baselines (name, version, platform, model, os_version, mode, saved_at, source_id) VALUES ('login', 1, 'ios', 'iPhone 17 Pro', '26.5', 'full', '2026-11-02T13:40:00.000Z', 'src-login')", - ) - .run(); - state.close(); - const reason = "runtime com.apple.CoreSimulator.SimRuntime.iOS-26-5 is not installed"; - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.setSources([ - { - id: "src-login", - kind: "baseline", - model: "iPhone 17 Pro", - osVersion: "26.5", - staleReason: reason, - }, - ]); - - const daemon = await startDaemon({ - clock, - configOverrides: STANDALONE_OFF, - dataDirectory: directory, - deviceDataTables: fakeDeviceDataTables, - drivers: [driver], - filesystem: new MemoryFilesystem(), - logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), - version: "1.2.3", - }); - runningDaemons.push(daemon); - - const stale = await historyEventsWhen(directory, hasEvent("baseline.stale")); - expect( - stale.filter((event) => event.event === "baseline.stale").map((event) => event.payload), - ).toEqual([{ name: "login", reason, version: 1 }]); - }); - - it("settles the requests a restart left open before core's device steps announce the capacity figures", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); - temporaryDirectories.push(directory); - const state = await openStateDatabase({ - dataDirectory: directory, - filesystem: new MemoryFilesystem(), - logger: new NoopLogger(), - oldFiles: [], - owners: [registryOwner, fakeDeviceDataTable], - sqlite: new NodeSqliteOpener(), - upgradeStep: "unused", - }); - const previous = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "1" }, - state, - }); - await previous.createLeaseRequest({ - ownerId: "agent-1", - request: ios, - requesterId: "agent-1", - }); - state.close(); - - const daemon = await startDaemon({ - clock, - configOverrides: STANDALONE_OFF, - dataDirectory: directory, - deviceDataTables: fakeDeviceDataTables, - drivers: [new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" })], - filesystem: new MemoryFilesystem(), - logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), - version: "1.2.3", - }); - runningDaemons.push(daemon); - - const names = (await historyEventsWhen(directory, hasEvent("capacity.changed"))).map( - (event) => event.event, - ); - expect(names.indexOf("lease.rejected")).toBeGreaterThan(-1); - expect(names.indexOf("capacity.changed")).toBeGreaterThan(names.indexOf("lease.rejected")); - }); - - it("ends the lease whose device is not running before core's device steps announce capacity, so a restart no longer holds it", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); - temporaryDirectories.push(directory); - const options = () => - ({ - clock, - configOverrides: STANDALONE_OFF, - dataDirectory: directory, - deviceDataTables: fakeDeviceDataTables, - drivers: [new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" })], - filesystem: new MemoryFilesystem(), - logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), - // why: capacity follows the host's cpus and memory, and a small runner has too few slots for three leases at once, so a later request waits for ever on the fake clock (30 s timeout in runs 39459769408 and 39457371326). - systemStats: new FakeSystemStats({ cpuCount: 16, totalRamBytes: 64 * 1024 ** 3 }), - version: "1.2.3", - }) as StartDaemonOptions; - const first = await startDaemon(options()); - // why: several leases, so a reconcile that runs beside core's steps lands some releases after the announcement. - const grants: Grant[] = []; - for (const requesterId of ["agent-1", "agent-2", "agent-3"]) { - grants.push((await first.dispatch("lease.request", { ...ios, requesterId }, agent)) as Grant); - } - await first.stop("test"); - // why: the history orders events by their time, and the second start's must follow the first's. - clock.advance(1); - const secondRunStart = clock.now(); - - const daemon = await startDaemon(options()); - runningDaemons.push(daemon); - - const leases = (await daemon.dispatch("list.get", { kind: "leases" }, admin)) as readonly { - readonly id: string; - }[]; - for (const grant of grants) { - expect(leases.map((lease) => lease.id)).not.toContain(grant.lease.id); - } - const secondRun = ( - await historyEventsWhen(directory, (all) => - all.some( - (event) => event.event === "capacity.changed" && event.timestamp >= secondRunStart, - ), - ) - ) - .filter((event) => event.timestamp >= secondRunStart) - .sort((left, right) => left.seq - right.seq) - .map((event) => event.event); - expect(secondRun.filter((name) => name === "lease.released")).toHaveLength(grants.length); - expect(secondRun.lastIndexOf("lease.released")).toBeLessThan( - secondRun.indexOf("capacity.changed"), - ); - // why: two daemon starts and a history wait of up to 4 s outrun the 5 s default on a loaded CI runner (run 39454292501 timed out at 5015 ms, 59 ms locally). - }, 30_000); - - it("closes state.db on stop, and lists the same device and lease after a worker is stopped and started again", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-")); - temporaryDirectories.push(directory); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const options = () => - ({ - clock, - configOverrides: STANDALONE_OFF, - dataDirectory: directory, - deviceDataTables: fakeDeviceDataTables, - drivers: [driver], - filesystem: new MemoryFilesystem(), - logger: new JsonLinesLogger({ clock, level: "debug", sink: new MemoryLogSink() }), - version: "1.2.3", - }) as StartDaemonOptions; - const first = await startDaemon(options()); - const grant = (await first.dispatch("lease.request", ios, agent)) as Grant; - const list = async (daemon: Awaited>, kind: string) => - ( - (await daemon.dispatch("list.get", { kind }, admin)) as readonly { readonly id: string }[] - ).map((row) => row.id); - const devicesBefore = await list(first, "devices"); - await first.stop("test"); - // why: closing the last connection checkpoints and removes the WAL, so a stop that left state.db open leaves it. - expect(existsSync(join(directory, "state.db-wal"))).toBe(false); - - const daemon = await startDaemon(options()); - runningDaemons.push(daemon); - - expect(devicesBefore).toContain(grant.device.id); - expect(await list(daemon, "devices")).toEqual(devicesBefore); - expect(await list(daemon, "leases")).toEqual([grant.lease.id]); - }); - - it("holds shutdown open while a release's erase runs, and does not read that erase as a stall", async () => { - const clock = new FakeClock(1_000); - const { daemon, driver, grant } = await leasedIos({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { destroy: 600_000 }, - platform: "ios", - }); - await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); - await settleRealTime(); - - clock.advance(300_000); - const devices = (await daemon.dispatch("list.get", { kind: "devices" }, admin)) as readonly { - readonly id: string; - readonly state: string; - readonly stalled?: true; - }[]; - expect(devices.map(({ id, state, stalled }) => ({ id, state, stalled }))).toEqual([ - { id: grant.device.id, state: "shutdown", stalled: undefined }, - ]); - - let stopped = false; - const stopping = daemon.stop("test").then(() => { - stopped = true; - }); - await settleRealTime(); - expect(stopped).toBe(false); - clock.advance(300_000); - await stopping; - expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); - }); - - it("cancels the expiry timer of an outstanding lease on shutdown, so nothing expires after it", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const { daemon, directory } = await start({ clock, drivers: [driver] }); - await daemon.dispatch("lease.request", { ...ios, ttlMs: 10_000 }, agent); - const leaseIds = async () => - (await queryState(directory, "SELECT id FROM leases")).map((row) => row.id); - const before = await leaseIds(); - expect(before).toHaveLength(1); - - await daemon.stop("test"); - clock.advance(10_000); - await settleRealTime(); - - expect(await leaseIds()).toEqual(before); - }); - - it("cancels the quarantine retry timer on shutdown, so no retry runs after it", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - const started = await start({ clock, drivers: [driver] }); - const grant = (await started.daemon.dispatch("lease.request", ios, agent)) as Grant; - await started.daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); - - await started.daemon.stop("test"); - clock.advance(30_000); - await settleRealTime(); - - expect(driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); - }); - - describe("a device quarantined by a failed delete", () => { - type CauseView = { - readonly id: string; - readonly quarantineCause?: { readonly kind: string; readonly message?: string }; - readonly state: string; - }; - - async function quarantinedAfterFailedPurge() { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - const { daemon } = await start({ - clock, - configOverrides: { warmPool: { enabled: false } }, - drivers: [driver], - }); - const grant = (await daemon.dispatch("lease.request", ios, agent)) as Grant; - await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); - await settleRealTime(); - return daemon; - } - - const devicesOf = async ( - daemon: Awaited>, - session: typeof agent | typeof admin, - ) => - ( - (await daemon.dispatch("status.get", {}, session)) as { - readonly devices: readonly CauseView[]; - } - ).devices; - - it("shows quarantineCause delete-failed with the driver's error in status.get and list.get", async () => { - const daemon = await quarantinedAfterFailedPurge(); - - const status = await devicesOf(daemon, admin); - const listed = (await daemon.dispatch( - "list.get", - { kind: "devices" }, - admin, - )) as readonly CauseView[]; - - for (const devices of [status, listed]) { - expect(devices).toHaveLength(1); - expect(devices[0]).toMatchObject({ state: "quarantined" }); - expect(devices[0]?.quarantineCause).toEqual({ - kind: "delete-failed", - message: "DriverCrashError: purge exploded", - }); - } - }); - }); - - it("probes a leased device again as time passes once startup is done", async () => { - const clock = new FakeClock(1_000); - const { driver } = await leasedIos({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const probesBefore = driver.calls.filter((call) => call.operation === "listManaged").length; - - clock.advance(30_000); - await settleRealTime(); - - expect(driver.calls.filter((call) => call.operation === "listManaged").length).toBeGreaterThan( - probesBefore, - ); - }); -}); - -describe("startDaemon startup readiness", () => { - it("claims the socket and answers hello/status.get while doctor.reconcile is still in flight", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-slow-")); - temporaryDirectories.push(directory); - const clock = new FakeClock(1_000); - const socketPath = join(directory, "daemon.sock"); - const startPromise = startDaemon({ - clock, - configOverrides: STANDALONE_OFF, - dataDirectory: directory, - drivers: [ - new FakeDriver({ - availableOsVersions: ["26.5"], - clock, - latencyMs: { listManaged: 30_000 }, - platform: "ios", - }), - ], - deviceDataTables: fakeDeviceDataTables, - filesystem: new MemoryFilesystem(), - socketPath, - version: "1.2.3", - } as StartDaemonOptions).then((daemon) => { - runningDaemons.push(daemon); - return daemon; - }); - - const client = await connectRetrying(socketPath); - try { - await client.request("hello", { - clientVersion: "test", - protocolVersion: DAEMON_PROTOCOL_VERSION, - }); - const starting = await client.request("status.get", {}); - expect(starting.payload).toMatchObject({ daemon: { health: "starting" } }); - - const parkedLease = client.request("lease.request", { - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - }); - let leaseSettled = false; - void parkedLease.then(() => { - leaseSettled = true; - }); - // why: Round-tripping another status.get proves the parked request was already - // dispatched (and is awaiting the readiness gate) without relying on a timer. - await client.request("status.get", {}); - expect(leaseSettled).toBe(false); - - clock.advance(30_000); - const daemon = await startPromise; - expect(daemon).toBeDefined(); - - await expect(parkedLease).resolves.toMatchObject({ ok: true }); - const running = await client.request("status.get", {}); - expect(running.payload).toMatchObject({ daemon: { health: "running" } }); - } finally { - client.socket.end(); - } - }); -}); - -describe("startDaemon with HTTP off", () => { - it("starts no HTTP gateway when http.enabled is false", async () => { - const { port } = await startOnFreePort((port) => - start({ configOverrides: { http: { enabled: false, host: "127.0.0.1", port } } }), - ); - await new Promise((resolve) => setTimeout(resolve, 200)); - const outcome = await new Promise<"connected" | "refused">((resolve) => { - const socket = connect(port, "127.0.0.1"); - socket.once("connect", () => { - socket.destroy(); - resolve("connected"); - }); - socket.once("error", () => resolve("refused")); - }); - expect(outcome).toBe("refused"); - }); -}); - -describe("startDaemon HTTP status for a quarantined device", () => { - it("gives an operator token the cause's message and an agent token its kind alone", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-http-cause-")); - temporaryDirectories.push(directory); - const clock = new FakeClock(1_000); - const filesystem = new MemoryFilesystem(); - await seedTokensDatabase(directory, [ - ["tok_agent", "slk_cause_agent", "agent"], - ["tok_op", "slk_cause_operator", "operator"], - ]); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - driver.failOn("destroy", 1, new DriverCrashError("purge exploded")); - const { port, started: daemon } = await startOnFreePort((port) => - startDaemon({ - deviceDataTables: fakeDeviceDataTables, - clock, - configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, - dataDirectory: directory, - drivers: [driver], - filesystem, - hostLoadProbe: null, - version: "1.2.3", - } as StartDaemonOptions), - ); - runningDaemons.push(daemon); - const session = { - manageEventSubscription: () => undefined, - principal: "agent-1", - role: "agent", - via: "socket", - } as const; - const grant = (await daemon.dispatch( - "lease.request", - { model: "iPhone 16", osVersion: "26.5", platform: "ios", requesterId: "agent-1" }, - session, - )) as { readonly lease: { readonly id: string } }; - await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, session); - await new Promise((resolve) => setTimeout(resolve, 25)); - - const causeFor = async (secret: string) => { - const status = (await pollUntilListening(port, secret)) as { - readonly devices?: readonly { - readonly quarantineCause?: unknown; - readonly state: string; - }[]; - }; - expect(status.devices?.[0]?.state).toBe("quarantined"); - return status.devices?.[0]?.quarantineCause; - }; - - expect(await causeFor("slk_cause_operator")).toEqual({ - kind: "delete-failed", - message: "DriverCrashError: purge exploded", - }); - expect(await causeFor("slk_cause_agent")).toEqual({ kind: "delete-failed" }); - }); -}); - -describe("startDaemon event history", () => { - const previousModule = process.env.SIMLOCK_DRIVERS_MODULE; - - afterEach(() => { - if (previousModule === undefined) { - delete process.env.SIMLOCK_DRIVERS_MODULE; - } else { - process.env.SIMLOCK_DRIVERS_MODULE = previousModule; - } - }); - - it("starts a daemon whose history path is a directory, sets it aside and writes a fresh file", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); - temporaryDirectories.push(directory); - await mkdir(join(directory, "history.db")); - - const { daemon } = await start({ dataDirectory: directory }); - - const events = await historyEventsWhen(directory, hasEvent("daemon.started")); - const names = await readdir(directory); - const status = (await daemon.dispatch( - "status.get", - {}, - { - manageEventSubscription: () => undefined, - principal: "operator", - role: "admin", - via: "socket", - }, - )) as { readonly history?: unknown }; - expect(daemon.health).toBe("running"); - expect(names).toContain("history.db.damaged-1000"); - expect((await stat(join(directory, "history.db.damaged-1000"))).isDirectory()).toBe(true); - expect((await stat(join(directory, "history.db"))).isFile()).toBe(true); - expect(events.length).toBeGreaterThan(0); - expect(status.history).toEqual({ opened: true, setAsideAt: 1_000, writing: true }); - }); - - it("a daemon that loses the socket claim leaves a damaged history.db byte-identical and in place", async () => { - const winner = await mkdtemp(join(tmpdir(), "simlock-main-claim-")); - const loser = await mkdtemp(join(tmpdir(), "simlock-main-claim-")); - temporaryDirectories.push(winner, loser); - const junk = "this is not a sqlite database ".repeat(20); - await writeFile(join(loser, "history.db"), junk); - const socketPath = join(winner, "daemon.sock"); - await start({ dataDirectory: winner, socketPath }); - - const outcome = await start({ dataDirectory: loser, socketPath }).then( - () => "resolved" as const, - () => "rejected" as const, - ); - - expect(outcome).toBe("rejected"); - expect( - await readdir(loser).then((names) => names.filter((name) => name.startsWith("history"))), - ).toEqual(["history.db"]); - expect(await readFile(join(loser, "history.db"), "utf8")).toBe(junk); - }); - - it("mints event ids from the idGenerator the daemon was given", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); - temporaryDirectories.push(directory); - process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( - `export function createDrivers({ eventBus }) { - eventBus.emit("component.installed", { componentId: "18.6", durationMs: 1, platform: "ios" }, "test"); - return []; - }`, - ); - - await start({ - dataDirectory: directory, - drivers: undefined, - idGenerator: { generate: () => "from-the-given-generator" }, - }); - - const ids = ( - await historyEventsWhen(directory, (events) => - events.some((event) => event.id === "evt_from-the-given-generator"), - ) - ).map((event) => event.id); - expect(ids).toContain("evt_from-the-given-generator"); - }); - - it("keeps component.installed and device.slimmed in the event history", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-event-file-")); - temporaryDirectories.push(directory); - process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( - `export function createDrivers({ eventBus }) { - eventBus.emit("component.installed", { componentId: "18.6", durationMs: 1, platform: "ios" }, "test"); - eventBus.emit("device.slimmed", { - address: "a", categories: [], deviceId: "d", durationMs: 1, labelCount: 0, - platform: "ios", signature: "s", unknownLabels: [], - }, "test"); - return []; - }`, - ); - - await start({ dataDirectory: directory, drivers: undefined }); - - const written = (await historyEventsWhen(directory, hasEvent("device.slimmed"))).map( - (event) => event.event, - ); - expect(written).toEqual(expect.arrayContaining(["component.installed", "device.slimmed"])); - }); -}); - -class UnmeasurableDiskFilesystem extends MemoryFilesystem { - async diskFree(_path: string): Promise { - throw Object.assign(new Error("statfs failed"), { code: "EIO" }); - } -} - -describe("startDaemon logger wiring", () => { - it("startDaemon hands its logger to the cleanup reaper: a scheduled cleanup run that fails is logged", async () => { - const { sink } = await start({ filesystem: new UnmeasurableDiskFilesystem() }); - - // why: `daemon.started` triggers a run nobody awaits; its failure has no caller to reach. - await pollUntil(() => - sink.records.some((record) => record.message === "scheduled cleanup run failed"), - ); - expect( - sink.records.filter((record) => record.message === "scheduled cleanup run failed"), - ).toMatchObject([ - { - level: "error", - module: "daemon.reaper", - fields: { step: "cleanup", error: expect.stringContaining("statfs failed") }, - }, - ]); - }); - - it("a gateway rejects worker.drain for an unknown worker with UNKNOWN_WORKER", async () => { - // why: A gateway serves HTTP; on its default port two overlapping runs of this file collide. - const { daemon } = await start({ role: "gateway" }); - - await expect( - daemon.dispatch( - "worker.drain", - { workerId: "wrk_ghost" }, + await expect( + daemon.dispatch( + "worker.drain", + { workerId: "wrk_ghost" }, { manageEventSubscription: () => undefined, via: "socket", @@ -1834,541 +504,3 @@ describe("daemon role (ADR 0034)", () => { }, ); }); - -describe("startDaemon state.db", () => { - const previousModule = process.env.SIMLOCK_DRIVERS_MODULE; - - afterEach(() => { - if (previousModule === undefined) { - delete process.env.SIMLOCK_DRIVERS_MODULE; - } else { - process.env.SIMLOCK_DRIVERS_MODULE = previousModule; - } - }); - - it.each([ - [ - "worker", - "state.json", - "with the older Simlock, release every lease and run `simlock nuke --delete-devices`; then remove and start again.", - ], - [ - "worker", - "tokens.json", - "with the older Simlock, release every lease and run `simlock nuke --delete-devices`; then remove and start again.", - ], - [ - "gateway", - "tokens.json", - "remove and start again; then create each worker's join token again with `simlock --gateway token create --role worker`, set it as that worker's `gateway.token`, and drain workers again.", - ], - [ - "gateway", - "workers.json", - "remove and start again; then create each worker's join token again with `simlock --gateway token create --role worker`, set it as that worker's `gateway.token`, and drain workers again.", - ], - ] as const)( - "a %s started with %s in its folder refuses with the old-file message and its step", - async (role, file, step) => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-old-file-")); - temporaryDirectories.push(directory); - const oldFile = join(directory, file); - await writeFile(oldFile, "[]", "utf8"); - - await expect(start({ dataDirectory: directory, role })).rejects.toMatchObject({ - message: `Found ${oldFile}, which this Simlock no longer reads. To upgrade: ${step.replace("", oldFile)} See https://callstackincubator.github.io/simlock/docs/worker/running-the-daemon#upgrade-simlock.`, - }); - expect(existsSync(join(directory, "state.db"))).toBe(false); - }, - ); - - it("a token created, then the daemon restarted on the same state.db, verifies and lists with every field", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-token-restart-")); - temporaryDirectories.push(directory); - const admin = { - manageEventSubscription: () => undefined, - principal: "admin", - role: "admin", - via: "socket", - } as const; - let port = 0; - const launch = async (now: number) => { - const launched = await startOnFreePort((candidate) => - startDaemon({ - clock: new FakeClock(now), - configOverrides: { - ...STANDALONE_OFF, - http: { enabled: true, host: "127.0.0.1", port: candidate }, - }, - dataDirectory: directory, - deviceDataTables: fakeDeviceDataTables, - drivers: [], - hostLoadProbe: null, - version: "1.2.3", - } as StartDaemonOptions), - ); - port = launched.port; - return launched.started; - }; - type Created = { readonly secret: string; readonly token: Record }; - const first = await launch(1_000); - const labelled = (await first.dispatch( - "token.create", - { label: "ci", role: "agent" }, - admin, - )) as Created; - const bare = (await first.dispatch("token.create", { role: "operator" }, admin)) as Created; - await first.stop(); - - const second = await launch(2_000); - runningDaemons.push(second); - - const listed = (await second.dispatch("token.list", {}, admin)) as { - readonly tokens: readonly Record[]; - }; - expect(listed.tokens).toEqual([labelled.token, bare.token]); - expect("label" in (listed.tokens[1] ?? {})).toBe(false); - - const statusWith = async (secret: string) => - ( - await fetch(`http://127.0.0.1:${port}/v1/status`, { - headers: { authorization: `Bearer ${secret}` }, - }) - ).status; - expect(await statusWith(labelled.secret)).toBe(200); - expect(await statusWith(bare.secret)).toBe(200); - expect(await statusWith("slk_not-a-created-secret")).toBe(401); - }); - - it("a start that fails after state.db opened closes it before it throws", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-closes-state-")); - temporaryDirectories.push(directory); - const state = await openStateDatabase({ - dataDirectory: directory, - filesystem: new MemoryFilesystem(), - logger: new NoopLogger(), - oldFiles: [], - owners: [registryOwner, fakeDeviceDataTable], - sqlite: new NodeSqliteOpener(), - upgradeStep: "unused", - }); - const previous = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "1" }, - state, - }); - await previous.registerDevice({ - driverData: { fakeDeviceId: "fk-1" }, - driverDeviceId: "drv-1", - provisionDuration: 0, - spec: { model: "iPhone 16", osVersion: "26.5", physical: false, platform: "ios" }, - }); - state.database.exec("DELETE FROM fake_device_data"); - state.close(); - expect(existsSync(join(directory, "state.db-wal"))).toBe(false); - - await expect(start({ dataDirectory: directory })).rejects.toMatchObject({ - name: "RegistryLoadError", - }); - - expect(existsSync(join(directory, "state.db-wal"))).toBe(false); - }); - - it("a worker drops a deleted device record older than the configured devices.deletedRetentionMs at start", async () => { - const clock = new FakeClock(1_000); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-deleted-retention-")); - temporaryDirectories.push(directory); - const state = await openStateDatabase({ - dataDirectory: directory, - filesystem: new MemoryFilesystem(), - logger: new NoopLogger(), - oldFiles: [], - owners: [registryOwner, fakeDeviceDataTable], - sqlite: new NodeSqliteOpener(), - upgradeStep: "unused", - }); - const previous = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "1" }, - state, - }); - const device = await previous.registerDevice({ - driverData: { fakeDeviceId: "fk-1" }, - driverDeviceId: "drv-1", - provisionDuration: 0, - spec: { model: "iPhone 16", osVersion: "26.5", physical: false, platform: "ios" }, - }); - await previous.markDeviceMissing(device.id, "doctor"); - state.close(); - // why: 30 minutes is inside the default hour and outside the ten minutes configured below. - clock.advance(30 * 60_000); - - await start({ - clock, - configOverrides: { devices: { deletedRetentionMs: 600_000 } }, - dataDirectory: directory, - }); - - expect(await queryState(directory, "SELECT id FROM devices")).toEqual([]); - }); - - it("a worker refuses a re-sent chosen lease id inside the configured lease.endedIdRetentionMs and grants it after", async () => { - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const { daemon } = await start({ - clock, - configOverrides: { lease: { endedIdRetentionMs: 600_000 } }, - drivers: [driver], - }); - const agent = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent", - role: "agent", - } as const; - const chosen = { - leaseId: "retained", - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - requesterId: "agent-1", - } as const; - await daemon.dispatch("lease.request", chosen, agent); - await daemon.dispatch("lease.release", { leaseId: "retained" }, agent); - - // why: 5 minutes is inside the ten minutes configured and far inside the default day. - clock.advance(5 * 60_000); - await expect(daemon.dispatch("lease.request", chosen, agent)).rejects.toMatchObject({ - code: "LEASE_ENDED", - }); - // why: 30 minutes is outside the ten minutes configured and inside the default day. - clock.advance(30 * 60_000); - await expect(daemon.dispatch("lease.request", chosen, agent)).resolves.toMatchObject({ - lease: { id: "retained" }, - }); - }); - - it("a worker's state.db holds both platforms' source-data tables, a gateway's holds neither", async () => { - const worker = await mkdtemp(join(tmpdir(), "simlock-main-source-owners-")); - const gateway = await mkdtemp(join(tmpdir(), "simlock-main-source-owners-")); - temporaryDirectories.push(worker, gateway); - await start({ dataDirectory: worker }); - await start({ dataDirectory: gateway, role: "gateway" }); - const owners = async (directory: string) => - (await queryState(directory, "SELECT owner FROM schema_versions")) - .map((row) => row["owner"]) - .filter((owner) => owner === "ios-source-data" || owner === "android-source-data") - .sort(); - - expect(await owners(worker)).toEqual(["android-source-data", "ios-source-data"]); - expect(await owners(gateway)).toEqual([]); - }); - - it("a second startDaemon on a folder whose daemon runs leaves state.db's schema_versions and bytes unchanged", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-live-state-")); - temporaryDirectories.push(directory); - await start({ dataDirectory: directory }); - const versions = () => queryState(directory, "SELECT owner, version FROM schema_versions"); - const before = await versions(); - const bytesBefore = await readFile(join(directory, "state.db")); - // why: a newer binary brings a newer version of an owner the live daemon's state.db has; opening would migrate it in. - const newerTable = { - ...fakeDeviceDataTable, - migrations: [ - ...fakeDeviceDataTable.migrations, - "CREATE TABLE newer_fake_data (id TEXT) STRICT", - ], - version: 2, - }; - - await expect( - start({ - dataDirectory: directory, - deviceDataTables: new Map([["ios", newerTable]]), - }), - ).rejects.toMatchObject({ name: "DaemonAlreadyRunningError" }); - - expect(await versions()).toEqual(before); - expect(await readFile(join(directory, "state.db"))).toEqual(bytesBefore); - }); - - it("refuses to start as already running when connecting to its endpoint times out, as a Windows pipe whose every instance is busy does", async () => { - const memory = new MemoryIpcTransport(); - const ipc = { - connect: async () => { - throw new IpcError("connection-timeout", "timed out", undefined); - }, - listen: memory.listen.bind(memory), - }; - - await expect(start({ ipc })).rejects.toMatchObject({ name: "DaemonAlreadyRunningError" }); - }); - - it("a SIMLOCK_DRIVERS_MODULE without a deviceDataTables export fails start naming it", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-main-no-tables-")); - temporaryDirectories.push(directory); - const modulePath = join(directory, "drivers.mjs"); - await writeFile(modulePath, "export function createDrivers() { return []; }\n", "utf8"); - process.env.SIMLOCK_DRIVERS_MODULE = modulePath; - - await expect( - start({ dataDirectory: directory, deviceDataTables: undefined, drivers: undefined }), - ).rejects.toMatchObject({ - message: `SIMLOCK_DRIVERS_MODULE ${modulePath} does not export deviceDataTables, a Map from platform to its device-data table`, - }); - expect(existsSync(join(directory, "state.db"))).toBe(false); - }); - - it("resolveDeviceDataTables answers the option's map when given, else the drivers module's export, else the production map", async () => { - expect(await resolveDeviceDataTables({ deviceDataTables: fakeDeviceDataTables })).toBe( - fakeDeviceDataTables, - ); - - // why: the module exports tables of its own, so a module checked first would answer a different map. - process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( - "export function createDrivers() { return []; }", - ); - const optionMap = new Map(fakeDeviceDataTables); - expect(await resolveDeviceDataTables({ deviceDataTables: optionMap })).toBe(optionMap); - - delete process.env.SIMLOCK_DRIVERS_MODULE; - const production = await resolveDeviceDataTables({}); - expect(production).toEqual(productionDeviceDataTables()); - expect(production.get("ios")).toBe(iosDeviceDataTable); - expect(production.get("android")).toBe(androidDeviceDataTable); - - process.env.SIMLOCK_DRIVERS_MODULE = await writeModule( - "export function createDrivers() { return []; }", - ); - const fromModule = await resolveDeviceDataTables({}); - expect([...fromModule.keys()].sort()).toEqual(["android", "ios"]); - expect(fromModule.get("ios")?.name).toBe("fake-device-data"); - expect(fromModule).not.toBe(fakeDeviceDataTables); - }); -}); - -describe("startDaemon worker lock", () => { - const LOCK_PATH = "/tmp/simlock-test/worker.lock"; - const OTHER_FOLDER = "/home/other/.simlock"; - - async function filesystemWithLockHeldBy( - owner: { readonly pid: number; readonly folder: string }, - path = LOCK_PATH, - ): Promise { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp(path); - await filesystem.writeFileAtomic( - join(path, "owner.json"), - JSON.stringify({ ...owner, startedAt: "2026-11-02T08:00:00.000Z" }), - ); - return filesystem; - } - - async function installAndroidSdk(filesystem: MemoryFilesystem): Promise { - vi.stubEnv("ANDROID_HOME", "/android-sdk"); - for (const binary of [ - "platform-tools/adb", - "emulator/emulator", - "cmdline-tools/latest/bin/avdmanager", - "cmdline-tools/latest/bin/sdkmanager", - ]) { - await filesystem.mkdirp(dirname(join("/android-sdk", binary))); - await filesystem.writeFileAtomic(join("/android-sdk", binary), "binary"); - } - } - - // why: real drivers are the ones discovered, so neither the drivers option nor the substitute module may be set. - async function startWithRealDrivers( - filesystem: MemoryFilesystem, - livePids: readonly number[] | FakeProcessSupervisor, - processRunner: ScriptedProcessRunner, - dataDirectory?: string, - extra: Partial = {}, - ) { - vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); - return start({ - ...extra, - drivers: undefined, - filesystem, - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - npm: new FakeNpmCli(), - processRunner, - processSupervisor: Array.isArray(livePids) - ? new FakeProcessSupervisor(livePids) - : (livePids as FakeProcessSupervisor), - workerLockPath: LOCK_PATH, - ...(dataDirectory === undefined ? {} : { dataDirectory }), - }); - } - - afterEach(() => { - vi.unstubAllEnvs(); - }); - - it("a worker whose lock is held by a live pid in another folder fails to start with the refusal message, before any driver is discovered", async () => { - const filesystem = await filesystemWithLockHeldBy({ folder: OTHER_FOLDER, pid: 4242 }); - // why: without an SDK discovery does nothing a test can see, so a lock taken after it would pass; with one, discovery creates the Android device root. - await installAndroidSdk(filesystem); - const processRunner = new ScriptedProcessRunner([]); - const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); - temporaryDirectories.push(dataDirectory); - - await expect( - startWithRealDrivers(filesystem, [4242], processRunner, dataDirectory), - ).rejects.toThrow( - `another Simlock worker runs for this user (pid 4242, folder ${OTHER_FOLDER}, since 2026-11-02T08:00:00.000Z); stop it first: SIMLOCK_HOME=${OTHER_FOLDER} simlock daemon stop`, - ); - expect(processRunner.calls).toEqual([]); - expect(await filesystem.exists(join(dataDirectory, "devices", "android"))).toBe(false); - }); - - it.each([ - { name: "real drivers", substitute: "none" }, - { name: "drivers passed to startDaemon", substitute: "passed" }, - { name: "SIMLOCK_DRIVERS_MODULE set", substitute: "module" }, - ] as const)( - "takes the worker lock when drivers are discovered, writing its pid, folder and start time, and reads a held one never otherwise: $name", - async ({ substitute }) => { - const held = { folder: OTHER_FOLDER, pid: 4242 }; - const filesystem = await filesystemWithLockHeldBy( - substitute === "none" ? { folder: "/gone", pid: 4343 } : held, - ); - vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); - if (substitute === "module") { - vi.stubEnv( - "SIMLOCK_DRIVERS_MODULE", - await writeModule(`export function createDrivers() { return []; }`), - ); - } - - const { directory } = await start({ - ...(substitute === "passed" ? {} : { drivers: undefined }), - filesystem, - hostInfo: new FakeHostInfo({ arch: "arm64", os: "macOS", osVersion: "15.5" }), - npm: new FakeNpmCli(), - processRunner: new ScriptedProcessRunner([]), - processSupervisor: new FakeProcessSupervisor([4242]), - workerLockPath: LOCK_PATH, - }); - - const owner = JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json"))); - expect(owner).toEqual( - substitute === "none" - ? { folder: directory, pid: process.pid, startedAt: "1970-01-01T00:00:01.000Z" } - : { ...held, startedAt: "2026-11-02T08:00:00.000Z" }, - ); - }, - ); - - it("a worker that wins the socket claim while the lock names its folder and another live pid rewrites owner.json with its own pid", async () => { - const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); - temporaryDirectories.push(dataDirectory); - const filesystem = await filesystemWithLockHeldBy({ folder: dataDirectory, pid: 4242 }); - - await startWithRealDrivers(filesystem, [4242], new ScriptedProcessRunner([]), dataDirectory); - - expect(JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json")))).toEqual({ - folder: dataDirectory, - pid: process.pid, - startedAt: "1970-01-01T00:00:01.000Z", - }); - }); - - it("a worker that wins the socket claim while a live pid in another folder holds the lock fails its start with the refusal message", async () => { - const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); - temporaryDirectories.push(dataDirectory); - const filesystem = await filesystemWithLockHeldBy({ folder: dataDirectory, pid: 4242 }); - const ownerFile = join(LOCK_PATH, "owner.json"); - const reads: string[] = []; - // why: the other worker takes the lock over between this worker's take and its check after the claim, which a start cannot be made to wait for. - const readFile = filesystem.readFile.bind(filesystem); - filesystem.readFile = async (path: string) => { - if (path === ownerFile) reads.push(path); - return path === ownerFile && reads.length > 1 - ? JSON.stringify({ folder: OTHER_FOLDER, pid: 4242, startedAt: "t0" }) - : readFile(path); - }; - - await expect( - startWithRealDrivers(filesystem, [4242], new ScriptedProcessRunner([]), dataDirectory), - ).rejects.toThrow( - `another Simlock worker runs for this user (pid 4242, folder ${OTHER_FOLDER}, since t0)`, - ); - }); - - it.each([ - { name: "a stop removes the lock it wrote", takenOver: false, failedStart: false }, - { name: "a stop leaves a lock another worker took over", takenOver: true, failedStart: false }, - { - name: "a start that fails after taking the lock removes it", - takenOver: false, - failedStart: true, - }, - ])("$name", async ({ failedStart, takenOver }) => { - const filesystem = new MemoryFilesystem(); - const ownerFile = join(LOCK_PATH, "owner.json"); - const taken = { folder: OTHER_FOLDER, pid: 4242, startedAt: "t0" }; - // why: the Android driver's disposal ends the adb server this record names, which is the one thing a test can see a driver do at disposal; the lock must still stand then (ADR 0059 §3). - await installAndroidSdk(filesystem); - const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-main-lock-")); - temporaryDirectories.push(dataDirectory); - await filesystem.mkdirp(dataDirectory); - await filesystem.writeFileAtomic( - join(dataDirectory, "adb-server.json"), - JSON.stringify({ pid: 4343, port: 5038, startedAt: 1 }), - ); - const supervisor = new FakeProcessSupervisor([4242, 4343]); - supervisor.answerCommandLine(4343, "/android-sdk/platform-tools/adb -P 5038 nodaemon server"); - const lockAtDisposal: Promise[] = []; - const signal = supervisor.signal.bind(supervisor); - supervisor.signal = (pid, kill) => { - lockAtDisposal.push(filesystem.exists(LOCK_PATH)); - signal(pid, kill); - supervisor.markDead(pid); - }; - const starting = startWithRealDrivers( - filesystem, - supervisor, - new ScriptedProcessRunner([]), - dataDirectory, - // why: a module named like a core route is refused by the module host, which runs after the lock is taken. - { - tcpProbe: new FakeTcpProbe([5038]), - ...(failedStart ? { optionalModules: [{ name: "status" } as never] } : {}), - }, - ); - - if (failedStart) { - await expect(starting).rejects.toThrow("status"); - } else { - const { daemon } = await starting; - if (takenOver) await filesystem.writeFileAtomic(ownerFile, JSON.stringify(taken)); - await daemon.stop("test"); - } - - if (!failedStart) expect(await Promise.all(lockAtDisposal)).toEqual([true]); - expect(await filesystem.exists(LOCK_PATH)).toBe(takenOver); - if (takenOver) expect(JSON.parse(await filesystem.readFile(ownerFile))).toEqual(taken); - }); - - it("a gateway never takes the worker lock, even when it is held", async () => { - const filesystem = await filesystemWithLockHeldBy({ folder: OTHER_FOLDER, pid: 4242 }); - vi.stubEnv("SIMLOCK_DRIVERS_MODULE", undefined as unknown as string); - - const { daemon } = await startGateway({ - // why: a gateway given no drivers is the case where a worker would discover them and take the lock. - drivers: undefined as never, - filesystem, - processSupervisor: new FakeProcessSupervisor([4242]), - workerLockPath: LOCK_PATH, - }); - - expect(daemon.health).toBe("running"); - expect(JSON.parse(await filesystem.readFile(join(LOCK_PATH, "owner.json")))).toMatchObject({ - pid: 4242, - }); - }); -}); diff --git a/src/daemon/test-main-history.ts b/src/daemon/test-main-history.ts new file mode 100644 index 000000000..2d8a43af8 --- /dev/null +++ b/src/daemon/test-main-history.ts @@ -0,0 +1,23 @@ +import { expect, vi } from "vitest"; +import { type EventEnvelope } from "../bus/index.js"; +import { readHistoryEvents } from "../history/testing.js"; + +export async function historyEventsWhen( + directory: string, + done: (events: readonly EventEnvelope[]) => boolean, +): Promise { + let events: EventEnvelope[] = []; + await vi.waitFor( + async () => { + events = await readHistoryEvents(directory); + expect(done(events)).toBe(true); + }, + { timeout: 4_000 }, + ); + return events; +} + +export const hasEvent = + (name: string) => + (events: readonly EventEnvelope[]): boolean => + events.some((event) => event.event === name); diff --git a/src/daemon/test-main-state.ts b/src/daemon/test-main-state.ts new file mode 100644 index 000000000..30e49ceb4 --- /dev/null +++ b/src/daemon/test-main-state.ts @@ -0,0 +1,16 @@ +import { join } from "node:path"; +import { NodeSqliteOpener } from "../ports/index.js"; + +export async function queryState( + directory: string, + sql: string, +): Promise[]> { + const database = await new NodeSqliteOpener().open(join(directory, "state.db"), { + readOnly: true, + }); + try { + return database.prepare(sql).all(); + } finally { + database.close(); + } +} From 3a778891c5e12204b5e2fdc65c0a301a6322e0ef Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:31:41 +0100 Subject: [PATCH 45/83] test(daemon): split optional-modules.test.ts by topic --- src/daemon/optional-modules-start.test.ts | 353 +++++++++++ src/daemon/optional-modules-stop.test.ts | 262 ++++++++ src/daemon/optional-modules.test.ts | 733 +--------------------- src/daemon/test-optional-modules.ts | 161 +++++ 4 files changed, 791 insertions(+), 718 deletions(-) create mode 100644 src/daemon/optional-modules-start.test.ts create mode 100644 src/daemon/optional-modules-stop.test.ts create mode 100644 src/daemon/test-optional-modules.ts diff --git a/src/daemon/optional-modules-start.test.ts b/src/daemon/optional-modules-start.test.ts new file mode 100644 index 000000000..721bd0406 --- /dev/null +++ b/src/daemon/optional-modules-start.test.ts @@ -0,0 +1,353 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; +import type { OptionalModuleDefinition, WorkerHooks } from "../modules/index.js"; +import { fakeModule, type FakeModule } from "../modules/testing.js"; +import { + FakeClock, + IpcError, + JsonLinesLogger, + MemoryFilesystem, + MemoryLogSink, +} from "../ports/index.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; +import type { DaemonServer } from "./server.js"; +import { settleWithin } from "./test-relay-support.js"; +import { + admin, + daemons, + definitions, + directories, + leaseAndRelease, + prepare, + servers, + sleep, + start, +} from "./test-optional-modules.js"; + +afterEach(async () => { + await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test").catch(() => undefined))); + await Promise.all(servers.splice(0).map((server) => new Promise((done) => server.close(done)))); + await Promise.all( + directories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +// why: a value no default uses, so a timer with it is the lease-end wait ceiling a participant arms. +const END_WAIT_MS = 7_777; + +describe("optional modules: a start that cannot go on", () => { + const cases: readonly { + readonly title: string; + readonly good: () => FakeModule[]; + readonly bad: () => FakeModule[]; + readonly names: readonly string[]; + readonly overrides?: Record; + }[] = [ + { + bad: () => [fakeModule({ capabilityName: "beta", name: "alpha" })], + good: () => [fakeModule({ name: "alpha" })], + names: ["alpha", "beta"], + title: "a capability name that differs from the definition's", + }, + { + bad: () => [fakeModule({ name: "alpha" }), fakeModule({ name: "alpha" })], + good: () => [fakeModule({ name: "alpha" }), fakeModule({ name: "bravo" })], + names: ["alpha"], + title: "two definitions with one name", + }, + { + bad: () => [fakeModule({ name: "agent-device" }), fakeModule({ name: "agent-device" })], + good: () => [fakeModule({ name: "agent-device" }), fakeModule({ name: "agent-devices" })], + names: ["agent-device"], + title: "two definitions named agent-device", + }, + { + bad: () => [fakeModule({ name: "Bad_Name" })], + good: () => [fakeModule({ name: "good-name" })], + names: ["Bad_Name"], + title: "a name that fails the capability name check", + }, + { + bad: () => Array.from({ length: 17 }, (_, i) => fakeModule({ name: `mod-${String(i)}` })), + good: () => Array.from({ length: 16 }, (_, i) => fakeModule({ name: `mod-${String(i)}` })), + names: ["mod-16", "17"], + title: "17 fake modules (17 capabilities)", + }, + { + bad: () => [fakeModule({ loadError: new Error("no such file"), name: "alpha" })], + good: () => [fakeModule({ name: "alpha" })], + names: ["alpha", "no such file"], + title: "a loadWorker() that throws", + }, + ]; + + it.each(cases)("the start fails, naming the module, for $title", async ({ bad, good, names }) => { + const turnOn = (modules: FakeModule[]) => modules.map((m) => m.definition.name); + const goodModules = good(); + await start({ modules: definitions(...goodModules), on: turnOn(goodModules) }); + + const failure = await start({ modules: definitions(...bad()), on: turnOn(bad()) }).then( + () => undefined, + (error: unknown) => error as Error, + ); + + expect(failure).toBeInstanceOf(Error); + for (const name of names) expect(failure?.message).toContain(name); + }); + + it("a module that calls ctx.cancelPending in createWorker fails the start, naming the module", async () => { + const quiet = fakeModule({ name: "alpha" }); + await start({ modules: definitions(quiet), on: ["alpha"] }); + + const failure = await start({ + modules: definitions( + fakeModule({ name: "alpha", onCreate: (ctx) => void ctx.cancelPending("agent-1") }), + ), + on: ["alpha"], + }).then( + () => undefined, + (error: unknown) => error as Error, + ); + + expect(failure).toBeInstanceOf(Error); + expect(failure?.message).toContain("alpha"); + expect(failure?.message).toContain("leasing is not ready yet"); + }); + + it("when the second of two modules throws in createWorker, the first module's afterSocketClaimed never runs", async () => { + const claimed = vi.fn(); + const first = fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" }); + const failure = await start({ + modules: definitions( + first, + fakeModule({ + name: "bravo", + onCreate: () => { + throw new Error("bravo broke"); + }, + }), + ), + on: ["alpha", "bravo"], + }).then( + () => undefined, + (error: unknown) => error as Error, + ); + await sleep(50); + + expect(failure).toBeInstanceOf(Error); + expect(failure?.message).toContain("bravo"); + expect({ claimed: claimed.mock.calls.length, created: first.creates }).toEqual({ + claimed: 0, + created: 1, + }); + }); + + it("when the HTTP gateway fails to bind after the socket is claimed, the daemon stops and each started module's beforeDrain and beforeDriversDispose run", async () => { + const taken = createServer(); + servers.push(taken); + await new Promise((resolve) => taken.listen(0, "127.0.0.1", resolve)); + const { port } = taken.address() as { port: number }; + const notes: string[] = []; + const hooks: WorkerHooks = { + beforeDrain: () => { + notes.push("drain"); + return Promise.resolve(); + }, + beforeDriversDispose: () => { + notes.push("dispose"); + return Promise.resolve(); + }, + }; + + const outcome = await prepare({ + modules: definitions(fakeModule({ hooks, name: "alpha" })), + notes, + on: ["alpha"], + overrides: { http: { enabled: true, host: "127.0.0.1", port } }, + }).then((prepared) => + startDaemon(prepared.startOptions).then( + () => "started", + () => "rejected", + ), + ); + await vi.waitFor(() => expect(notes).toContain("dispose")); + + expect(outcome).toBe("rejected"); + expect(notes.filter((note) => note === "drain" || note === "dispose")).toEqual([ + "drain", + "dispose", + ]); + }); +}); + +describe("optional modules: afterSocketClaimed", () => { + it("runs once after the socket is claimed, and the start does not wait for it", async () => { + let socketPath = ""; + const seen: boolean[] = []; + const never = new Promise(() => undefined); + const prepared = await prepare({ + modules: definitions( + fakeModule({ + hooks: { + afterSocketClaimed: () => { + seen.push(existsSync(socketPath)); + return never; + }, + }, + name: "alpha", + }), + ), + on: ["alpha"], + }); + socketPath = join(prepared.directory, "daemon.sock"); + + const started = await settleWithin(startDaemon(prepared.startOptions), 5_000); + if (started.state === "fulfilled") daemons.push(started.value); + await sleep(50); + + expect(started.state).toBe("fulfilled"); + expect(seen).toEqual([true]); + }); + + it.each([ + // why: the loser refuses at the probe, before it opens state.db, so it builds no module at all + ["at the probe", undefined, 0], + // why: the loser's probe finds nothing answering, then its claim finds the address taken + [ + "at the claim", + { + connect: () => + Promise.reject(new IpcError("endpoint-not-found", "nothing answers", undefined)), + listen: () => Promise.reject(new IpcError("address-in-use", "address taken", undefined)), + } as StartDaemonOptions["ipc"], + 1, + ], + ] as const)( + "never runs in a daemon that lost the socket race %s", + async (_where, loserIpc, created) => { + const claimed = vi.fn(); + const filesystem = new MemoryFilesystem(); + const directory = await mkdtemp(join(tmpdir(), "simlock-modules-race-")); + directories.push(directory); + const options = ( + modules: readonly OptionalModuleDefinition[], + ipc?: StartDaemonOptions["ipc"], + ) => + ({ + clock: new FakeClock(1_000), + configOverrides: { capabilities: { alpha: { enabled: true } }, http: { enabled: false } }, + dataDirectory: directory, + drivers: [ + new FakeDriver({ + availableOsVersions: ["26.5"], + clock: new FakeClock(1_000), + platform: "ios", + }), + ], + filesystem, + ...(ipc === undefined ? {} : { ipc }), + logger: new JsonLinesLogger({ + clock: new FakeClock(1_000), + level: "debug", + sink: new MemoryLogSink(), + }), + optionalModules: modules, + deviceDataTables: fakeDeviceDataTables, + version: "1.2.3", + }) as StartDaemonOptions; + const first = await startDaemon(options([])); + daemons.push(first); + + const loser = fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" }); + const second = await startDaemon(options(definitions(loser), loserIpc)).then( + () => "started", + () => "rejected", + ); + await sleep(50); + + expect({ claimed: claimed.mock.calls.length, created: loser.creates, second }).toEqual({ + claimed: 0, + created, + second: "rejected", + }); + }, + ); + + it.each([ + [ + "throws", + () => { + throw new Error("hook broke"); + }, + ], + ["rejects", () => Promise.reject(new Error("hook broke"))], + ] as const)( + "an afterSocketClaimed that %s does not stop the daemon starting", + async (_how, hook) => { + const claimed = vi.fn(hook); + const { daemon } = await start({ + modules: definitions(fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" })), + on: ["alpha"], + }); + + await vi.waitFor(() => expect(claimed).toHaveBeenCalledTimes(1)); + expect(daemon.health).toBe("running"); + }, + ); +}); + +describe("optional modules: beforeDeviceReclaim", () => { + it("a release destroys the device when no turned-on module has beforeDeviceReclaim", async () => { + const notes: string[] = []; + const { daemon } = await start({ + modules: definitions(fakeModule({ name: "alpha" })), + notes, + on: ["alpha"], + overrides: { lease: { endWaitMs: END_WAIT_MS } }, + }); + + await leaseAndRelease(daemon); + + await vi.waitFor(() => expect(notes).toContain("driver destroy")); + }); + + it("beforeDeviceReclaim runs after a release commits and before the device's destroy starts", async () => { + const notes: string[] = []; + let live: DaemonServer | undefined; + const { daemon } = await start({ + modules: definitions( + fakeModule({ + hooks: { + beforeDeviceReclaim: async (lease) => { + const listed = (await live?.dispatch("lease.list", {}, admin)) as { + leases: { id: string }[]; + }; + notes.push( + listed.leases.some((entry) => entry.id === lease.leaseId) + ? "lease still listed" + : "lease gone", + ); + await sleep(30); + notes.push("hook done"); + }, + }, + name: "alpha", + }), + ), + notes, + on: ["alpha"], + overrides: { lease: { endWaitMs: END_WAIT_MS } }, + }); + live = daemon; + + await leaseAndRelease(daemon); + await vi.waitFor(() => expect(notes).toContain("driver destroy")); + + expect(notes).toEqual(["lease gone", "hook done", "driver destroy"]); + }); +}); diff --git a/src/daemon/optional-modules-stop.test.ts b/src/daemon/optional-modules-stop.test.ts new file mode 100644 index 000000000..106208b85 --- /dev/null +++ b/src/daemon/optional-modules-stop.test.ts @@ -0,0 +1,262 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { agentDeviceModuleWith, FakeAgentDevice } from "../agent-device/testing.js"; +import { readHistoryEvents } from "../history/testing.js"; +import { fakeModule } from "../modules/testing.js"; +import { + admin, + agent, + agentHeaders, + daemons, + definitions, + directories, + ios, + servers, + sleep, + start, +} from "./test-optional-modules.js"; + +afterEach(async () => { + await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test").catch(() => undefined))); + await Promise.all(servers.splice(0).map((server) => new Promise((done) => server.close(done)))); + await Promise.all( + directories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("optional modules: stop hooks", () => { + it("beforeDrain finishes before the HTTP gateway stops", async () => { + let origin: string | undefined; + const notes: string[] = []; + const rig = await start( + { + http: true, + modules: definitions( + fakeModule({ + hooks: { + beforeDrain: async () => { + await sleep(50); + const health = await fetch(`${origin}/v1/healthz`).then( + (response) => String(response.status), + () => "refused", + ); + notes.push(`gateway ${health}`); + }, + }, + name: "alpha", + }), + ), + on: ["alpha"], + }, + () => { + notes.length = 0; + }, + ); + origin = rig.origin; + + await rig.daemon.stop("test"); + + expect(notes).toEqual(["gateway 200"]); + }); + + it("by the time the stop ends, agent-device's waiting allocate is cancelled through leasing and another module's beforeDrain has run", async () => { + const drainRan: boolean[] = []; + let directory = ""; + const rig = await start( + { + http: true, + modules: [ + agentDeviceModuleWith({ upstream: new FakeAgentDevice() }), + ...definitions( + fakeModule({ + hooks: { + beforeDrain: async () => { + drainRan.push(true); + }, + }, + name: "alpha", + }), + ), + ], + on: ["agent-device", "alpha"], + phones: true, + overrides: { + capacity: { config: { maxRunning: 1 }, strategy: "fixed" }, + }, + }, + () => { + drainRan.length = 0; + }, + ); + directory = rig.directory; + const headers = await agentHeaders(rig.daemon); + await rig.daemon.dispatch("lease.request", ios, agent); + const allocate = fetch(`${rig.origin}/v1/agent-device/rpc`, { + body: JSON.stringify({ + id: 1, + jsonrpc: "2.0", + method: "agent_device.lease.allocate", + params: { + clientId: "client-1", + leaseId: "2".padStart(32, "0"), + leaseProvider: "proxy", + platform: "ios", + runId: "run-1", + ttlMs: 120_000, + }, + }), + headers: { ...headers, "content-type": "application/json" }, + method: "POST", + }).catch(() => undefined); + await vi.waitFor(async () => + expect( + ((await rig.daemon.dispatch("list.get", { kind: "requests" }, admin)) as unknown[]).length, + ).toBe(1), + ); + + await rig.daemon.stop("test"); + await allocate; + + const history = await readHistoryEvents(directory); + const cancelled = history.filter( + (event) => + event.event === "lease.rejected" && JSON.stringify(event.payload).includes('"cancelled"'), + ); + expect(drainRan).toEqual([true]); + expect(cancelled).toHaveLength(1); + }); + + it("beforeDriversDispose finishes before the driver's dispose starts", async () => { + const notes: string[] = []; + const { daemon } = await start({ + modules: definitions( + fakeModule({ + hooks: { + beforeDriversDispose: async () => { + await sleep(30); + notes.push("hook done"); + }, + }, + name: "alpha", + }), + ), + notes, + on: ["alpha"], + }); + + await daemon.stop("test"); + + expect(notes).toEqual(["hook done", "driver dispose"]); + }); + + it("with two modules whose beforeDrain hangs and settles at once, both start together; on a fake clock the step ends at 30 s, and the stop goes on", async () => { + const started: string[] = []; + const { clock, daemon } = await start({ + modules: definitions( + fakeModule({ + hooks: { + beforeDrain: () => { + started.push("hangs"); + return new Promise(() => undefined); + }, + }, + name: "hangs", + }), + fakeModule({ + hooks: { + beforeDrain: () => { + started.push("quick"); + return Promise.resolve(); + }, + }, + name: "quick", + }), + ), + on: ["hangs", "quick"], + }); + let stopped = false; + + const stopping = daemon.stop("test").then(() => { + stopped = true; + }); + await vi.waitFor(() => expect([...started].sort()).toEqual(["hangs", "quick"])); + clock.advance(29_999); + await sleep(30); + const stoppedEarly = stopped; + clock.advance(1); + await stopping; + + expect(stoppedEarly).toBe(false); + }); + + it("a beforeDrain that throws does not keep the HTTP gateway from stopping", async () => { + const { daemon, origin } = await start({ + http: true, + modules: definitions( + fakeModule({ + hooks: { + beforeDrain: () => Promise.reject(new Error("drain broke")), + }, + name: "alpha", + }), + ), + on: ["alpha"], + }); + expect((await fetch(`${origin}/v1/healthz`)).status).toBe(200); + + await daemon.stop("test"); + const afterStop = await fetch(`${origin}/v1/healthz`).then( + () => "answered", + () => "refused", + ); + + expect(afterStop).toBe("refused"); + }); + + it("a beforeDriversDispose that throws does not keep the drivers from being disposed", async () => { + const notes: string[] = []; + const { daemon } = await start({ + modules: definitions( + fakeModule({ + hooks: { beforeDriversDispose: () => Promise.reject(new Error("dispose broke")) }, + name: "alpha", + }), + ), + notes, + on: ["alpha"], + }); + + await daemon.stop("test"); + + expect(notes).toContain("driver dispose"); + }); + + it("a beforeDriversDispose that runs past 30 s on a fake clock does not keep the drivers from being disposed", async () => { + const notes: string[] = []; + const { clock, daemon } = await start({ + modules: definitions( + fakeModule({ + hooks: { + beforeDriversDispose: () => { + notes.push("hook started"); + return new Promise(() => undefined); + }, + }, + name: "alpha", + }), + ), + notes, + on: ["alpha"], + }); + + const stopping = daemon.stop("test"); + await vi.waitFor(() => expect(notes).toContain("hook started")); + clock.advance(29_999); + await sleep(30); + expect(notes).not.toContain("driver dispose"); + clock.advance(1); + await stopping; + + expect(notes).toContain("driver dispose"); + }); +}); diff --git a/src/daemon/optional-modules.test.ts b/src/daemon/optional-modules.test.ts index ca679aa34..b6728b262 100644 --- a/src/daemon/optional-modules.test.ts +++ b/src/daemon/optional-modules.test.ts @@ -1,50 +1,21 @@ -import { existsSync } from "node:fs"; -import { mkdtemp, rm } from "node:fs/promises"; -import { createServer, type Server } from "node:net"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { startOnFreePort } from "../test-support/loopback-port.js"; +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; import { agentDeviceModuleWith, FakeAgentDevice } from "../agent-device/testing.js"; -import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; -import type { OptionalModuleDefinition, WorkerHooks } from "../modules/index.js"; -import { readHistoryEvents } from "../history/testing.js"; -import { fakeModule, type FakeModule } from "../modules/testing.js"; -import { - FakeClock, - IpcError, - JsonLinesLogger, - MemoryFilesystem, - MemoryLogSink, -} from "../ports/index.js"; -import { startDaemon, type StartDaemonOptions } from "./main.js"; +import { fakeModule } from "../modules/testing.js"; import type { DaemonServer } from "./server.js"; import { openRelay, openUplink, relayInput, settleWithin } from "./test-relay-support.js"; - -const admin = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "test-operator", - role: "admin", -} as const; -const agent = { - manageEventSubscription: () => undefined, - via: "socket", - principal: "agent-1", - role: "agent", -} as const; -const ios = { - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", - requesterId: "agent-1", -} as const; - -const daemons: DaemonServer[] = []; -const directories: string[] = []; -const servers: Server[] = []; +import { + admin, + agent, + agentHeaders, + daemons, + definitions, + directories, + ios, + leaseAndRelease, + servers, + start, +} from "./test-optional-modules.js"; afterEach(async () => { await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test").catch(() => undefined))); @@ -54,121 +25,6 @@ afterEach(async () => { ); }); -class NotingDriver extends FakeDriver { - constructor( - options: ConstructorParameters[0], - readonly notes: string[], - ) { - super(options); - } - - // why: a released virtual device is destroyed, never reclaimed (ADR 0061). - override async destroy(...args: Parameters) { - this.notes.push("driver destroy"); - return super.destroy(...args); - } - - async dispose(): Promise { - this.notes.push("driver dispose"); - } -} - -interface Rig { - readonly daemon: DaemonServer; - readonly clock: FakeClock; - readonly sink: MemoryLogSink; - readonly directory: string; - readonly notes: string[]; - readonly origin: string | undefined; - readonly driver: NotingDriver; -} - -interface RigOptions { - readonly modules: readonly OptionalModuleDefinition[]; - readonly on?: readonly string[]; - readonly http?: boolean; - // why: agent-device's allocate names no model, so the driver must list a default phone for it. - readonly phones?: boolean; - readonly extra?: Partial; - readonly overrides?: Record; - readonly notes?: string[]; -} - -function definitions(...modules: FakeModule[]): OptionalModuleDefinition[] { - return modules.map((module) => module.definition as OptionalModuleDefinition); -} - -async function prepare(options: RigOptions, port?: number) { - const directory = await mkdtemp(join(tmpdir(), "simlock-modules-")); - directories.push(directory); - const clock = new FakeClock(1_000); - const sink = new MemoryLogSink(); - const notes = options.notes ?? []; - const driver = new NotingDriver( - { - availableOsVersions: ["26.5"], - clock, - platform: "ios", - ...(options.phones === true - ? { - defaultModels: { phone: ["iPhone 16"] }, - knownModels: ["iPhone 16"], - modelClasses: { "iPhone 16": "phone" as const }, - } - : {}), - }, - notes, - ); - const startOptions = { - clock, - configOverrides: { - capabilities: Object.fromEntries((options.on ?? []).map((name) => [name, { enabled: true }])), - // why: a standalone worker serves HTTP by default (ADR 0046); a rig that asks for no HTTP pins it off. - ...(port === undefined - ? options.extra?.role === "gateway" - ? {} - : { http: { enabled: false } } - : { http: { enabled: true, host: "127.0.0.1", port } }), - ...options.overrides, - }, - dataDirectory: directory, - drivers: [driver], - filesystem: new MemoryFilesystem(), - hostLoadProbe: null, - logger: new JsonLinesLogger({ clock, level: "debug", sink }), - optionalModules: options.modules, - socketPath: join(directory, "daemon.sock"), - deviceDataTables: fakeDeviceDataTables, - version: "1.2.3", - ...options.extra, - } as StartDaemonOptions; - return { - clock, - directory, - driver, - notes, - origin: port === undefined ? undefined : `http://127.0.0.1:${String(port)}`, - sink, - startOptions, - }; -} - -// why: `reset` empties what a test collects from its modules' hooks, so a start that lost its port and was stopped leaves nothing in it. -async function start(options: RigOptions, reset?: () => void): Promise { - const { started } = await startOnFreePort(async (port) => { - reset?.(); - const prepared = await prepare(options, options.http === true ? port : undefined); - return { ...prepared, daemon: await startDaemon(prepared.startOptions) }; - }); - daemons.push(started.daemon); - return started; -} - -async function agentHeaders(daemon: DaemonServer): Promise> { - const { secret } = await daemon.dispatch("token.create", { label: "a", role: "agent" }, admin); - return { authorization: `Bearer ${secret}` }; -} - async function capabilities(daemon: DaemonServer) { const status = (await daemon.dispatch("status.get", {}, agent)) as { readonly capabilities?: readonly { name: string; state: string; reason?: string }[]; @@ -176,16 +32,6 @@ async function capabilities(daemon: DaemonServer) { return status.capabilities ?? []; } -async function leaseAndRelease(daemon: DaemonServer): Promise { - const grant = (await daemon.dispatch("lease.request", ios, agent)) as { lease: { id: string } }; - await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); - return grant.lease.id; -} - -// why: a value no default uses, so a timer with it is the lease-end wait ceiling a participant arms. -const END_WAIT_MS = 7_777; -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - describe("optional modules: turning on", () => { it("a module with no capabilities section, or with enabled false, is reported off in status.get, and its loadWorker is never called", async () => { const bare = fakeModule({ name: "alpha" }); @@ -426,151 +272,6 @@ describe("optional modules: doctor findings", () => { }); }); -describe("optional modules: a start that cannot go on", () => { - const cases: readonly { - readonly title: string; - readonly good: () => FakeModule[]; - readonly bad: () => FakeModule[]; - readonly names: readonly string[]; - readonly overrides?: Record; - }[] = [ - { - bad: () => [fakeModule({ capabilityName: "beta", name: "alpha" })], - good: () => [fakeModule({ name: "alpha" })], - names: ["alpha", "beta"], - title: "a capability name that differs from the definition's", - }, - { - bad: () => [fakeModule({ name: "alpha" }), fakeModule({ name: "alpha" })], - good: () => [fakeModule({ name: "alpha" }), fakeModule({ name: "bravo" })], - names: ["alpha"], - title: "two definitions with one name", - }, - { - bad: () => [fakeModule({ name: "agent-device" }), fakeModule({ name: "agent-device" })], - good: () => [fakeModule({ name: "agent-device" }), fakeModule({ name: "agent-devices" })], - names: ["agent-device"], - title: "two definitions named agent-device", - }, - { - bad: () => [fakeModule({ name: "Bad_Name" })], - good: () => [fakeModule({ name: "good-name" })], - names: ["Bad_Name"], - title: "a name that fails the capability name check", - }, - { - bad: () => Array.from({ length: 17 }, (_, i) => fakeModule({ name: `mod-${String(i)}` })), - good: () => Array.from({ length: 16 }, (_, i) => fakeModule({ name: `mod-${String(i)}` })), - names: ["mod-16", "17"], - title: "17 fake modules (17 capabilities)", - }, - { - bad: () => [fakeModule({ loadError: new Error("no such file"), name: "alpha" })], - good: () => [fakeModule({ name: "alpha" })], - names: ["alpha", "no such file"], - title: "a loadWorker() that throws", - }, - ]; - - it.each(cases)("the start fails, naming the module, for $title", async ({ bad, good, names }) => { - const turnOn = (modules: FakeModule[]) => modules.map((m) => m.definition.name); - const goodModules = good(); - await start({ modules: definitions(...goodModules), on: turnOn(goodModules) }); - - const failure = await start({ modules: definitions(...bad()), on: turnOn(bad()) }).then( - () => undefined, - (error: unknown) => error as Error, - ); - - expect(failure).toBeInstanceOf(Error); - for (const name of names) expect(failure?.message).toContain(name); - }); - - it("a module that calls ctx.cancelPending in createWorker fails the start, naming the module", async () => { - const quiet = fakeModule({ name: "alpha" }); - await start({ modules: definitions(quiet), on: ["alpha"] }); - - const failure = await start({ - modules: definitions( - fakeModule({ name: "alpha", onCreate: (ctx) => void ctx.cancelPending("agent-1") }), - ), - on: ["alpha"], - }).then( - () => undefined, - (error: unknown) => error as Error, - ); - - expect(failure).toBeInstanceOf(Error); - expect(failure?.message).toContain("alpha"); - expect(failure?.message).toContain("leasing is not ready yet"); - }); - - it("when the second of two modules throws in createWorker, the first module's afterSocketClaimed never runs", async () => { - const claimed = vi.fn(); - const first = fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" }); - const failure = await start({ - modules: definitions( - first, - fakeModule({ - name: "bravo", - onCreate: () => { - throw new Error("bravo broke"); - }, - }), - ), - on: ["alpha", "bravo"], - }).then( - () => undefined, - (error: unknown) => error as Error, - ); - await sleep(50); - - expect(failure).toBeInstanceOf(Error); - expect(failure?.message).toContain("bravo"); - expect({ claimed: claimed.mock.calls.length, created: first.creates }).toEqual({ - claimed: 0, - created: 1, - }); - }); - - it("when the HTTP gateway fails to bind after the socket is claimed, the daemon stops and each started module's beforeDrain and beforeDriversDispose run", async () => { - const taken = createServer(); - servers.push(taken); - await new Promise((resolve) => taken.listen(0, "127.0.0.1", resolve)); - const { port } = taken.address() as { port: number }; - const notes: string[] = []; - const hooks: WorkerHooks = { - beforeDrain: () => { - notes.push("drain"); - return Promise.resolve(); - }, - beforeDriversDispose: () => { - notes.push("dispose"); - return Promise.resolve(); - }, - }; - - const outcome = await prepare({ - modules: definitions(fakeModule({ hooks, name: "alpha" })), - notes, - on: ["alpha"], - overrides: { http: { enabled: true, host: "127.0.0.1", port } }, - }).then((prepared) => - startDaemon(prepared.startOptions).then( - () => "started", - () => "rejected", - ), - ); - await vi.waitFor(() => expect(notes).toContain("dispose")); - - expect(outcome).toBe("rejected"); - expect(notes.filter((note) => note === "drain" || note === "dispose")).toEqual([ - "drain", - "dispose", - ]); - }); -}); - describe("optional modules: the context a module receives", () => { it("events.subscribe delivers a bus event and the module receives its config", async () => { const delivered: string[] = []; @@ -596,407 +297,3 @@ describe("optional modules: the context a module receives", () => { expect(seen.config).toEqual({ greeting: "hello" }); }); }); - -describe("optional modules: afterSocketClaimed", () => { - it("runs once after the socket is claimed, and the start does not wait for it", async () => { - let socketPath = ""; - const seen: boolean[] = []; - const never = new Promise(() => undefined); - const prepared = await prepare({ - modules: definitions( - fakeModule({ - hooks: { - afterSocketClaimed: () => { - seen.push(existsSync(socketPath)); - return never; - }, - }, - name: "alpha", - }), - ), - on: ["alpha"], - }); - socketPath = join(prepared.directory, "daemon.sock"); - - const started = await settleWithin(startDaemon(prepared.startOptions), 5_000); - if (started.state === "fulfilled") daemons.push(started.value); - await sleep(50); - - expect(started.state).toBe("fulfilled"); - expect(seen).toEqual([true]); - }); - - it.each([ - // why: the loser refuses at the probe, before it opens state.db, so it builds no module at all - ["at the probe", undefined, 0], - // why: the loser's probe finds nothing answering, then its claim finds the address taken - [ - "at the claim", - { - connect: () => - Promise.reject(new IpcError("endpoint-not-found", "nothing answers", undefined)), - listen: () => Promise.reject(new IpcError("address-in-use", "address taken", undefined)), - } as StartDaemonOptions["ipc"], - 1, - ], - ] as const)( - "never runs in a daemon that lost the socket race %s", - async (_where, loserIpc, created) => { - const claimed = vi.fn(); - const filesystem = new MemoryFilesystem(); - const directory = await mkdtemp(join(tmpdir(), "simlock-modules-race-")); - directories.push(directory); - const options = ( - modules: readonly OptionalModuleDefinition[], - ipc?: StartDaemonOptions["ipc"], - ) => - ({ - clock: new FakeClock(1_000), - configOverrides: { capabilities: { alpha: { enabled: true } }, http: { enabled: false } }, - dataDirectory: directory, - drivers: [ - new FakeDriver({ - availableOsVersions: ["26.5"], - clock: new FakeClock(1_000), - platform: "ios", - }), - ], - filesystem, - ...(ipc === undefined ? {} : { ipc }), - logger: new JsonLinesLogger({ - clock: new FakeClock(1_000), - level: "debug", - sink: new MemoryLogSink(), - }), - optionalModules: modules, - deviceDataTables: fakeDeviceDataTables, - version: "1.2.3", - }) as StartDaemonOptions; - const first = await startDaemon(options([])); - daemons.push(first); - - const loser = fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" }); - const second = await startDaemon(options(definitions(loser), loserIpc)).then( - () => "started", - () => "rejected", - ); - await sleep(50); - - expect({ claimed: claimed.mock.calls.length, created: loser.creates, second }).toEqual({ - claimed: 0, - created, - second: "rejected", - }); - }, - ); - - it.each([ - [ - "throws", - () => { - throw new Error("hook broke"); - }, - ], - ["rejects", () => Promise.reject(new Error("hook broke"))], - ] as const)( - "an afterSocketClaimed that %s does not stop the daemon starting", - async (_how, hook) => { - const claimed = vi.fn(hook); - const { daemon } = await start({ - modules: definitions(fakeModule({ hooks: { afterSocketClaimed: claimed }, name: "alpha" })), - on: ["alpha"], - }); - - await vi.waitFor(() => expect(claimed).toHaveBeenCalledTimes(1)); - expect(daemon.health).toBe("running"); - }, - ); -}); - -describe("optional modules: beforeDeviceReclaim", () => { - it("a release destroys the device when no turned-on module has beforeDeviceReclaim", async () => { - const notes: string[] = []; - const { daemon } = await start({ - modules: definitions(fakeModule({ name: "alpha" })), - notes, - on: ["alpha"], - overrides: { lease: { endWaitMs: END_WAIT_MS } }, - }); - - await leaseAndRelease(daemon); - - await vi.waitFor(() => expect(notes).toContain("driver destroy")); - }); - - it("beforeDeviceReclaim runs after a release commits and before the device's destroy starts", async () => { - const notes: string[] = []; - let live: DaemonServer | undefined; - const { daemon } = await start({ - modules: definitions( - fakeModule({ - hooks: { - beforeDeviceReclaim: async (lease) => { - const listed = (await live?.dispatch("lease.list", {}, admin)) as { - leases: { id: string }[]; - }; - notes.push( - listed.leases.some((entry) => entry.id === lease.leaseId) - ? "lease still listed" - : "lease gone", - ); - await sleep(30); - notes.push("hook done"); - }, - }, - name: "alpha", - }), - ), - notes, - on: ["alpha"], - overrides: { lease: { endWaitMs: END_WAIT_MS } }, - }); - live = daemon; - - await leaseAndRelease(daemon); - await vi.waitFor(() => expect(notes).toContain("driver destroy")); - - expect(notes).toEqual(["lease gone", "hook done", "driver destroy"]); - }); -}); - -describe("optional modules: stop hooks", () => { - it("beforeDrain finishes before the HTTP gateway stops", async () => { - let origin: string | undefined; - const notes: string[] = []; - const rig = await start( - { - http: true, - modules: definitions( - fakeModule({ - hooks: { - beforeDrain: async () => { - await sleep(50); - const health = await fetch(`${origin}/v1/healthz`).then( - (response) => String(response.status), - () => "refused", - ); - notes.push(`gateway ${health}`); - }, - }, - name: "alpha", - }), - ), - on: ["alpha"], - }, - () => { - notes.length = 0; - }, - ); - origin = rig.origin; - - await rig.daemon.stop("test"); - - expect(notes).toEqual(["gateway 200"]); - }); - - it("by the time the stop ends, agent-device's waiting allocate is cancelled through leasing and another module's beforeDrain has run", async () => { - const drainRan: boolean[] = []; - let directory = ""; - const rig = await start( - { - http: true, - modules: [ - agentDeviceModuleWith({ upstream: new FakeAgentDevice() }), - ...definitions( - fakeModule({ - hooks: { - beforeDrain: async () => { - drainRan.push(true); - }, - }, - name: "alpha", - }), - ), - ], - on: ["agent-device", "alpha"], - phones: true, - overrides: { - capacity: { config: { maxRunning: 1 }, strategy: "fixed" }, - }, - }, - () => { - drainRan.length = 0; - }, - ); - directory = rig.directory; - const headers = await agentHeaders(rig.daemon); - await rig.daemon.dispatch("lease.request", ios, agent); - const allocate = fetch(`${rig.origin}/v1/agent-device/rpc`, { - body: JSON.stringify({ - id: 1, - jsonrpc: "2.0", - method: "agent_device.lease.allocate", - params: { - clientId: "client-1", - leaseId: "2".padStart(32, "0"), - leaseProvider: "proxy", - platform: "ios", - runId: "run-1", - ttlMs: 120_000, - }, - }), - headers: { ...headers, "content-type": "application/json" }, - method: "POST", - }).catch(() => undefined); - await vi.waitFor(async () => - expect( - ((await rig.daemon.dispatch("list.get", { kind: "requests" }, admin)) as unknown[]).length, - ).toBe(1), - ); - - await rig.daemon.stop("test"); - await allocate; - - const history = await readHistoryEvents(directory); - const cancelled = history.filter( - (event) => - event.event === "lease.rejected" && JSON.stringify(event.payload).includes('"cancelled"'), - ); - expect(drainRan).toEqual([true]); - expect(cancelled).toHaveLength(1); - }); - - it("beforeDriversDispose finishes before the driver's dispose starts", async () => { - const notes: string[] = []; - const { daemon } = await start({ - modules: definitions( - fakeModule({ - hooks: { - beforeDriversDispose: async () => { - await sleep(30); - notes.push("hook done"); - }, - }, - name: "alpha", - }), - ), - notes, - on: ["alpha"], - }); - - await daemon.stop("test"); - - expect(notes).toEqual(["hook done", "driver dispose"]); - }); - - it("with two modules whose beforeDrain hangs and settles at once, both start together; on a fake clock the step ends at 30 s, and the stop goes on", async () => { - const started: string[] = []; - const { clock, daemon } = await start({ - modules: definitions( - fakeModule({ - hooks: { - beforeDrain: () => { - started.push("hangs"); - return new Promise(() => undefined); - }, - }, - name: "hangs", - }), - fakeModule({ - hooks: { - beforeDrain: () => { - started.push("quick"); - return Promise.resolve(); - }, - }, - name: "quick", - }), - ), - on: ["hangs", "quick"], - }); - let stopped = false; - - const stopping = daemon.stop("test").then(() => { - stopped = true; - }); - await vi.waitFor(() => expect([...started].sort()).toEqual(["hangs", "quick"])); - clock.advance(29_999); - await sleep(30); - const stoppedEarly = stopped; - clock.advance(1); - await stopping; - - expect(stoppedEarly).toBe(false); - }); - - it("a beforeDrain that throws does not keep the HTTP gateway from stopping", async () => { - const { daemon, origin } = await start({ - http: true, - modules: definitions( - fakeModule({ - hooks: { - beforeDrain: () => Promise.reject(new Error("drain broke")), - }, - name: "alpha", - }), - ), - on: ["alpha"], - }); - expect((await fetch(`${origin}/v1/healthz`)).status).toBe(200); - - await daemon.stop("test"); - const afterStop = await fetch(`${origin}/v1/healthz`).then( - () => "answered", - () => "refused", - ); - - expect(afterStop).toBe("refused"); - }); - - it("a beforeDriversDispose that throws does not keep the drivers from being disposed", async () => { - const notes: string[] = []; - const { daemon } = await start({ - modules: definitions( - fakeModule({ - hooks: { beforeDriversDispose: () => Promise.reject(new Error("dispose broke")) }, - name: "alpha", - }), - ), - notes, - on: ["alpha"], - }); - - await daemon.stop("test"); - - expect(notes).toContain("driver dispose"); - }); - - it("a beforeDriversDispose that runs past 30 s on a fake clock does not keep the drivers from being disposed", async () => { - const notes: string[] = []; - const { clock, daemon } = await start({ - modules: definitions( - fakeModule({ - hooks: { - beforeDriversDispose: () => { - notes.push("hook started"); - return new Promise(() => undefined); - }, - }, - name: "alpha", - }), - ), - notes, - on: ["alpha"], - }); - - const stopping = daemon.stop("test"); - await vi.waitFor(() => expect(notes).toContain("hook started")); - clock.advance(29_999); - await sleep(30); - expect(notes).not.toContain("driver dispose"); - clock.advance(1); - await stopping; - - expect(notes).toContain("driver dispose"); - }); -}); diff --git a/src/daemon/test-optional-modules.ts b/src/daemon/test-optional-modules.ts new file mode 100644 index 000000000..5d7d17f8a --- /dev/null +++ b/src/daemon/test-optional-modules.ts @@ -0,0 +1,161 @@ +import { mkdtemp } from "node:fs/promises"; +import { type Server } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { startOnFreePort } from "../test-support/loopback-port.js"; +import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; +import type { OptionalModuleDefinition } from "../modules/index.js"; +import { type FakeModule } from "../modules/testing.js"; +import { FakeClock, JsonLinesLogger, MemoryFilesystem, MemoryLogSink } from "../ports/index.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; +import type { DaemonServer } from "./server.js"; + +export const admin = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "test-operator", + role: "admin", +} as const; + +export const agent = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "agent-1", + role: "agent", +} as const; + +export const ios = { + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", + requesterId: "agent-1", +} as const; + +export const daemons: DaemonServer[] = []; + +export const directories: string[] = []; + +export const servers: Server[] = []; + +export class NotingDriver extends FakeDriver { + constructor( + options: ConstructorParameters[0], + readonly notes: string[], + ) { + super(options); + } + + // why: a released virtual device is destroyed, never reclaimed (ADR 0061). + override async destroy(...args: Parameters) { + this.notes.push("driver destroy"); + return super.destroy(...args); + } + + async dispose(): Promise { + this.notes.push("driver dispose"); + } +} + +export interface Rig { + readonly daemon: DaemonServer; + readonly clock: FakeClock; + readonly sink: MemoryLogSink; + readonly directory: string; + readonly notes: string[]; + readonly origin: string | undefined; + readonly driver: NotingDriver; +} + +export interface RigOptions { + readonly modules: readonly OptionalModuleDefinition[]; + readonly on?: readonly string[]; + readonly http?: boolean; + // why: agent-device's allocate names no model, so the driver must list a default phone for it. + readonly phones?: boolean; + readonly extra?: Partial; + readonly overrides?: Record; + readonly notes?: string[]; +} + +export function definitions(...modules: FakeModule[]): OptionalModuleDefinition[] { + return modules.map((module) => module.definition as OptionalModuleDefinition); +} + +export async function prepare(options: RigOptions, port?: number) { + const directory = await mkdtemp(join(tmpdir(), "simlock-modules-")); + directories.push(directory); + const clock = new FakeClock(1_000); + const sink = new MemoryLogSink(); + const notes = options.notes ?? []; + const driver = new NotingDriver( + { + availableOsVersions: ["26.5"], + clock, + platform: "ios", + ...(options.phones === true + ? { + defaultModels: { phone: ["iPhone 16"] }, + knownModels: ["iPhone 16"], + modelClasses: { "iPhone 16": "phone" as const }, + } + : {}), + }, + notes, + ); + const startOptions = { + clock, + configOverrides: { + capabilities: Object.fromEntries((options.on ?? []).map((name) => [name, { enabled: true }])), + // why: a standalone worker serves HTTP by default (ADR 0046); a rig that asks for no HTTP pins it off. + ...(port === undefined + ? options.extra?.role === "gateway" + ? {} + : { http: { enabled: false } } + : { http: { enabled: true, host: "127.0.0.1", port } }), + ...options.overrides, + }, + dataDirectory: directory, + drivers: [driver], + filesystem: new MemoryFilesystem(), + hostLoadProbe: null, + logger: new JsonLinesLogger({ clock, level: "debug", sink }), + optionalModules: options.modules, + socketPath: join(directory, "daemon.sock"), + deviceDataTables: fakeDeviceDataTables, + version: "1.2.3", + ...options.extra, + } as StartDaemonOptions; + return { + clock, + directory, + driver, + notes, + origin: port === undefined ? undefined : `http://127.0.0.1:${String(port)}`, + sink, + startOptions, + }; +} + +// why: `reset` empties what a test collects from its modules' hooks, so a start that lost its port and was stopped leaves nothing in it. +export async function start(options: RigOptions, reset?: () => void): Promise { + const { started } = await startOnFreePort(async (port) => { + reset?.(); + const prepared = await prepare(options, options.http === true ? port : undefined); + return { ...prepared, daemon: await startDaemon(prepared.startOptions) }; + }); + daemons.push(started.daemon); + return started; +} + +export async function agentHeaders(daemon: DaemonServer): Promise> { + const { secret } = await daemon.dispatch("token.create", { label: "a", role: "agent" }, admin); + return { authorization: `Bearer ${secret}` }; +} + +export async function leaseAndRelease(daemon: DaemonServer): Promise { + const grant = (await daemon.dispatch("lease.request", ios, agent)) as { lease: { id: string } }; + await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); + return grant.lease.id; +} + +export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); From 74f28b6a9993586863ddb10a13485ee02527ff92 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:26:18 +0100 Subject: [PATCH 46/83] test(agent-device): split edge.test.ts by topic --- .../edge-allocate-placement.test.ts | 560 ++++ src/agent-device/edge-allocate.test.ts | 449 ++++ .../edge-fake-agent-device.test.ts | 107 + src/agent-device/edge-heartbeat.test.ts | 762 ++++++ src/agent-device/edge-release.test.ts | 338 +++ src/agent-device/edge.test.ts | 2365 +---------------- src/agent-device/test-edge-front.ts | 33 + 7 files changed, 2360 insertions(+), 2254 deletions(-) create mode 100644 src/agent-device/edge-allocate-placement.test.ts create mode 100644 src/agent-device/edge-allocate.test.ts create mode 100644 src/agent-device/edge-fake-agent-device.test.ts create mode 100644 src/agent-device/edge-heartbeat.test.ts create mode 100644 src/agent-device/edge-release.test.ts create mode 100644 src/agent-device/test-edge-front.ts diff --git a/src/agent-device/edge-allocate-placement.test.ts b/src/agent-device/edge-allocate-placement.test.ts new file mode 100644 index 000000000..504a8c432 --- /dev/null +++ b/src/agent-device/edge-allocate-placement.test.ts @@ -0,0 +1,560 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { DispatchError } from "../daemon/dispatch.js"; +import { FakeClock, NodeSqliteOpener } from "../ports/index.js"; +import { FakeAgentDevice } from "./testing.js"; +import { + heartbeat, + phoneSerial, + setUpWithPhone, + advance, + release, + grantedDevice, + track, + listed, + tapDispatch, + idOf, +} from "./test-edge.js"; +import { + secrets, + notAvailable, + queueTimeoutMessage, + maxTtlMs, + adminSession, + type Rig, + androidDriver, + sleep, + setUp, + send, + rpcBody, + envelopeOf, + hex, + allocateParams, + allocate, + granted, + command, + leases, + requestedSpecs, + receivedAllocate, + allocatesReceived, + refusal, + runCommand, +} from "./test-support.js"; +import { waiting, phoneModels, expectEndedBySimlock } from "./test-edge-front.js"; + +function allocatePhone(rig: Rig, n: number, extra: Record = {}) { + return allocate(rig, n, { platform: "android", ...extra, physical: true }); +} + +describe("the agent-device front, through the whole worker daemon", () => { + it("allocate with `physical: true` leases an enrolled phone through the phone daemon, and the reply names the device with `physical: true` (`false` for a virtual one)", async () => { + const cases = [ + { + address: { serial: "R58Y61A60HN" }, + asked: { platform: "android" }, + extra: { platform: "android" }, + id: "R58Y61A60HN", + phone: { class: "phone", model: "Pixel 8", osVersion: "15" }, + platform: "android", + }, + { + address: { udid: "00008030-000270803C28802E" }, + asked: { model: "iPhone 15 Pro", platform: "ios" }, + extra: { device: "iPhone 15 Pro" }, + id: "00008030-000270803C28802E", + phone: { class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" }, + platform: "ios", + }, + { + address: { udid: "00008030-000270803C28802E" }, + asked: { osVersion: ">=18", platform: "ios" }, + extra: { osVersion: ">=18" }, + id: "00008030-000270803C28802E", + phone: { class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" }, + platform: "ios", + }, + ] as const; + for (const { address, asked, extra, id, phone, platform } of cases) { + const rig = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: ["35"], + clock, + knownModels: [phone.model], + physicalDevices: [{ ids: [id], ...phone }], + platform, + }), + ], + }); + await rig.daemon.dispatch("device.add", { id, platform }, adminSession); + rig.driver.setPhysicalPresence(id, true); + const answer = await envelopeOf( + await allocate(rig, 1, { platform, ...extra, physical: true }), + ); + expect(answer.result).toMatchObject({ + data: { + lease: { + device: { ...address, model: phone.model, osVersion: phone.osVersion, physical: true }, + }, + }, + ok: true, + }); + expect(answer.result).not.toHaveProperty("data.lease.device.deviceMode"); + expect(await requestedSpecs(rig, hex(1))).toMatchObject([{ ...asked, physical: true }]); + expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toMatchObject([ + { params: { leaseId: hex(1), ...address } }, + ]); + // why: agent-device's daemon refuses any spec key it receives, so `physical` stays with simlock. + expect(rig.phone.rpcRequests("agent_device.lease.allocate")[0]?.params).not.toHaveProperty( + "physical", + ); + expect(allocatesReceived(rig)).toEqual([]); + if (platform === "ios") { + // why: a physical iPhone has no `device.rotate`, so its `orientation auto` must reach agent-device, not be refused as an unknown name. + await command(rig, "orientation", 1, {}, { positionals: ["auto"] }); + expect(rig.phone.rpcRequests("agent_device.command")).toMatchObject([ + { params: { command: "orientation", positionals: ["auto"] } }, + ]); + } + } + }); + + it("allocate with `physical: true` that simlock refuses (an `imageTag`, a model no enrolled phone has, an OS range none fits) answers `refused by simlock: ` and leaves no lease", async () => { + const rig = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: ["35"], + clock, + knownModels: ["Pixel 8"], + physicalDevices: [ + { class: "phone", ids: ["R58Y61A60HN"], model: "Pixel 8", osVersion: "15" }, + ], + platform: "android", + }), + ], + }); + await rig.daemon.dispatch( + "device.add", + { id: "R58Y61A60HN", platform: "android" }, + adminSession, + ); + rig.driver.setPhysicalPresence("R58Y61A60HN", true); + const cases = [ + [{ imageTag: "default" }, "imageTag: `imageTag` has no meaning for a physical device"], + [{ device: "Pixel 9" }, "no enrolled physical device can match android model Pixel 9"], + [ + { osVersion: ">=16" }, + "no enrolled physical device can match android class phone os in the requested range", + ], + ] as const; + + for (const [index, [extra, message]] of cases.entries()) { + const answer = await envelopeOf( + await allocate(rig, index + 1, { platform: "android", ...extra, physical: true }), + ); + + expect(answer.error).toMatchObject(refusal(`refused by simlock: ${message}`)); + } + expect(await leases(rig)).toEqual([]); + expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toEqual([]); + }); + + it("allocate with `physical: true` while the phone daemon is not running answers `agent-device is not available on this simlock: phone daemon: ` and releases the lease; a virtual allocate is granted, a command on a live phone lease answers the same, and with the simulator daemon down a physical allocate answers the simulator daemon's reason", async () => { + const down = [ + ["starting", "phone daemon: starting", true], + ["adb not found", "phone daemon: adb not found", false], + ] as const; + for (const [reason, message, retriable] of down) { + const rig = await setUpWithPhone({ maxRunning: 4 }); + rig.phone.setUnavailable(reason, retriable); + + const answer = await envelopeOf(await allocatePhone(rig, 1)); + + expect(answer.error).toMatchObject(refusal(`${notAvailable}${message}`, "COMMAND_FAILED")); + expect(answer.error?.data).toMatchObject({ + details: { reason: "SIMLOCK_UNAVAILABLE" }, + retriable, + }); + await expect.poll(() => listed(rig)).toEqual([]); + await granted(rig, 2, { platform: "android" }); + expect(await listed(rig)).toEqual([hex(2)]); + } + + const live = await setUpWithPhone(); + await granted(live, 1, { platform: "android", physical: true }); + live.phone.setUnavailable("starting"); + const liveCommand = await envelopeOf(await command(live, "snapshot", 1)); + expect(liveCommand.error).toMatchObject( + refusal(`${notAvailable}phone daemon: starting`, "COMMAND_FAILED"), + ); + const beforeRules = await envelopeOf( + await runCommand(live, "snapshot", 1, { flags: { serial: "emulator-9999" } }), + ); + expect(beforeRules.error).toMatchObject( + refusal(`${notAvailable}phone daemon: starting`, "COMMAND_FAILED"), + ); + + const simulatorsDown = await setUpWithPhone(); + simulatorsDown.fake.setUnavailable("found 0.20.3"); + const simulatorsDownAnswer = await envelopeOf(await allocatePhone(simulatorsDown, 1)); + expect(simulatorsDownAnswer.error).toMatchObject( + refusal(`${notAvailable}found 0.20.3`, "COMMAND_FAILED"), + ); + expect(await listed(simulatorsDown)).toEqual([]); + }); + + it("a command, a heartbeat, a release and a `devices` naming a phone lease reach the phone daemon, those naming a simulator lease reach the simulator daemon, and `devices` with one lease of each answers both devices, each from its daemon", async () => { + const rig = await setUpWithPhone(); + await granted(rig, 1, { physical: true, platform: "android" }); + await granted(rig, 2, { platform: "android" }); + rig.phone.listDevices([{ id: phoneSerial }, { id: "emulator-5600" }]); + rig.fake.listDevices([{ id: idOf(rig, 2) }, { id: "emulator-5601" }]); + const reached = (fake: FakeAgentDevice, method: string) => fake.rpcRequests(method).length; + const calls = [ + ["agent_device.command", (n: number) => command(rig, "snapshot", n)], + ["agent_device.lease.heartbeat", (n: number) => heartbeat(rig, n)], + ["agent_device.lease.release", (n: number) => release(rig, n)], + ] as const; + + for (const [method, send] of calls) { + const [phoneBefore, simulatorBefore] = [ + reached(rig.phone, method), + reached(rig.fake, method), + ]; + await send(1); + expect([reached(rig.phone, method), reached(rig.fake, method)]).toEqual([ + phoneBefore + 1, + simulatorBefore, + ]); + await send(2); + expect([reached(rig.phone, method), reached(rig.fake, method)]).toEqual([ + phoneBefore + 1, + simulatorBefore + 1, + ]); + } + + await granted(rig, 3, { physical: true, platform: "android" }); + await granted(rig, 4, { platform: "android" }); + rig.fake.listDevices([{ id: idOf(rig, 4) }, { id: "emulator-5601" }]); + const answer = await envelopeOf(await runCommand(rig, "devices", undefined)); + const data = answer.result?.["data"] as { devices: { id: string }[] } | undefined; + expect(data?.devices.map((device) => device.id).sort()).toEqual( + [phoneSerial, idOf(rig, 4)].sort(), + ); + }); + + it("a client that sends the instance headers of the health it read, as the agent-device CLI does, is granted a phone and its later calls on that phone lease reach the phone daemon", async () => { + const rig = await setUpWithPhone(); + const health = (await (await send(rig, "/health")).json()) as { instanceId: string }; + const pinned = { + headers: { + "x-agent-device-instance": health.instanceId, + "x-agent-device-upstream-instance": "upstream-from-health", + }, + }; + + const allocated = await envelopeOf( + await allocate(rig, 1, { physical: true, platform: "android" }, pinned), + ); + const later = await envelopeOf(await command(rig, "snapshot", 1, pinned)); + const beat = await envelopeOf(await heartbeat(rig, 1, {}, pinned)); + + expect(allocated.result).toMatchObject({ ok: true }); + expect(later.error).toBeUndefined(); + expect(beat.error).toBeUndefined(); + expect(rig.phone.rpcRequests("agent_device.command")).toHaveLength(1); + expect(rig.phone.rpcRequests("agent_device.lease.heartbeat")).toHaveLength(1); + }); + + it("when simlock ends a phone lease (presence lost, an operator's release of all) the release reaches the phone daemon, not the simulator daemon, and the agent's next heartbeat answers ended by simlock", async () => { + const endings = [ + async (rig: Rig) => { + rig.driver.setPhysicalPresence(phoneSerial, false); + await advance(rig, 120_000); + }, + async (rig: Rig) => { + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + }, + ]; + for (const end of endings) { + const rig = await setUpWithPhone(); + await granted(rig, 1, { physical: true, platform: "android" }); + + await end(rig); + + await expect.poll(() => listed(rig)).toEqual([]); + await expect.poll(() => rig.phone.rpcRequests("agent_device.lease.release")).toHaveLength(1); + expect(rig.fake.rpcRequests("agent_device.lease.release")).toEqual([]); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + } + }); + + it("a virtual allocate whose address is also its driver id reaches the simulator daemon, not the phone daemon", async () => { + const rig = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + addressIsDeviceId: true, + availableOsVersions: ["35"], + clock, + defaultModels: { phone: ["Pixel 8"] }, + knownModels: ["Pixel 8"], + modelClasses: { "Pixel 8": "phone" }, + physicalDevices: [ + { class: "phone", ids: [phoneSerial], model: "Pixel 8", osVersion: "15" }, + ], + platform: "android", + }), + ], + }); + await rig.daemon.dispatch("device.add", { id: phoneSerial, platform: "android" }, adminSession); + rig.driver.setPhysicalPresence(phoneSerial, true); + + await granted(rig, 1, { platform: "android" }); + + expect(rig.fake.rpcRequests("agent_device.lease.allocate")).toMatchObject([ + { params: { leaseId: hex(1), serial: expect.stringMatching(/^fake-android-/) } }, + ]); + expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toEqual([]); + }); + + it("after a phone lease ends and its lease id has left the ended-id window, an allocate that reuses it for a virtual device reaches the simulator daemon", async () => { + const rig = await setUpWithPhone(); + await granted(rig, 1, { physical: true, platform: "android" }); + await release(rig, 1); + await expect.poll(() => listed(rig)).toEqual([]); + rig.clock.advance(25 * 60 * 60_000); + + await granted(rig, 1, { platform: "android" }); + + expect(allocatesReceived(rig, 1)).toHaveLength(1); + expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toHaveLength(1); + }); + + it("after the grant, the allocate the agent-device daemon receives names the granted device's address by `udid` on iOS and by `serial` on Android (the emulator serial, not the AVD name), and carries no `device`, `deviceKey` or `target`; the CLI gets the daemon's reply with the same lease ID", async () => { + const ios = await setUp(); + const iosAnswer = await envelopeOf( + await allocate(ios, 1, { device: "iPhone 17 Pro", deviceKey: "saved", target: "mobile" }), + ); + const iosDevice = await grantedDevice(ios); + const iosParams = receivedAllocate(ios, 1); + + expect(iosAnswer.result).toMatchObject({ data: { lease: { leaseId: hex(1) } }, ok: true }); + expect(iosParams["udid"]).toBe(iosDevice.address); + expect(iosParams).not.toHaveProperty("serial"); + for (const dropped of ["device", "deviceKey", "target"]) { + expect(iosParams).not.toHaveProperty(dropped); + } + + const android = await setUp({ drivers: (clock) => [androidDriver(clock)] }); + const androidAnswer = await envelopeOf( + await allocate(android, 2, { device: "Pixel 8", platform: "android" }), + ); + const androidDevice = await grantedDevice(android); + const androidParams = receivedAllocate(android, 2); + + expect(androidAnswer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); + expect(androidParams["serial"]).toBe(androidDevice.address); + expect(androidParams["serial"]).not.toBe(androidDevice.driverDeviceId); + expect(androidParams).not.toHaveProperty("udid"); + }); + + it("with every matching device leased, allocate waits, and is granted and answered once that device is released and reclaimed", async () => { + const rig = await setUp({ maxRunning: 1 }); + expect((await envelopeOf(await allocate(rig, 1))).result).toMatchObject({ ok: true }); + + const second = track(allocate(rig, 2, {}, { token: secrets.two })); + await expect + .poll(async () => (await waiting(rig)).map((request) => request.requesterId)) + .toEqual([hex(2)]); + expect(second.done()).toBe(false); + + await release(rig, 1); + const answer = await envelopeOf(await second.promise); + + expect(answer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2)]); + }); + + it("a CLI that hangs up while its allocate waits, and one that hangs up before simlock admitted the request, leave simlock's queue, and neither request is granted when a device frees", async () => { + const rig = await setUp({ maxRunning: 1 }); + await granted(rig, 1); + const hangUp = new AbortController(); + const second = track(allocate(rig, 2, {}, { signal: hangUp.signal, token: secrets.two })); + await sleep(50); + expect(second.done()).toBe(false); + await expect.poll(async () => (await waiting(rig)).length).toBe(1); + + hangUp.abort(); + await second.promise.catch(() => undefined); + await expect.poll(async () => (await waiting(rig)).length).toBe(0); + await release(rig, 1); + await sleep(50); + + expect(await leases(rig)).toEqual([]); + expect(allocatesReceived(rig, 2)).toEqual([]); + + const slow = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + ...phoneModels, + availableOsVersions: ["26.5"], + clock, + knownModels: ["iPhone 16"], + latencyMs: { listManaged: 30_000 }, + platform: "ios", + }), + ], + slowStart: true, + }); + const early = new AbortController(); + const parked = track( + fetch(`${slow.origin}/v1/agent-device/rpc`, { + body: rpcBody("agent_device.lease.allocate", allocateParams(3)), + headers: { authorization: `Bearer ${secrets.one}`, "content-type": "application/json" }, + method: "POST", + signal: early.signal, + }), + ); + await sleep(100); + expect(parked.done()).toBe(false); + early.abort(); + await parked.promise.catch(() => undefined); + await sleep(50); + slow.clock.advance(30_000); + const ready = { ...slow, daemon: await slow.started }; + await sleep(100); + + expect(await leases(ready)).toEqual([]); + expect(await requestedSpecs(ready, hex(3))).toEqual([]); + expect(allocatesReceived(ready, 3)).toEqual([]); + }); + + it("a CLI that hangs up after device work started for it gets no agent-device lease: the grant that lands later is released at once and never reaches the agent-device daemon; when that device work fails and the request goes back to the queue, it is never granted", async () => { + const slowDevice = (clock: FakeClock) => [ + new FakeDriver({ + ...phoneModels, + availableOsVersions: ["26.5"], + clock, + knownModels: ["iPhone 16"], + latencyMs: { makeReady: 10_000 }, + platform: "ios", + }), + ]; + const rig = await setUp({ drivers: slowDevice }); + const hangUp = new AbortController(); + const first = track(allocate(rig, 1, {}, { signal: hangUp.signal })); + await sleep(50); + expect(first.done()).toBe(false); + await expect + .poll(() => rig.driver.calls.some((call) => call.operation === "makeReady")) + .toBe(true); + + hangUp.abort(); + await first.promise.catch(() => undefined); + await advance(rig, 12_000); + await expect.poll(async () => (await leases(rig)).length).toBe(0); + + expect(allocatesReceived(rig, 1)).toEqual([]); + // why: the released device is deleted and the pool boots another on the fake clock, which the daemon's stop waits for. + await advance(rig, 12_000); + + const failing = await setUp({ drivers: slowDevice }); + failing.driver.failOn("makeReady", 1, new Error("boot failed")); + const abort = new AbortController(); + const second = track(allocate(failing, 2, {}, { signal: abort.signal })); + await sleep(50); + expect(second.done()).toBe(false); + await expect + .poll(() => failing.driver.calls.some((call) => call.operation === "makeReady")) + .toBe(true); + abort.abort(); + await second.promise.catch(() => undefined); + + await advance(failing, 15_000); + + await expect.poll(async () => (await waiting(failing)).length).toBe(0); + expect(await leases(failing)).toEqual([]); + expect(allocatesReceived(failing, 2)).toEqual([]); + }); + + it("`DELETE /v1/lease-requests/:id` for the front's waiting request answers the allocate `refused by simlock:` with simlock's message, and nothing is leased", async () => { + const rig = await setUp({ maxRunning: 1 }); + await granted(rig, 1, {}, { token: secrets.two }); + const second = track(allocate(rig, 2)); + await expect.poll(async () => (await waiting(rig)).length).toBe(1); + const [request] = await waiting(rig); + + const cancelled = await fetch(`${rig.origin}/v1/lease-requests/${request?.id}`, { + headers: { authorization: `Bearer ${secrets.one}` }, + method: "DELETE", + }); + const answer = await envelopeOf(await second.promise); + + expect(cancelled.status).toBe(204); + expect(answer.error?.message).toMatch(/^refused by simlock: .+/); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + expect(await waiting(rig)).toEqual([]); + }); + + it("stopping the daemon while an allocate waits ends with no simlock lease, and the lease request it waited on is cancelled, not open, in state.db", async () => { + const rig = await setUp({ maxRunning: 1 }); + await granted(rig, 1, {}, { token: secrets.two }); + const second = track(allocate(rig, 2)); + await expect.poll(async () => (await waiting(rig)).length).toBe(1); + + await rig.daemon.stop("test"); + await second.promise.catch(() => undefined); + + const database = await new NodeSqliteOpener().open(rig.stateDbPath, { readOnly: true }); + try { + const leaseIds = database.prepare("SELECT id FROM leases").all(); + const requests = database.prepare("SELECT lease_id, state FROM lease_requests").all(); + expect(second.done()).toBe(true); + expect(leaseIds).toEqual([{ id: hex(1) }]); + expect(requests.filter((row) => row.lease_id === hex(2))).toEqual([ + expect.objectContaining({ state: "cancelled" }), + ]); + expect(requests.some((row) => row.state === "open")).toBe(false); + } finally { + database.close(); + } + }); + + it("an allocate still waiting 330 s answers `timed out waiting for a device; nothing was leased`, and lease request failures carry their retry classification", async () => { + const rig = await setUp({ maxRunning: 1 }); + await granted(rig, 1, { ttlMs: maxTtlMs }, { token: secrets.two }); + const second = track(allocate(rig, 2)); + await expect.poll(async () => (await waiting(rig)).length).toBe(1); + + await advance(rig, 329_000); + expect(second.done()).toBe(false); + await advance(rig, 1_000); + expect(second.done()).toBe(true); + const answer = await envelopeOf(await second.promise); + + expect(answer.error).toMatchObject(refusal(queueTimeoutMessage, "COMMAND_FAILED")); + expect(answer.error?.data).toMatchObject({ + details: { reason: "NO_DEVICE_IN_TIME" }, + retriable: true, + }); + expect(await waiting(rig)).toEqual([]); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + + const tap = tapDispatch(rig); + for (const [code, reason] of [ + ["NO_CAPACITY", "SIMLOCK_UNAVAILABLE"], + ["DAEMON_STOPPING", "SIMLOCK_UNAVAILABLE"], + ["DAEMON_CONNECTION_LOST", "SIMLOCK_UNAVAILABLE"], + ["WORKER_UNREACHABLE", "WORKER_UNREACHABLE"], + ["INVALID_ARGS", "REFUSED_BY_SIMLOCK"], + ] as const) { + tap.rejectNext("lease.request", new DispatchError(code, `failure ${code}`)); + const response = await allocate(rig, 3); + expect(response.status).toBe(400); + expect((await envelopeOf(response)).error).toMatchObject({ + message: `refused by simlock: failure ${code}`, + data: { code: "INVALID_ARGS", details: { reason }, retriable: code !== "INVALID_ARGS" }, + }); + } + }); +}); diff --git a/src/agent-device/edge-allocate.test.ts b/src/agent-device/edge-allocate.test.ts new file mode 100644 index 000000000..209090aaf --- /dev/null +++ b/src/agent-device/edge-allocate.test.ts @@ -0,0 +1,449 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { idOf } from "./test-edge.js"; +import { + principals, + defaultTtlMs, + maxTtlMs, + scope, + type Rig, + taggedAndroidDriver, + setUp, + rpc, + envelopeOf, + hex, + allocateParams, + allocate, + granted, + leases, + requestedSpecs, + receivedAllocate, + allocatesReceived, + simlockMessage, + refusal, +} from "./test-support.js"; +import { waiting } from "./test-edge-front.js"; + +describe("the agent-device front, through the whole worker daemon", () => { + it("allocate `--platform ios` makes a simlock lease whose ID and requester are the CLI's lease ID, owned by the token's principal, and the lease request names no osVersion, mode, imageTag or physical device", async () => { + const rig = await setUp(); + + const answer = await allocate(rig, 1); + + expect((await envelopeOf(answer)).result).toMatchObject({ ok: true }); + expect(await leases(rig)).toMatchObject([ + { id: hex(1), ownerId: principals.one, requesterId: hex(1) }, + ]); + expect(await requestedSpecs(rig, hex(1))).toEqual([{ platform: "ios" }]); + + const bare = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: [], + clock, + knownModels: ["iPhone 16"], + platform: "ios", + }), + ], + }); + const refused = await allocate(bare, 2); + expect((await envelopeOf(refused)).error?.message).toContain("refused by simlock: "); + expect(bare.driver.calls.map((call) => call.operation)).not.toContain("installComponent"); + }); + + it('allocate with `osVersion` "26.0", ">=18 <27" or, on Android, "34" makes a lease request that carries it as typed', async () => { + const rig = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: ["18.4", "26.0", "27.0"], + clock, + defaultModels: { phone: ["iPhone 16"] }, + knownModels: ["iPhone 16"], + modelClasses: { "iPhone 16": "phone" }, + platform: "ios", + }), + new FakeDriver({ + availableOsVersions: ["34", "35"], + clock, + defaultModels: { phone: ["Pixel 8"] }, + knownModels: ["Pixel 8"], + modelClasses: { "Pixel 8": "phone" }, + platform: "android", + }), + ], + maxRunning: 8, + }); + + await granted(rig, 1, { osVersion: "26.0" }); + await granted(rig, 2, { osVersion: ">=18 <27" }); + await granted(rig, 3, { osVersion: "34", platform: "android" }); + + expect(await requestedSpecs(rig, hex(1))).toEqual([{ osVersion: "26.0", platform: "ios" }]); + expect(await requestedSpecs(rig, hex(2))).toEqual([{ osVersion: ">=18 <27", platform: "ios" }]); + expect(await requestedSpecs(rig, hex(3))).toEqual([{ osVersion: "34", platform: "android" }]); + }); + + it('allocate on Android with `imageTag` "google_apis_playstore", with and without `osVersion`, makes a lease request that carries it', async () => { + const rig = await setUp({ drivers: (clock) => [taggedAndroidDriver(clock)], maxRunning: 8 }); + const android = { platform: "android" }; + + await granted(rig, 1, { ...android, imageTag: "google_apis_playstore" }); + await granted(rig, 2, { ...android, imageTag: "google_apis_playstore", osVersion: "33" }); + + expect(await requestedSpecs(rig, hex(1))).toEqual([ + { imageTag: "google_apis_playstore", platform: "android" }, + ]); + expect(await requestedSpecs(rig, hex(2))).toEqual([ + { imageTag: "google_apis_playstore", osVersion: "33", platform: "android" }, + ]); + }); + + it("the allocate the agent-device daemon receives carries neither `osVersion` nor `imageTag`, whether the CLI sent one or both", async () => { + const rig = await setUp({ drivers: (clock) => [taggedAndroidDriver(clock)], maxRunning: 8 }); + const android = { platform: "android" }; + + await granted(rig, 1, { ...android, osVersion: "34" }); + await granted(rig, 2, { ...android, imageTag: "google_apis_playstore" }); + await granted(rig, 3, { ...android, imageTag: "google_apis_playstore", osVersion: "33" }); + + for (const n of [1, 2, 3]) { + const received = receivedAllocate(rig, n); + expect(received).not.toHaveProperty("osVersion"); + expect(received).not.toHaveProperty("imageTag"); + } + }); + + it("the relayed allocate reply carries `data.lease.device` as `{ model, osVersion, deviceMode, physical: false, udid or serial }`, `deviceMode` being the mode the device reports, on iOS and on Android without a tag, and with `imageTag` on Android with one, the daemon's `deviceKey` and `data.device` unchanged", async () => { + const rig: Rig = await setUp({ + drivers: (clock) => [ + new FakeDriver({ + availableOsVersions: ["18.4", "26.0", "27.0"], + clock, + defaultModels: { phone: ["iPhone 16"] }, + knownModels: ["iPhone 16"], + mode: "slim", + modelClasses: { "iPhone 16": "phone" }, + platform: "ios", + }), + taggedAndroidDriver(clock), + ], + maxRunning: 8, + }); + const cases = [ + [ + 1, + { osVersion: ">=18 <27" }, + { deviceMode: "slim", model: "iPhone 16", osVersion: "26.0", physical: false }, + ], + [2, {}, { deviceMode: "slim", model: "iPhone 16", osVersion: "27.0", physical: false }], + [ + 3, + { imageTag: "google_apis_playstore", platform: "android" }, + { + deviceMode: "full", + imageTag: "google_apis_playstore", + model: "Pixel 8", + osVersion: "34", + physical: false, + }, + ], + [ + 4, + { platform: "android" }, + { deviceMode: "full", model: "Pixel 8", osVersion: "35", physical: false }, + ], + ] as const; + for (const [n, extra, device] of cases) { + rig.fake.nameDeviceKeyInNextReply(`key-${n}`); + rig.fake.nameDeviceInNextReply({}); + const result = (await envelopeOf(await allocate(rig, n, extra, {}))).result as { + data: { device: unknown; lease: Record }; + }; + expect(result.data.lease["device"]).toEqual({ + ...device, + ["platform" in extra && extra.platform === "android" ? "serial" : "udid"]: idOf(rig, n), + }); + expect(idOf(rig, n)).not.toBe("undefined"); + expect(result.data.lease["deviceKey"]).toBe(`key-${n}`); + expect(result.data.device).toMatchObject({ id: expect.any(String) }); + } + }); + + it('allocate with `osVersion` "^18", with `imageTag` on iOS (with a model and without) or with an OS the driver lacks answers `refused by simlock:` with simlock\'s own message and leaves no lease; under `downloads.policy: always` the missing OS is installed for a named model and still refused without one', async () => { + const rig = await setUp(); + const refusals: readonly Record[] = [ + { osVersion: "^18" }, + { device: "iPhone 17 Pro", imageTag: "default" }, + { imageTag: "default" }, + { osVersion: "26.1" }, + ]; + + for (const [index, extra] of refusals.entries()) { + const expected = await simlockMessage(rig, { + leaseId: hex(90 + index), + platform: "ios", + ...(typeof extra["device"] === "string" ? { model: extra["device"] } : {}), + ...(extra["imageTag"] === undefined ? {} : { imageTag: extra["imageTag"] }), + ...(extra["osVersion"] === undefined ? {} : { osVersion: extra["osVersion"] }), + }); + const answer = await envelopeOf(await allocate(rig, index + 1, extra)); + expect(answer.error?.message).toBe(`refused by simlock: ${expected}`); + } + expect(await leases(rig)).toEqual([]); + expect(rig.driver.calls.map((call) => call.operation)).not.toContain("installComponent"); + + const always = await setUp({ downloads: "always" }); + const installs = () => + always.driver.calls.filter((call) => call.operation === "installComponent").length; + const bare = await envelopeOf(await allocate(always, 1, { osVersion: "26.1" })); + expect(bare.error?.message).toContain("refused by simlock: "); + expect(installs()).toBe(0); + const named = await envelopeOf( + await allocate(always, 2, { device: "iPhone 17 Pro", osVersion: "26.1" }), + ); + expect(named.result).toMatchObject({ ok: true }); + expect(installs()).toBe(1); + }); + + it("allocate with `physical: false` and `physical: null` are granted", async () => { + const rig = await setUp({ maxRunning: 8 }); + + await granted(rig, 2, { physical: false }); + await granted(rig, 3, { physical: null }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2), hex(3)]); + }); + + it('allocate with an unmapped `target`, "watch" or "", alone or beside `device: "iPhone 17 Pro"`, answers `refused: unknown --target ; use mobile, tv or desktop` and leaves no lease', async () => { + const rig = await setUp(); + const cases = [ + [{ target: "watch" }, "watch"], + [{ target: "" }, ""], + [{ device: "iPhone 17 Pro", target: "watch" }, "watch"], + ] as const; + + for (const [index, [extra, value]] of cases.entries()) { + const answer = await envelopeOf(await allocate(rig, index + 1, extra)); + + expect(answer.error).toMatchObject( + refusal(`refused: unknown --target ${value}; use mobile, tv or desktop`), + ); + } + expect(await leases(rig)).toEqual([]); + expect(allocatesReceived(rig)).toEqual([]); + }); + + it("the simlock lease's TTL is the CLI's `ttlMs`, 5 s when the CLI sent less, and `lease.defaultTtlMs` when it sent none; the allocate the agent-device daemon receives carries `ttlMs` equal to the ceiling in each case", async () => { + const rig = await setUp(); + + await granted(rig, 1, { ttlMs: 120_000 }); + await granted(rig, 2, { ttlMs: 1_000 }); + await granted(rig, 3, { ttlMs: undefined }); + + const byId = new Map((await leases(rig)).map((lease) => [lease.id, lease.ttlMs])); + expect(byId.get(hex(1))).toBe(120_000); + expect(byId.get(hex(2))).toBe(5_000); + expect(byId.get(hex(3))).toBe(defaultTtlMs); + expect(allocatesReceived(rig).map((request) => request.params?.["ttlMs"])).toEqual([ + rig.fake.leaseMaxTtlMs, + rig.fake.leaseMaxTtlMs, + rig.fake.leaseMaxTtlMs, + ]); + }); + + it('allocate `--target mobile`, `tv` and `desktop` ask simlock for class `phone`, `tv` and `desktop`; allocate `--device "iPhone 17 Pro"`, alone and with `--target tv`, asks for that model and no class', async () => { + const rig = await setUp({ maxRunning: 8 }); + + await granted(rig, 1, { target: "mobile" }); + await granted(rig, 2, { target: "tv" }); + await granted(rig, 3, { target: "desktop" }); + await granted(rig, 4, { device: "iPhone 17 Pro" }); + await granted(rig, 5, { device: "iPhone 17 Pro", target: "tv" }); + + expect(await requestedSpecs(rig, hex(1))).toEqual([{ class: "phone", platform: "ios" }]); + expect(await requestedSpecs(rig, hex(2))).toEqual([{ class: "tv", platform: "ios" }]); + expect(await requestedSpecs(rig, hex(3))).toEqual([{ class: "desktop", platform: "ios" }]); + expect(await requestedSpecs(rig, hex(4))).toEqual([ + { model: "iPhone 17 Pro", platform: "ios" }, + ]); + expect(await requestedSpecs(rig, hex(5))).toEqual([ + { model: "iPhone 17 Pro", platform: "ios" }, + ]); + }); + + it("an allocate whose `ttlMs` is above `lease.maxTtlMs` answers `refused by simlock:` with simlock's `BAD_REQUEST` message and leaves no simlock lease; one with `ttlMs: null` gets `lease.defaultTtlMs`", async () => { + const rig = await setUp(); + expect((await envelopeOf(await allocate(rig, 1, { ttlMs: null }))).result).toMatchObject({ + ok: true, + }); + expect((await leases(rig)).map((lease) => lease.ttlMs)).toEqual([defaultTtlMs]); + + const over = maxTtlMs + 1; + const expected = await simlockMessage(rig, { leaseId: hex(9), platform: "ios", ttlMs: over }); + const answer = await envelopeOf(await allocate(rig, 2, { ttlMs: over })); + + expect(answer.error?.message).toBe(`refused by simlock: ${expected}`); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + }); + + it('allocate `--device "My test phone"` answers `refused by simlock:` with simlock\'s message, and leaves no simlock lease', async () => { + const rig = await setUp(); + expect( + (await envelopeOf(await allocate(rig, 1, { device: "iPhone 17 Pro" }))).result, + ).toMatchObject({ ok: true }); + const expected = await simlockMessage(rig, { + leaseId: hex(9), + model: "My test phone", + platform: "ios", + }); + + const answer = await envelopeOf(await allocate(rig, 2, { device: "My test phone" })); + + expect(answer.error).toMatchObject(refusal(`refused by simlock: ${expected}`)); + expect(answer.error?.data).toMatchObject({ + details: { reason: "REFUSED_BY_SIMLOCK" }, + retriable: false, + }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + }); + + describe("each input refused before simlock", () => { + const tooOld = "refused: this agent-device is too old for simlock; upgrade it"; + const ttlRefusal = "refused: ttlMs must be a positive number of milliseconds"; + const plainOnly = "refused: simlock leases only plain devices"; + const lifecycle = "refused: send lease calls as lease RPCs"; + const cases: readonly { + readonly title: string; + readonly message: string; + readonly method?: string; + readonly params: (n: number) => Record; + }[] = [ + { + message: "refused: only iOS and Android", + params: (n) => allocateParams(n, { platform: "macos" }), + title: "--platform macos", + }, + { + message: "refused: simlock picks the device", + params: (n) => allocateParams(n, { udid: "ABC" }), + title: "--udid", + }, + { + message: "refused: simlock picks the device", + params: (n) => allocateParams(n, { serial: "emulator-5554" }), + title: "--serial", + }, + { + message: "refused: name --platform ios or android", + params: (n) => allocateParams(n, { platform: undefined }), + title: "no platform", + }, + { + message: tooOld, + params: (n) => allocateParams(n, { leaseId: undefined }), + title: "no lease ID", + }, + { + message: tooOld, + params: (n) => allocateParams(n, { leaseId: hex(n).toUpperCase().replace(/[0-9]/g, "A") }), + title: "a lease ID that is not 32 lowercase hex", + }, + { + message: ttlRefusal, + params: (n) => allocateParams(n, { ttlMs: "abc" }), + title: 'a ttlMs of "abc"', + }, + { + message: ttlRefusal, + params: (n) => allocateParams(n, { ttlMs: -1 }), + title: "a ttlMs of -1", + }, + { + message: ttlRefusal, + params: (n) => allocateParams(n, { ttlMs: 0 }), + title: "a ttlMs of 0", + }, + { + message: plainOnly, + params: (n) => allocateParams(n, { backend: "cloud" }), + title: "backend", + }, + { + message: plainOnly, + params: (n) => allocateParams(n, { backend: "android-instance" }), + title: "the Android backend on an iOS allocate", + }, + { + message: plainOnly, + params: (n) => allocateParams(n, { providerName: "x" }), + title: "a provider* key", + }, + { + message: plainOnly, + params: (n) => allocateParams(n, { leaseProvider: "aws" }), + title: 'leaseProvider: "aws"', + }, + ...["agent_device.command", "agent-device.command"].flatMap((method) => + ["lease_allocate", "lease_heartbeat", "lease_release"].map((name) => ({ + message: lifecycle, + method, + params: (n: number) => ({ command: name, meta: { leaseId: hex(n) } }), + title: `${method} carrying ${name}`, + })), + ), + ]; + + it.each(cases)( + "$title answers its message, simlock stores no lease request, and the agent-device daemon receives nothing", + async ({ message, method, params }) => { + const rig = await setUp(); + const accepted = await allocate(rig, 1); + expect((await envelopeOf(accepted)).result).toMatchObject({ ok: true }); + const heldBefore = (await leases(rig)).map((lease) => lease.id); + const receivedBefore = rig.fake.requests.length; + + const answer = await rpc(rig, method ?? "agent_device.lease.allocate", params(2)); + + expect((await envelopeOf(answer)).error).toMatchObject(refusal(message)); + expect(await requestedSpecs(rig, hex(2))).toEqual([]); + expect(await waiting(rig)).toEqual([]); + expect((await leases(rig)).map((lease) => lease.id)).toEqual(heldBefore); + expect(rig.fake.requests).toHaveLength(receivedBefore); + }, + ); + }); + + it("an allocate in the shape connect proxy sends, with backend ios-instance, is granted and keeps the backend toward the agent-device daemon", async () => { + const rig = await setUp(); + + const answer = await envelopeOf( + await allocate(rig, 1, { + backend: "ios-instance", + runId: "proxy-1", + session: "adc-1", + tenantId: "proxy", + }), + ); + + expect(answer.result).toMatchObject({ ok: true }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + expect(allocatesReceived(rig, 1).map((received) => received.params?.["backend"])).toEqual([ + "ios-instance", + ]); + }); + + it("allocate and release sent as `agent-device.lease.allocate` and `agent-device.lease.release` act exactly as the `agent_device.lease.*` spelling", async () => { + const rig = await setUp(); + + const allocated = await rpc(rig, "agent-device.lease.allocate", allocateParams(1)); + + expect((await envelopeOf(allocated)).result).toMatchObject({ ok: true }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + expect(rig.fake.leases.has(hex(1))).toBe(true); + + const released = await rpc(rig, "agent-device.lease.release", { ...scope, leaseId: hex(1) }); + + expect((await envelopeOf(released)).result).toMatchObject({ ok: true }); + expect(await leases(rig)).toEqual([]); + expect(rig.fake.leases.has(hex(1))).toBe(false); + }); +}); diff --git a/src/agent-device/edge-fake-agent-device.test.ts b/src/agent-device/edge-fake-agent-device.test.ts new file mode 100644 index 000000000..cb4c0c92b --- /dev/null +++ b/src/agent-device/edge-fake-agent-device.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { FakeAgentDevice } from "./testing.js"; +import { track } from "./test-edge.js"; +import { scope, type Envelope, sleep, hex } from "./test-support.js"; + +describe("FakeAgentDevice as agent-device behaves", () => { + const udid = "00000000-0000-0000-0000-000000000001"; + + function call( + fake: FakeAgentDevice, + method: string, + params: Record, + tenant: string | null = "tok_a", + ): Promise { + return fake.fetch( + new Request(`${fake.baseUrl}/rpc`, { + body: JSON.stringify({ id: 1, jsonrpc: "2.0", method, params }), + headers: { + authorization: `Bearer ${fake.token}`, + "content-type": "application/json", + ...(tenant === null ? {} : { "x-agent-device-tenant": tenant }), + }, + method: "POST", + }), + ); + } + + const allocation = (n: number, extra: Record = {}) => ({ + ...scope, + deviceKey: "key-1", + leaseId: hex(n), + platform: "ios", + udid, + ...extra, + }); + + it("refuses a release missing the tenant, the run ID, the lease provider, the client ID or the device key with LEASE_SCOPE_REQUIRED, and one with a different client ID with LEASE_SCOPE_MISMATCH, as agent-device does", async () => { + const fake = new FakeAgentDevice(); + await call(fake, "agent_device.lease.allocate", allocation(1)); + const full = { ...allocation(1), tenantId: "tok_a" }; + delete (full as Record)["platform"]; + delete (full as Record)["udid"]; + + const accepted = (await ( + await call(fake, "agent_device.lease.release", full) + ).json()) as Envelope; + expect(accepted.result).toMatchObject({ released: true }); + await call(fake, "agent_device.lease.allocate", allocation(1)); + + const reasonOf = async (params: Record, tenant: string | null = "tok_a") => { + const answer = (await ( + await call(fake, "agent_device.lease.release", params, tenant) + ).json()) as { + readonly error?: { readonly data?: { readonly details?: { readonly reason?: string } } }; + }; + return answer.error?.data?.details?.reason; + }; + const { tenantId: _tenantId, ...untenanted } = full; + expect(await reasonOf(untenanted, null)).toBe("LEASE_SCOPE_REQUIRED"); + for (const missing of ["runId", "leaseProvider", "clientId", "deviceKey"]) { + const partial = { ...full } as Record; + delete partial[missing]; + expect(await reasonOf(partial), missing).toBe("LEASE_SCOPE_REQUIRED"); + } + expect(await reasonOf({ ...full, clientId: "client-2" })).toBe("LEASE_SCOPE_MISMATCH"); + }); + + it("answers an allocate for a device a live lease holds with DEVICE_IN_USE, and a release of a lease with a held command only after the command is let go", async () => { + const fake = new FakeAgentDevice(); + const codeOf = async (response: Response) => + ((await response.json()) as Envelope).error?.data?.code; + expect( + await codeOf(await call(fake, "agent_device.lease.allocate", allocation(1))), + ).toBeUndefined(); + expect( + await codeOf( + await call(fake, "agent_device.lease.allocate", allocation(2, { udid: "other-device" })), + ), + ).toBeUndefined(); + + expect(await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3)))).toBe( + "DEVICE_IN_USE", + ); + + fake.holdCommand(); + const running = track( + call(fake, "agent_device.command", { command: "snapshot", meta: { leaseId: hex(1) } }), + ); + await sleep(20); + const full = { ...allocation(1), tenantId: "tok_a" }; + const releasing = track(call(fake, "agent_device.lease.release", full)); + await sleep(50); + + expect(releasing.done()).toBe(false); + expect(await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3)))).toBe( + "DEVICE_IN_USE", + ); + fake.releaseCommand(); + await running.promise; + expect(((await (await releasing.promise).json()) as Envelope).result).toMatchObject({ + released: true, + }); + expect( + await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3))), + ).toBeUndefined(); + }); +}); diff --git a/src/agent-device/edge-heartbeat.test.ts b/src/agent-device/edge-heartbeat.test.ts new file mode 100644 index 000000000..bf15c183d --- /dev/null +++ b/src/agent-device/edge-heartbeat.test.ts @@ -0,0 +1,762 @@ +import { describe, expect, it } from "vitest"; +import { + heartbeat, + advance, + release, + track, + listed, + type DispatchTap, + tapDispatch, + runningCommand, +} from "./test-edge.js"; +import { + secrets, + principals, + defaultTtlMs, + maxTtlMs, + scope, + adminSession, + sessionFor, + type Rig, + sleep, + setUp, + envelopeOf, + hex, + allocate, + granted, + command, + leases, + allocatesReceived, + refusal, +} from "./test-support.js"; +import { expectEndedBySimlock, expectLeaseGone } from "./test-edge-front.js"; + +function heartbeatsReceived(rig: Rig): number { + return rig.fake.rpcRequests("agent_device.lease.heartbeat").length; +} + +// why: a lease the front asked for in this process, whose CLI hung up and whose agent-device release failed, stays in simlock with agent-device still holding it, and the front map has no entry for it. +async function keptWhileAgentDeviceHolds(rig: Rig, n: number): Promise { + const releases = rig.fake.rpcRequests("agent_device.lease.release").length; + rig.fake.holdAllocate(); + rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "release failed" }); + const hangUp = new AbortController(); + const pending = track(allocate(rig, n, { ttlMs: 60_000 }, { signal: hangUp.signal })); + await expect.poll(() => allocatesReceived(rig, n).length).toBe(1); + hangUp.abort(); + await pending.promise.catch(() => undefined); + await sleep(50); + rig.fake.letAllocateGo(); + await expect + .poll(() => rig.fake.rpcRequests("agent_device.lease.release").length) + .toBe(releases + 1); +} + +async function deadlineOf(rig: Rig, n: number): Promise { + return (await leases(rig)).find((lease) => lease.id === hex(n))?.ttlDeadline; +} + +function frontCalls(tap: DispatchTap): readonly string[] { + return tap.operations.filter((name) => name === "lease.renew" || name === "lease.list"); +} + +async function renewMessage(rig: Rig, leaseId: string, ttlMs: number): Promise { + try { + await rig.daemon.dispatch("lease.renew", { leaseId, ttlMs }, sessionFor(principals.one)); + } catch (error) { + return (error as Error).message; + } + throw new Error("simlock accepted a renewal the test expected it to refuse"); +} + +describe("the agent-device front, through the whole worker daemon", () => { + it("a heartbeat reaches the agent-device daemon before simlock, carrying `ttlMs` equal to the ceiling, and once it answers, the simlock lease's deadline is now plus the CLI's `ttlMs`", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + const allocated = await deadlineOf(rig, 1); + let deadlineAtDaemon: number | undefined; + rig.fake.onHeartbeat = async () => { + deadlineAtDaemon = await deadlineOf(rig, 1); + }; + await advance(rig, 10_000); + + const answer = await envelopeOf( + await heartbeat(rig, 1, { ttlMs: 120_000 }, { token: secrets.one }), + ); + + expect(answer.result).toMatchObject({ ok: true }); + expect(deadlineAtDaemon).toBe(allocated); + expect(rig.fake.rpcRequests("agent_device.lease.heartbeat")[0]?.params).toMatchObject({ + ...scope, + leaseId: hex(1), + ttlMs: rig.fake.leaseMaxTtlMs, + }); + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 120_000); + }); + + it.each([ + { cap: undefined, expected: 300_000, row: "300000 with no cap", sent: 300_000 }, + { cap: 10_000, expected: 10_000, row: "300000 with cap 10000", sent: 300_000 }, + { cap: 600_000, expected: 300_000, row: "300000 with cap 600000", sent: 300_000 }, + { cap: 10_000, expected: 10_000, row: "none with cap 10000", sent: undefined }, + { cap: undefined, expected: defaultTtlMs, row: "none with no cap", sent: undefined }, + { cap: undefined, expected: 5_000, row: "1 with no cap", sent: 1 }, + ])( + "a CLI `ttlMs` of $row gives the simlock lease, its heartbeat renewal and its renewal interval a TTL of $expected ms", + async ({ cap, expected, sent }) => { + const rig = await setUp(cap === undefined ? {} : { leaseTtlCapMs: cap }); + const ttl = expected as number; + await granted(rig, 1, { ttlMs: sent }); + expect((await leases(rig))[0]?.ttlMs).toBe(ttl); + await advance(rig, 1_000); + + const answer = await envelopeOf( + await heartbeat(rig, 1, sent === undefined ? {} : { ttlMs: sent }), + ); + + expect(answer.result).toMatchObject({ ok: true }); + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + ttl); + const settled = await deadlineOf(rig, 1); + await runningCommand(rig, 1); + await advance(rig, Math.floor(ttl / 3) - 700, 500); + expect(await deadlineOf(rig, 1)).toBe(settled); + await advance(rig, 1_400, 500); + const renewed = (await deadlineOf(rig, 1)) as number; + expect(renewed).toBeGreaterThan(settled as number); + expect(renewed - (settled as number)).toBeLessThanOrEqual(ttl / 3 + 700); + }, + 60_000, + ); + + it("a heartbeat with a longer `ttlMs` than the allocate's moves the deadline to now plus that TTL, and a following heartbeat with no `ttlMs` renews with it too", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + await advance(rig, 5_000); + + await heartbeat(rig, 1, { ttlMs: 300_000 }); + + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 300_000); + await advance(rig, 20_000); + + await heartbeat(rig, 1); + + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 300_000); + expect((await leases(rig))[0]?.ttlMs).toBe(300_000); + }); + + it("after the agent's own `simlock lease renew` with a longer TTL, a heartbeat and a running command's renewals leave that later deadline in place", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + await rig.daemon.dispatch( + "lease.renew", + { leaseId: hex(1), ttlMs: 900_000 }, + sessionFor(principals.one), + ); + const later = await deadlineOf(rig, 1); + expect(later).toBe(rig.clock.now() + 900_000); + await advance(rig, 5_000); + + await heartbeat(rig, 1, { ttlMs: 120_000 }); + + expect(await deadlineOf(rig, 1)).toBe(later); + const running = await runningCommand(rig, 1); + await advance(rig, 130_000, 5_000); + expect(await deadlineOf(rig, 1)).toBe(later); + rig.fake.releaseCommand(); + await running.promise; + await expect.poll(() => running.done()).toBe(true); + expect(await deadlineOf(rig, 1)).toBe(later); + }); + + it("a heartbeat the agent-device daemon cannot answer returns that error and leaves the simlock lease's deadline where it was", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + const before = await deadlineOf(rig, 1); + await advance(rig, 10_000); + rig.fake.failNextHeartbeat({ code: "COMMAND_FAILED", message: "daemon is busy" }); + + const failed = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); + + expect(failed.error).toMatchObject({ + data: { code: "COMMAND_FAILED" }, + message: "daemon is busy", + }); + expect(await deadlineOf(rig, 1)).toBe(before); + + rig.fake.setUnreachable(true); + const unreachable = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); + + expect(unreachable.error).toBeDefined(); + expect(unreachable.result).toBeUndefined(); + expect(await deadlineOf(rig, 1)).toBe(before); + + rig.fake.setUnreachable(false); + await heartbeat(rig, 1, { ttlMs: 60_000 }); + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 60_000); + }); + + it("with no cap set, a heartbeat whose `ttlMs` is above `lease.maxTtlMs` answers `COMMAND_FAILED` with simlock's `BAD_REQUEST` message and leaves the deadline where it was, and a following heartbeat with a valid `ttlMs` renews the lease", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + const before = await deadlineOf(rig, 1); + await advance(rig, 10_000); + const message = await renewMessage(rig, hex(1), maxTtlMs + 1); + + const refused = await envelopeOf(await heartbeat(rig, 1, { ttlMs: maxTtlMs + 1 })); + + expect(refused.error).toMatchObject({ data: { code: "COMMAND_FAILED" }, message }); + expect(await deadlineOf(rig, 1)).toBe(before); + + const renewed = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); + + expect(renewed.result).toMatchObject({ ok: true }); + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 120_000); + expect((await leases(rig))[0]?.ttlMs).toBe(120_000); + }); + + it("a heartbeat for a lease the agent released through the front answers `LEASE_NOT_FOUND`, and a following allocate gets a new lease", async () => { + const rig = await setUp(); + await granted(rig, 1); + await release(rig, 1); + + const answer = await envelopeOf(await heartbeat(rig, 1)); + + expectLeaseGone(answer, "LEASE_NOT_FOUND"); + expect(await listed(rig)).toEqual([]); + + await granted(rig, 2); + + expect(await listed(rig)).toEqual([hex(2)]); + }); + + it("one TTL after the agent's release, a heartbeat for that lease answers `LEASE_ENDED_BY_HOST`", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + await release(rig, 1); + expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); + await advance(rig, 61_000, 5_000); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + }); + + it.each([ + [ + "`simlock release --all`", + 5_000, + async (rig: Rig) => { + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + }, + ], + [ + "expiry", + 5_000, + async (rig: Rig) => { + await advance(rig, 6_000); + }, + ], + [ + "the device is lost", + 600_000, + async (rig: Rig) => { + rig.driver.setManagedReality({ devices: [], processes: [] }); + await advance(rig, 120_000, 10_000); + }, + ], + ])( + "after %s, a heartbeat for the lease answers `LEASE_ENDED_BY_HOST` with the message and reason, and makes no new simlock lease", + async (_cause, ttl, end) => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: ttl }); + await end(rig); + await expect.poll(() => listed(rig)).toEqual([]); + + const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 5_000 })); + + expectEndedBySimlock(answer); + expect(await listed(rig)).toEqual([]); + }, + ); + + it.each([ + [ + "`LEASE_NOT_FOUND` with the simlock lease already released", + async (rig: Rig) => { + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + expect(await listed(rig)).toEqual([]); + }, + ], + [ + "`LEASE_EXPIRED`", + async (rig: Rig) => { + rig.fake.loseLease(hex(1), "LEASE_EXPIRED"); + expect(await listed(rig)).toEqual([hex(1)]); + }, + ], + [ + "`LEASE_REVOKED`", + async (rig: Rig) => { + rig.fake.loseLease(hex(1), "LEASE_REVOKED"); + expect(await listed(rig)).toEqual([hex(1)]); + }, + ], + ] as const)( + "a heartbeat the agent-device daemon answers with %s, for a lease that really ended, releases the simlock lease and answers `LEASE_ENDED_BY_HOST`", + async (_case, end) => { + const rig: Rig = await setUp(); + await granted(rig, 1); + await end(rig); + + const answer = await envelopeOf(await heartbeat(rig, 1)); + + expectEndedBySimlock(answer); + expect(await listed(rig)).toEqual([]); + }, + ); + + it("a heartbeat naming a lease ID the front map lacks, which simlock kept while agent-device still holds it: it is released with the heartbeat's scope first, then the simlock lease; when agent-device is unreachable, neither lease is touched; each answers as in the table", async () => { + const rig: Rig = await setUp(); + await keptWhileAgentDeviceHolds(rig, 2); + await keptWhileAgentDeviceHolds(rig, 3); + expect(await listed(rig)).toEqual([hex(2), hex(3)]); + + let heldAtDaemonRelease: readonly string[] = []; + rig.fake.onRelease = async () => { + heldAtDaemonRelease = await listed(rig); + }; + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 2))); + expect(heldAtDaemonRelease).toEqual([hex(2), hex(3)]); + expect(rig.fake.rpcRequests("agent_device.lease.release")[0]?.params).toMatchObject(scope); + expect(rig.fake.leases.has(hex(2))).toBe(false); + expect(await listed(rig)).toEqual([hex(3)]); + + const before = await deadlineOf(rig, 3); + await advance(rig, 10_000); + rig.fake.setUnreachable(true); + const unreachable = await envelopeOf(await heartbeat(rig, 3)); + rig.fake.setUnreachable(false); + + expect(unreachable.error).toBeDefined(); + expect(unreachable.error?.data?.code).not.toBe("UNAUTHORIZED"); + expect(await listed(rig)).toEqual([hex(3)]); + expect(await deadlineOf(rig, 3)).toBe(before); + expect(rig.fake.leases.has(hex(3))).toBe(true); + }); + + it("a heartbeat from a second token naming the first token's lease ID answers `LEASE_ENDED_BY_HOST`, and the first token's lease is neither released nor renewed", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + const before = await deadlineOf(rig, 1); + await advance(rig, 10_000); + + const answer = await envelopeOf( + await heartbeat(rig, 1, { ttlMs: 60_000 }, { token: secrets.two }), + ); + + expectEndedBySimlock(answer); + expect(await listed(rig)).toEqual([hex(1)]); + expect(await deadlineOf(rig, 1)).toBe(before); + expect(rig.fake.leases.has(hex(1))).toBe(true); + + const own = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); + + expect(own.result).toMatchObject({ ok: true }); + expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 60_000); + }); + + it("a heartbeat naming a lease ID the front map lacks, when the agent-device daemon fails to release its lease, answers that error and leaves the simlock lease in place", async () => { + const rig: Rig = await setUp(); + await keptWhileAgentDeviceHolds(rig, 1); + rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon cannot release" }); + + const failed = await envelopeOf(await heartbeat(rig, 1)); + + expect(failed.error).toMatchObject({ + data: { code: "COMMAND_FAILED" }, + message: "daemon cannot release", + }); + expect(await listed(rig)).toEqual([hex(1)]); + }); + + it("a heartbeat for a lease the agent-device daemon lost, when simlock cannot release its lease, answers `COMMAND_FAILED` with simlock's message and marks nothing, and the next heartbeat releases it and answers `LEASE_ENDED_BY_HOST`", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1); + rig.fake.loseLease(hex(1), "LEASE_EXPIRED"); + const tap = tapDispatch(rig); + tap.rejectNext("lease.release"); + + const failed = await envelopeOf(await heartbeat(rig, 1)); + + expect(failed.error).toMatchObject({ + data: { code: "COMMAND_FAILED" }, + message: "simlock could not do that", + }); + expect(await listed(rig)).toEqual([hex(1)]); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + expect(await listed(rig)).toEqual([]); + }); + + it.each([0, -5, "soon", true])( + "a heartbeat with a `ttlMs` of %j is refused with `ttlMs must be a positive number of milliseconds`, and neither daemon is reached", + async (ttlMs) => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + const before = await deadlineOf(rig, 1); + const received = heartbeatsReceived(rig); + await advance(rig, 10_000); + + const refused = await envelopeOf(await heartbeat(rig, 1, { ttlMs })); + + expect(refused.error).toMatchObject( + refusal("refused: ttlMs must be a positive number of milliseconds"), + ); + expect(heartbeatsReceived(rig)).toBe(received); + expect(await deadlineOf(rig, 1)).toBe(before); + }, + ); + + it("after a heartbeat answered `LEASE_ENDED_BY_HOST` because simlock ended the lease, the next heartbeat answers it again and the agent-device daemon receives nothing for it", async () => { + const rig = await setUp(); + await granted(rig, 1); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + const received = rig.fake.requests.length; + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + + expect(rig.fake.requests).toHaveLength(received); + }); + + it("a lease released through the front while a command runs stays ended by the agent: a renewal that fires between the simlock release and the end of the agent's release, and the command's final renewal, mark nothing, and the next heartbeat still answers `LEASE_NOT_FOUND`", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + const running = await runningCommand(rig, 1); + const tap = tapDispatch(rig); + tap.afterNext("lease.release", async () => { + await advance(rig, 12_000, 4_000); + }); + + rig.fake.answerReleaseWhileCommandsRun(); + + const released = await envelopeOf(await release(rig, 1)); + rig.fake.releaseCommand(); + await running.promise; + + expect(released.result).toMatchObject({ ok: true, released: true }); + expect(await listed(rig)).toEqual([]); + expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); + }); + + it("after `simlock release --all`, the agent's release answers success, a heartbeat for the old ID then answers `LEASE_ENDED_BY_HOST` (ended by simlock, not `LEASE_NOT_FOUND`), and the next allocate gets a new lease; an entry found unlisted by another allocate's check answers its next heartbeat `LEASE_ENDED_BY_HOST` without reaching the agent-device daemon", async () => { + const rig = await setUp(); + await granted(rig, 1); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + + const released = await envelopeOf(await release(rig, 1)); + + expect(released.result).toMatchObject({ ok: true, released: false }); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + await granted(rig, 2); + expect(await listed(rig)).toEqual([hex(2)]); + + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + await advance(rig, 125_000, 25_000); + await granted(rig, 3); + const received = heartbeatsReceived(rig); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 2))); + expect(heartbeatsReceived(rig)).toBe(received); + }); + + it("after `simlock release --all` and the agent's release, a heartbeat answers `LEASE_ENDED_BY_HOST` without reaching the agent-device daemon, and a running command's renewals stop", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await runningCommand(rig, 1); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + rig.fake.answerReleaseWhileCommandsRun(); + await release(rig, 1); + const received = rig.fake.requests.length; + const tap = tapDispatch(rig); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + await advance(rig, 120_000, 10_000); + + expect(rig.fake.requests).toHaveLength(received); + rig.fake.releaseCommand(); + expect(tap.operations).not.toContain("lease.list"); + expect(tap.operations).not.toContain("lease.renew"); + }); + + it("a release for a lease simlock ended answers the agent-device daemon's reply as a success, and the agent-device lease is released", async () => { + const rig = await setUp(); + await granted(rig, 1); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + const tap = tapDispatch(rig); + + const answer = await envelopeOf(await release(rig, 1)); + + expect(answer.result).toMatchObject({ ok: true, released: false }); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expect(tap.operations).not.toContain("lease.release"); + }); + + it("while a command for the lease runs longer than its TTL, the simlock lease is renewed every TTL/3 and does not expire", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await runningCommand(rig, 1); + const tap = tapDispatch(rig); + const deadlines: (number | undefined)[] = []; + + for (let tick = 0; tick < 9; tick += 1) { + await advance(rig, 10_000, 5_000); + deadlines.push(await deadlineOf(rig, 1)); + } + + expect(deadlines.every((deadline) => deadline !== undefined)).toBe(true); + expect(deadlines.at(-1)).toBeGreaterThan(rig.clock.now()); + expect(new Set(deadlines).size).toBeGreaterThan(5); + const checks = tap.operations.filter((name) => name === "lease.list").length; + expect(checks).toBeGreaterThanOrEqual(8); + expect(checks).toBeLessThanOrEqual(9); + }); + + it("a command that runs on past `lease.maxTtlMs` stops counting as running: its renewals stop, and the lease is no longer kept", async () => { + const rig = await setUp(); + await granted(rig, 1); + const tap = tapDispatch(rig); + await runningCommand(rig, 1); + + await advance(rig, maxTtlMs + 400_000, 100_000); + const calls = frontCalls(tap).length; + await advance(rig, 1_000_000, 100_000); + + expect(calls).toBeGreaterThan(0); + expect(frontCalls(tap)).toHaveLength(calls); + expect(await listed(rig)).toEqual([]); + }, 60_000); + + it("when the command ends, the lease is renewed once more, and its deadline does not move again over the next TTL", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + const running = await runningCommand(rig, 1); + await advance(rig, 25_000, 5_000); + rig.fake.releaseCommand(); + await running.promise; + + await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); + const final = await deadlineOf(rig, 1); + await advance(rig, 29_000, 5_000); + + expect(await deadlineOf(rig, 1)).toBe(final); + }); + + it("a command agent-device answers with a 5xx reaches the CLI unchanged", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1); + rig.fake.failNextCommandWithStatus(500, "snapshot runner crashed"); + + const response = await command(rig, "snapshot", 1); + + expect(response.status).toBe(500); + expect(await response.text()).toBe("snapshot runner crashed"); + }); + + it("a command that fails, and one whose CLI hangs up, stop the renewals the same way", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await granted(rig, 2, { ttlMs: 30_000 }); + rig.fake.failNextCommand({ code: "COMMAND_FAILED", message: "the command failed" }); + const failing = await runningCommand(rig, 1); + await advance(rig, 15_000, 5_000); + rig.fake.releaseCommand(); + const failure = await envelopeOf(await failing.promise); + expect(failure.error).toMatchObject({ message: "the command failed" }); + await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); + const afterFailure = await deadlineOf(rig, 1); + + const hangUp = new AbortController(); + const hanging = await runningCommand(rig, 2, { signal: hangUp.signal }); + await advance(rig, 15_000, 5_000); + hangUp.abort(); + await hanging.promise.catch(() => undefined); + await expect.poll(() => deadlineOf(rig, 2)).toBe(rig.clock.now() + 30_000); + const afterHangUp = await deadlineOf(rig, 2); + + await advance(rig, 14_000, 2_000); + expect(await deadlineOf(rig, 1)).toBe(afterFailure); + expect(await deadlineOf(rig, 2)).toBe(afterHangUp); + }); + + it("with two overlapping commands on one lease, renewals go on until the second one ends", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + const firstHangUp = new AbortController(); + await runningCommand(rig, 1, { signal: firstHangUp.signal }); + const second = await runningCommand(rig, 1); + await advance(rig, 20_000, 5_000); + firstHangUp.abort(); + + await advance(rig, 40_000, 5_000); + + expect(await deadlineOf(rig, 1)).toBeGreaterThan(rig.clock.now()); + rig.fake.releaseCommand(); + await second.promise; + await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); + const final = await deadlineOf(rig, 1); + await advance(rig, 29_000, 5_000); + expect(await deadlineOf(rig, 1)).toBe(final); + }); + + it("a renewal that fails because simlock ended the lease mid-command stops the renewals, and the command's reply still reaches the CLI; a renewal refused for another reason is tried again at the next tick", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + const tap = tapDispatch(rig); + const running = await runningCommand(rig, 1); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + await advance(rig, 15_000, 5_000); + const calls = frontCalls(tap).length; + + await advance(rig, 60_000, 5_000); + + expect(frontCalls(tap)).toHaveLength(calls); + rig.fake.releaseCommand(); + const reply = await envelopeOf(await running.promise); + expect(reply.result).toMatchObject({ command: "snapshot", ok: true }); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + + await granted(rig, 2, { ttlMs: 30_000 }); + const started = await deadlineOf(rig, 2); + await runningCommand(rig, 2); + tap.rejectNext("lease.renew"); + await advance(rig, 11_000, 1_000); + expect(await deadlineOf(rig, 2)).toBe(started); + + await advance(rig, 10_000, 1_000); + + expect(await deadlineOf(rig, 2)).toBeGreaterThan(started as number); + }); + + it('an agent release whose `lease.release` simlock refuses with another error clears "release in progress": the simlock lease stays, and a later renewal that finds it gone marks it ended by simlock', async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + const tap = tapDispatch(rig); + tap.rejectNext("lease.release"); + + const refused = await envelopeOf(await release(rig, 1)); + + expect(refused.result).toMatchObject({ ok: true }); + expect(await listed(rig)).toEqual([hex(1)]); + await rig.daemon.dispatch("lease.release-all", {}, adminSession); + + const answer = await envelopeOf(await heartbeat(rig, 1)); + + expectEndedBySimlock(answer); + const received = heartbeatsReceived(rig); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + expect(heartbeatsReceived(rig)).toBe(received); + }); + + it("a heartbeat lowering the TTL from 300 s to 5 s while a command runs re-arms the renewals, and the lease does not expire before the command ends", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 300_000 }); + await runningCommand(rig, 1); + await advance(rig, 1_000); + + await heartbeat(rig, 1, { ttlMs: 5_000 }); + + await advance(rig, 360_000, 2_000); + expect(await listed(rig)).toEqual([hex(1)]); + expect(await deadlineOf(rig, 1)).toBeGreaterThan(rig.clock.now()); + }, 60_000); + + it("a command naming a lease ID the front map does not hold for its token, or the lease of an ended entry, starts no renewals", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await granted(rig, 2, { ttlMs: 30_000 }); + await release(rig, 2); + const tap = tapDispatch(rig); + await runningCommand(rig, 1, { token: secrets.two }); + await runningCommand(rig, 2); + + await advance(rig, 40_000, 5_000); + + expect(frontCalls(tap)).toEqual([]); + expect(await listed(rig)).toEqual([]); + }); + + it("a release through the front while a command runs stops that lease's renewals", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await runningCommand(rig, 1); + await advance(rig, 5_000, 5_000); + rig.fake.answerReleaseWhileCommandsRun(); + await release(rig, 1); + const tap = tapDispatch(rig); + + await advance(rig, 60_000, 5_000); + + expect(frontCalls(tap)).toEqual([]); + rig.fake.releaseCommand(); + }); + + it("a lease whose CLI stops sending heartbeats expires one TTL after its last heartbeat, and its device can be leased again", async () => { + const rig = await setUp({ maxRunning: 1 }); + await granted(rig, 1, { ttlMs: 30_000 }); + await advance(rig, 10_000, 5_000); + await heartbeat(rig, 1, { ttlMs: 30_000 }); + await advance(rig, 29_000, 5_000); + expect(await listed(rig)).toEqual([hex(1)]); + + await advance(rig, 3_000, 1_000); + + expect(await listed(rig)).toEqual([]); + await granted(rig, 2); + expect(await listed(rig)).toEqual([hex(2)]); + }); + + it("after the same restart (same `FakeDriver`, lease still listed), a release for a lease made before the restart answers the agent-device daemon's reply as a success and releases that simlock lease", async () => { + const first = await setUp(); + await granted(first, 1); + const rig = await first.restart(); + expect(await listed(rig)).toEqual([hex(1)]); + + const answer = await envelopeOf(await release(rig, 1)); + + expect(answer.result).toMatchObject({ ok: true, released: true }); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expect(await listed(rig)).toEqual([]); + }); + + it("a release with no map entry from a second token, naming the first token's lease ID, answers the agent-device daemon's reply and leaves the first token's simlock lease in place", async () => { + const first = await setUp(); + await granted(first, 1); + const rig: Rig = await first.restart(); + rig.fake.letAnyTenantRelease(); + + const answer = await envelopeOf(await release(rig, 1, { token: secrets.two })); + + expect(answer.result).toMatchObject({ ok: true, released: true }); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expect(await listed(rig)).toEqual([hex(1)]); + }); + + it("stopping the daemon while a command runs renews nothing once the stop began: the lease's deadline, read after a restart on the same data directory and the same `FakeDriver` (the lease still listed), is the one it had before the stop", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 30_000 }); + await runningCommand(rig, 1); + await advance(rig, 12_000, 4_000); + const before = await deadlineOf(rig, 1); + const tap = tapDispatch(rig); + + const stopping = rig.daemon.stop("test"); + await advance(rig, 3_000, 1_000).catch(() => undefined); + await stopping; + expect(frontCalls(tap)).toEqual([]); + const restarted = await rig.restart(); + + expect(await listed(restarted)).toEqual([hex(1)]); + expect(await deadlineOf(restarted, 1)).toBe(before); + }); +}); diff --git a/src/agent-device/edge-release.test.ts b/src/agent-device/edge-release.test.ts new file mode 100644 index 000000000..af3f4dfa2 --- /dev/null +++ b/src/agent-device/edge-release.test.ts @@ -0,0 +1,338 @@ +import { describe, expect, it } from "vitest"; +import { FakeDriver } from "../core/testing.js"; +import { + heartbeat, + advance, + release, + liveStates, + track, + listed, + simlockPicks, + batch, +} from "./test-edge.js"; +import { + secrets, + scope, + type Rig, + sleep, + setUp, + envelopeOf, + hex, + allocate, + granted, + command, + leases, + allocatesReceived, + runCommand, + forwarded, + refused, +} from "./test-support.js"; +import { phoneModels, expectLeaseGone } from "./test-edge-front.js"; + +describe("the agent-device front, through the whole worker daemon", () => { + it("an agent-device allocate that answers an error, and a grant that names no device address, release the simlock lease, and the CLI gets the error", async () => { + const rig: Rig = await setUp(); + rig.fake.failNextAllocate({ + code: "NO_DEVICE", + message: "no such device", + details: { reason: "AGENT_DEVICE_REASON" }, + }); + + const answer = await envelopeOf(await allocate(rig, 1)); + + expect(answer.error).toEqual({ + code: -32000, + data: { + code: "NO_DEVICE", + message: "no such device", + details: { reason: "AGENT_DEVICE_REASON" }, + }, + message: "no such device", + }); + await expect.poll(async () => (await leases(rig)).length).toBe(0); + + class NoAddressDriver extends FakeDriver { + override async makeReady(...arguments_: Parameters) { + const ready = await super.makeReady(...arguments_); + return { ...ready, address: undefined as unknown as string }; + } + } + const bare = await setUp({ + drivers: (clock) => [ + new NoAddressDriver({ + ...phoneModels, + availableOsVersions: ["26.5"], + clock, + knownModels: ["iPhone 16"], + platform: "ios", + }), + ], + }); + + const noAddress = await envelopeOf(await allocate(bare, 2)); + + expect(noAddress.error?.data).toMatchObject({ + code: "COMMAND_FAILED", + details: { reason: "SIMLOCK_FAILED" }, + retriable: false, + }); + await expect.poll(async () => (await leases(bare)).length).toBe(0); + expect(allocatesReceived(bare, 2)).toEqual([]); + }); + + it("an agent-device allocate with no usable reply (an unreachable daemon, a non-JSON body) releases the simlock lease and answers `COMMAND_FAILED`; when agent-device answers the follow-up release with a scope error, the simlock lease stays until its TTL, and when it answers `LEASE_NOT_FOUND`, the simlock lease is released", async () => { + const rig: Rig = await setUp(); + expect((await envelopeOf(await allocate(rig, 1))).result).toMatchObject({ ok: true }); + rig.fake.setUnreachable(true); + const unreachable = await envelopeOf(await allocate(rig, 2)); + rig.fake.setUnreachable(false); + + expect(unreachable.error?.data?.code).toBe("COMMAND_FAILED"); + await expect.poll(async () => (await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + + rig.fake.answerNextAllocateWith("bad gateway"); + const notJson = await envelopeOf(await allocate(rig, 3)); + + expect(notJson.error?.data?.code).toBe("COMMAND_FAILED"); + await expect.poll(async () => (await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + + rig.fake.answerNextAllocateWith("bad gateway"); + rig.fake.failNextRelease({ code: "UNAUTHORIZED", message: "lease scope mismatch" }); + const kept = await envelopeOf(await allocate(rig, 4, { ttlMs: 60_000 })); + + expect(kept.error?.data?.code).toBe("COMMAND_FAILED"); + expect((await leases(rig)).map((lease) => lease.id)).toContain(hex(4)); + await advance(rig, 61_000); + await expect + .poll(async () => (await leases(rig)).map((lease) => lease.id)) + .not.toContain(hex(4)); + + rig.fake.answerNextAllocateWith("bad gateway"); + rig.fake.failNextRelease({ code: "LEASE_NOT_FOUND", message: "no such lease" }); + const gone = await envelopeOf(await allocate(rig, 5, { ttlMs: 60_000 })); + + expect(gone.error?.data?.code).toBe("COMMAND_FAILED"); + await expect + .poll(async () => (await leases(rig)).map((lease) => lease.id)) + .not.toContain(hex(5)); + }); + + it("the follow-up release after an allocate with no usable reply carries the allocate's runId, leaseProvider and clientId and not its deviceKey", async () => { + const rig: Rig = await setUp(); + rig.fake.answerNextAllocateWith("bad gateway"); + + const answer = await envelopeOf(await allocate(rig, 1, { deviceKey: "saved-device" })); + + expect(answer.error?.data?.code).toBe("COMMAND_FAILED"); + const params = rig.fake.rpcRequests("agent_device.lease.release")[0]?.params; + expect(params).toMatchObject({ ...scope, leaseId: hex(1) }); + expect(params).not.toHaveProperty("deviceKey"); + }); + + it("a lease whose allocate reply names device key `ios:mobile:` keeps it: a command naming that lease with that `deviceKey` reaches agent-device", async () => { + const rig: Rig = await setUp(); + const key = "ios:mobile:00008110-001A2B3C4D5E"; + rig.fake.nameDeviceKeyInNextReply(key); + await granted(rig, 1); + + await forwarded(rig, () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: key } })); + }); + + it("an allocate reply whose lease `deviceKey` is over 128 characters still succeeds and keeps the lease, with the device key left unset: no release is sent and a command naming the lease with a `deviceKey` is refused", async () => { + const rig: Rig = await setUp(); + rig.fake.nameDeviceKeyInNextReply("x".repeat(129)); + + const answer = await envelopeOf(await allocate(rig, 1)); + + expect(answer.error).toBeUndefined(); + expect(rig.fake.rpcRequests("agent_device.lease.release")).toHaveLength(0); + expect((await leases(rig)).map((lease) => lease.id)).toContain(hex(1)); + await refused( + rig, + () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: "x".repeat(129) } }), + simlockPicks, + ); + }); + + it("a CLI that hangs up while agent-device makes its lease leaves neither lease: the agent-device lease is released with the reply's scope, then the simlock lease; when that release fails, the simlock lease stays until its TTL", async () => { + const rig: Rig = await setUp(); + rig.fake.holdAllocate(); + rig.fake.nameDeviceKeyInNextReply("device-key-1"); + const hangUp = new AbortController(); + const first = track(allocate(rig, 1, {}, { signal: hangUp.signal })); + await sleep(50); + expect(first.done()).toBe(false); + await expect.poll(() => allocatesReceived(rig, 1).length).toBe(1); + + hangUp.abort(); + await first.promise.catch(() => undefined); + await sleep(50); + rig.fake.letAllocateGo(); + + await expect.poll(async () => (await leases(rig)).length).toBe(0); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expect(rig.fake.rpcRequests("agent_device.lease.release")[0]?.params).toMatchObject({ + ...scope, + deviceKey: "device-key-1", + }); + + rig.fake.holdAllocate(); + rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "release failed" }); + const abort = new AbortController(); + const second = track(allocate(rig, 2, { ttlMs: 60_000 }, { signal: abort.signal })); + await sleep(50); + expect(second.done()).toBe(false); + await expect.poll(() => allocatesReceived(rig, 2).length).toBe(1); + abort.abort(); + await second.promise.catch(() => undefined); + await sleep(50); + rig.fake.letAllocateGo(); + await expect.poll(() => rig.fake.rpcRequests("agent_device.lease.release").length).toBe(2); + + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2)]); + await advance(rig, 61_000); + await expect.poll(async () => (await leases(rig)).length).toBe(0); + }); + + it("release goes to the agent-device daemon first, then frees the simlock lease and the device; a release after `close`, which already freed the simlock lease, reaches the daemon with no lease held and is answered `released: false`", async () => { + const rig = await setUp(); + let heldAtDaemonRelease: readonly string[] = []; + rig.fake.onRelease = async () => { + heldAtDaemonRelease = (await leases(rig)).map((lease) => lease.id); + }; + await granted(rig, 1); + + const released = await envelopeOf(await release(rig, 1)); + + expect(released.result).toMatchObject({ ok: true, released: true }); + expect(heldAtDaemonRelease).toEqual([hex(1)]); + expect(await leases(rig)).toEqual([]); + await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); + + await granted(rig, 2); + await command(rig, "close", 2); + await expect.poll(async () => leases(rig)).toEqual([]); + const afterClose = await envelopeOf(await release(rig, 2)); + + expect(afterClose.result).toMatchObject({ ok: true, released: false }); + expect(heldAtDaemonRelease).toEqual([]); + expect(await leases(rig)).toEqual([]); + await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); + }); + + it("a runner allocates with retainOnClose, closes, and its lease stays until release", async () => { + const cases = [ + (rig: Rig) => command(rig, "close", 1), + (rig: Rig) => command(rig, "close", 1, {}, { positionals: ["com.example.app"] }), + (rig: Rig) => batch(rig, 1, [{ command: "snapshot" }, { command: "close" }]), + ]; + for (const close of cases) { + const rig: Rig = await setUp(); + expect( + (await envelopeOf(await allocate(rig, 1, { retainOnClose: true }))).result, + ).toMatchObject({ ok: true, data: { lease: { retainOnClose: true } } }); + expect((await envelopeOf(await close(rig))).result).toMatchObject({ ok: true }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); + expect((await envelopeOf(await command(rig, "snapshot", 1))).result).toMatchObject({ + ok: true, + }); + expect((await envelopeOf(await release(rig, 1))).result).toMatchObject({ + ok: true, + released: true, + }); + expect(await leases(rig)).toEqual([]); + await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); + } + }); + + const ended = [{ leaseId: hex(1), outcome: "released" }]; + it.each([ + [ + "a `close` that succeeds, in `data`", + { result: { data: { endedLeases: ended } } }, + (rig: Rig) => command(rig, "close", 1), + ], + [ + "a `close` that fails, in `error.details`", + { error: { data: { details: { endedLeases: ended } } } }, + (rig: Rig) => { + rig.fake.failNextCommand({ code: "COMMAND_FAILED", message: "close failed" }); + return command(rig, "close", 1); + }, + ], + [ + "a `batch` with a close step, in `data` beside `results`", + { result: { data: { endedLeases: ended, results: [] } } }, + (rig: Rig) => batch(rig, 1, [{ command: "snapshot" }, { command: "close" }]), + ], + ] as const)( + "%s: a command that ends the agent-device lease and names it in `endedLeases` releases the simlock lease and frees the device though the CLI sends no release, and a later heartbeat answers `LEASE_NOT_FOUND`", + async (_case, reply, send) => { + const rig = await setUp(); + await granted(rig, 1); + + expect(await envelopeOf(await send(rig))).toMatchObject(reply); + await expect.poll(async () => listed(rig)).toEqual([]); + await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); + expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); + }, + ); + + // why: agent-device 0.21.28 streams a reply when the CLI runs without --json; the JSON-RPC answer is the `response` of a `type: "response"` line, not the line itself. + it("a close streamed in agent-device's own line shape releases the simlock lease", async () => { + const rig = await setUp(); + await granted(rig, 1); + const chunks = [ + `${JSON.stringify({ event: { kind: "command", message: "Closing" }, type: "progress" })}\n`, + `${JSON.stringify({ + response: { + id: "x", + jsonrpc: "2.0", + result: { data: { endedLeases: ended, session: "t:adc-1" }, ok: true }, + }, + type: "response", + })}\n`, + ]; + rig.fake.streamNextCommand(chunks); + + const streamed = await command(rig, "close", 1); + + expect(await streamed.text()).toBe(chunks.join("")); + await expect.poll(async () => listed(rig), { timeout: 3_000 }).toEqual([]); + await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); + }); + + it("a release the agent-device daemon fails keeps the simlock lease and answers the daemon's error", async () => { + const rig = await setUp(); + await granted(rig, 1); + rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon could not release" }); + + const answer = await envelopeOf(await release(rig, 1)); + + expect(answer.error).toMatchObject({ + data: { code: "COMMAND_FAILED" }, + message: "daemon could not release", + }); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + }); + + it("a release from a second token naming the first token's lease ID leaves the first token's simlock lease in place", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1); + + const answer = await envelopeOf(await release(rig, 1, { token: secrets.two })); + + expect(answer.error).toBeDefined(); + expect(rig.fake.rpcRequests("agent_device.lease.release")).toHaveLength(1); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + expect(rig.fake.leases.has(hex(1))).toBe(true); + + rig.fake.letAnyTenantRelease(); + const crossed = await envelopeOf(await release(rig, 1, { token: secrets.two })); + + expect(crossed.result).toMatchObject({ ok: true }); + expect(rig.fake.leases.has(hex(1))).toBe(false); + expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + }); +}); diff --git a/src/agent-device/edge.test.ts b/src/agent-device/edge.test.ts index 3d0e54c66..f1b8a617a 100644 --- a/src/agent-device/edge.test.ts +++ b/src/agent-device/edge.test.ts @@ -1,7 +1,4 @@ import { describe, expect, it } from "vitest"; - -import { FakeDriver } from "../core/testing.js"; -import { DispatchError } from "../daemon/dispatch.js"; import { FakeClock, FakeProcessSupervisor, @@ -9,66 +6,31 @@ import { JsonLinesLogger, MemoryFilesystem, MemoryLogSink, - NodeSqliteOpener, ScriptedProcessRunner, } from "../ports/index.js"; import { FakeAgentDevice, FakeAgentDeviceHealth } from "./testing.js"; import { AgentDeviceSupervisor } from "./host/agent-device-supervisor.js"; import { SupervisorAgentDeviceUpstream } from "./host/upstream.js"; -import { - heartbeat, - phoneSerial, - setUpWithPhone, - advance, - release, - liveStates, - grantedDevice, - track, - listed, - type DispatchTap, - tapDispatch, - runningCommand, - simlockPicks, - idOf, - batch, -} from "./test-edge.js"; +import { setUpWithPhone, track } from "./test-edge.js"; import { secrets, - principals, notAvailable, agentTokenOnly, - queueTimeoutMessage, - defaultTtlMs, - maxTtlMs, - scope, adminSession, - sessionFor, - type Envelope, type Rig, - androidDriver, - taggedAndroidDriver, sleep, setUp, send, - rpcBody, - rpc, bodyOf, envelopeOf, hex, - allocateParams, allocate, granted, command, leases, - requestedSpecs, - receivedAllocate, - allocatesReceived, - simlockMessage, refusal, - runCommand, - forwarded, - refused, } from "./test-support.js"; +import { waiting } from "./test-edge-front.js"; // why: Fallow reads a `rig` binding's type only from an annotation in this file, not from an imported // setUp's return type, so a `rig` whose test calls a FakeAgentDevice member only these tests use is @@ -120,79 +82,6 @@ async function unavailableSupervisor(version: string, reason: "wrong-version" | return new SupervisorAgentDeviceUpstream(supervisor); } -const phoneModels = { - defaultModels: { phone: ["iPhone 16"] }, - modelClasses: { "iPhone 16": "phone" }, -} as const; - -function allocatePhone(rig: Rig, n: number, extra: Record = {}) { - return allocate(rig, n, { platform: "android", ...extra, physical: true }); -} - -async function waiting( - rig: Rig, -): Promise { - return (await rig.daemon.dispatch("list.get", { kind: "requests" }, adminSession)) as never; -} - -const endedMessage = "lease ended by simlock; run disconnect, then open"; - -function heartbeatsReceived(rig: Rig): number { - return rig.fake.rpcRequests("agent_device.lease.heartbeat").length; -} - -function expectEndedBySimlock(answer: Envelope): void { - expect(answer.result).toBeUndefined(); - expect(answer.error).toMatchObject({ - data: { - code: "UNAUTHORIZED", - details: { reason: "LEASE_ENDED_BY_HOST" }, - message: endedMessage, - retriable: false, - }, - }); -} - -// why: a lease the front asked for in this process, whose CLI hung up and whose agent-device release failed, stays in simlock with agent-device still holding it, and the front map has no entry for it. -async function keptWhileAgentDeviceHolds(rig: Rig, n: number): Promise { - const releases = rig.fake.rpcRequests("agent_device.lease.release").length; - rig.fake.holdAllocate(); - rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "release failed" }); - const hangUp = new AbortController(); - const pending = track(allocate(rig, n, { ttlMs: 60_000 }, { signal: hangUp.signal })); - await expect.poll(() => allocatesReceived(rig, n).length).toBe(1); - hangUp.abort(); - await pending.promise.catch(() => undefined); - await sleep(50); - rig.fake.letAllocateGo(); - await expect - .poll(() => rig.fake.rpcRequests("agent_device.lease.release").length) - .toBe(releases + 1); -} - -function expectLeaseGone(answer: Envelope, reason: string): void { - expect(answer.error).toMatchObject({ - data: { code: "UNAUTHORIZED", details: { reason } }, - }); -} - -async function deadlineOf(rig: Rig, n: number): Promise { - return (await leases(rig)).find((lease) => lease.id === hex(n))?.ttlDeadline; -} - -function frontCalls(tap: DispatchTap): readonly string[] { - return tap.operations.filter((name) => name === "lease.renew" || name === "lease.list"); -} - -async function renewMessage(rig: Rig, leaseId: string, ttlMs: number): Promise { - try { - await rig.daemon.dispatch("lease.renew", { leaseId, ttlMs }, sessionFor(principals.one)); - } catch (error) { - return (error as Error).message; - } - throw new Error("simlock accepted a renewal the test expected it to refuse"); -} - describe("the agent-device front, through the whole worker daemon", () => { it("with capabilities.agent-device.enabled false, and unset on a gateway-mode daemon, /v1/agent-device/health answers 404, as today", async () => { const enabled = await setUp(); @@ -306,2184 +195,152 @@ describe("the agent-device front, through the whole worker daemon", () => { expect(rig.fake.requests.length).toBeGreaterThan(before); }); - it("allocate `--platform ios` makes a simlock lease whose ID and requester are the CLI's lease ID, owned by the token's principal, and the lease request names no osVersion, mode, imageTag or physical device", async () => { + it("a command's reply, a streamed progress body included, reaches the CLI unchanged, and a CLI that hangs up during it aborts the request the agent-device daemon was answering", async () => { const rig = await setUp(); + const chunks = [ + '{"type":"progress","event":{"kind":"command","message":"one"}}\n', + '{"type":"progress","event":{"kind":"command","message":"two"}}\n', + '{"type":"response","response":{"jsonrpc":"2.0","id":"x","result":{"ok":true,"data":{}}}}\n', + ]; + rig.fake.streamNextCommand(chunks); - const answer = await allocate(rig, 1); - - expect((await envelopeOf(answer)).result).toMatchObject({ ok: true }); - expect(await leases(rig)).toMatchObject([ - { id: hex(1), ownerId: principals.one, requesterId: hex(1) }, - ]); - expect(await requestedSpecs(rig, hex(1))).toEqual([{ platform: "ios" }]); - - const bare = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: [], - clock, - knownModels: ["iPhone 16"], - platform: "ios", - }), - ], - }); - const refused = await allocate(bare, 2); - expect((await envelopeOf(refused)).error?.message).toContain("refused by simlock: "); - expect(bare.driver.calls.map((call) => call.operation)).not.toContain("installComponent"); - }); - - it('allocate with `osVersion` "26.0", ">=18 <27" or, on Android, "34" makes a lease request that carries it as typed', async () => { - const rig = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: ["18.4", "26.0", "27.0"], - clock, - defaultModels: { phone: ["iPhone 16"] }, - knownModels: ["iPhone 16"], - modelClasses: { "iPhone 16": "phone" }, - platform: "ios", - }), - new FakeDriver({ - availableOsVersions: ["34", "35"], - clock, - defaultModels: { phone: ["Pixel 8"] }, - knownModels: ["Pixel 8"], - modelClasses: { "Pixel 8": "phone" }, - platform: "android", - }), - ], - maxRunning: 8, - }); - - await granted(rig, 1, { osVersion: "26.0" }); - await granted(rig, 2, { osVersion: ">=18 <27" }); - await granted(rig, 3, { osVersion: "34", platform: "android" }); - - expect(await requestedSpecs(rig, hex(1))).toEqual([{ osVersion: "26.0", platform: "ios" }]); - expect(await requestedSpecs(rig, hex(2))).toEqual([{ osVersion: ">=18 <27", platform: "ios" }]); - expect(await requestedSpecs(rig, hex(3))).toEqual([{ osVersion: "34", platform: "android" }]); - }); - - it('allocate on Android with `imageTag` "google_apis_playstore", with and without `osVersion`, makes a lease request that carries it', async () => { - const rig = await setUp({ drivers: (clock) => [taggedAndroidDriver(clock)], maxRunning: 8 }); - const android = { platform: "android" }; - - await granted(rig, 1, { ...android, imageTag: "google_apis_playstore" }); - await granted(rig, 2, { ...android, imageTag: "google_apis_playstore", osVersion: "33" }); - - expect(await requestedSpecs(rig, hex(1))).toEqual([ - { imageTag: "google_apis_playstore", platform: "android" }, - ]); - expect(await requestedSpecs(rig, hex(2))).toEqual([ - { imageTag: "google_apis_playstore", osVersion: "33", platform: "android" }, - ]); - }); + const streamed = await command( + rig, + "install", + undefined, + {}, + { meta: { uploadedArtifactId: "art-1" } }, + ); - it("the allocate the agent-device daemon receives carries neither `osVersion` nor `imageTag`, whether the CLI sent one or both", async () => { - const rig = await setUp({ drivers: (clock) => [taggedAndroidDriver(clock)], maxRunning: 8 }); - const android = { platform: "android" }; + expect(await streamed.text()).toBe(chunks.join("")); - await granted(rig, 1, { ...android, osVersion: "34" }); - await granted(rig, 2, { ...android, imageTag: "google_apis_playstore" }); - await granted(rig, 3, { ...android, imageTag: "google_apis_playstore", osVersion: "33" }); + const plain = await envelopeOf(await command(rig, "snapshot", undefined)); + expect(plain.result).toMatchObject({ command: "snapshot", ok: true }); - for (const n of [1, 2, 3]) { - const received = receivedAllocate(rig, n); - expect(received).not.toHaveProperty("osVersion"); - expect(received).not.toHaveProperty("imageTag"); - } - }); + rig.fake.holdCommand(); + const hangUp = new AbortController(); + const held = track(command(rig, "snapshot", undefined, { signal: hangUp.signal })); + await sleep(50); + expect(held.done()).toBe(false); + await expect.poll(() => rig.fake.rpcRequests("agent_device.command").length).toBe(3); + hangUp.abort(); + await held.promise.catch(() => undefined); - it("the relayed allocate reply carries `data.lease.device` as `{ model, osVersion, deviceMode, physical: false, udid or serial }`, `deviceMode` being the mode the device reports, on iOS and on Android without a tag, and with `imageTag` on Android with one, the daemon's `deviceKey` and `data.device` unchanged", async () => { - const rig: Rig = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: ["18.4", "26.0", "27.0"], - clock, - defaultModels: { phone: ["iPhone 16"] }, - knownModels: ["iPhone 16"], - mode: "slim", - modelClasses: { "iPhone 16": "phone" }, - platform: "ios", - }), - taggedAndroidDriver(clock), - ], - maxRunning: 8, - }); - const cases = [ - [ - 1, - { osVersion: ">=18 <27" }, - { deviceMode: "slim", model: "iPhone 16", osVersion: "26.0", physical: false }, - ], - [2, {}, { deviceMode: "slim", model: "iPhone 16", osVersion: "27.0", physical: false }], - [ - 3, - { imageTag: "google_apis_playstore", platform: "android" }, - { - deviceMode: "full", - imageTag: "google_apis_playstore", - model: "Pixel 8", - osVersion: "34", - physical: false, - }, - ], - [ - 4, - { platform: "android" }, - { deviceMode: "full", model: "Pixel 8", osVersion: "35", physical: false }, - ], - ] as const; - for (const [n, extra, device] of cases) { - rig.fake.nameDeviceKeyInNextReply(`key-${n}`); - rig.fake.nameDeviceInNextReply({}); - const result = (await envelopeOf(await allocate(rig, n, extra, {}))).result as { - data: { device: unknown; lease: Record }; - }; - expect(result.data.lease["device"]).toEqual({ - ...device, - ["platform" in extra && extra.platform === "android" ? "serial" : "udid"]: idOf(rig, n), - }); - expect(idOf(rig, n)).not.toBe("undefined"); - expect(result.data.lease["deviceKey"]).toBe(`key-${n}`); - expect(result.data.device).toMatchObject({ id: expect.any(String) }); - } + await expect.poll(() => rig.fake.abortedCommands).toEqual(["snapshot"]); }); - it('allocate with `osVersion` "^18", with `imageTag` on iOS (with a model and without) or with an OS the driver lacks answers `refused by simlock:` with simlock\'s own message and leaves no lease; under `downloads.policy: always` the missing OS is installed for a named model and still refused without one', async () => { + it("an upload preflight answers a ticket URL under `/v1/agent-device/upload/direct/` on simlock's own origin, carrying the agent's own simlock token, and a `PUT` to it reaches the agent-device daemon", async () => { const rig = await setUp(); - const refusals: readonly Record[] = [ - { osVersion: "^18" }, - { device: "iPhone 17 Pro", imageTag: "default" }, - { imageTag: "default" }, - { osVersion: "26.1" }, - ]; - for (const [index, extra] of refusals.entries()) { - const expected = await simlockMessage(rig, { - leaseId: hex(90 + index), - platform: "ios", - ...(typeof extra["device"] === "string" ? { model: extra["device"] } : {}), - ...(extra["imageTag"] === undefined ? {} : { imageTag: extra["imageTag"] }), - ...(extra["osVersion"] === undefined ? {} : { osVersion: extra["osVersion"] }), - }); - const answer = await envelopeOf(await allocate(rig, index + 1, extra)); - expect(answer.error?.message).toBe(`refused by simlock: ${expected}`); - } - expect(await leases(rig)).toEqual([]); - expect(rig.driver.calls.map((call) => call.operation)).not.toContain("installComponent"); + const preflight = await send(rig, "/upload/preflight", { body: "{}" }); + expect(preflight.status).toBe(200); + const ticket = ( + (await preflight.json()) as { upload: { url: string; headers: Record } } + ).upload; - const always = await setUp({ downloads: "always" }); - const installs = () => - always.driver.calls.filter((call) => call.operation === "installComponent").length; - const bare = await envelopeOf(await allocate(always, 1, { osVersion: "26.1" })); - expect(bare.error?.message).toContain("refused by simlock: "); - expect(installs()).toBe(0); - const named = await envelopeOf( - await allocate(always, 2, { device: "iPhone 17 Pro", osVersion: "26.1" }), + expect(ticket.url.startsWith(`${rig.origin}/v1/agent-device/upload/direct/`)).toBe(true); + const carried = Object.entries(ticket.headers).find( + ([name]) => name.toLowerCase() === "authorization", ); - expect(named.result).toMatchObject({ ok: true }); - expect(installs()).toBe(1); - }); - - it("allocate with `physical: true` leases an enrolled phone through the phone daemon, and the reply names the device with `physical: true` (`false` for a virtual one)", async () => { - const cases = [ - { - address: { serial: "R58Y61A60HN" }, - asked: { platform: "android" }, - extra: { platform: "android" }, - id: "R58Y61A60HN", - phone: { class: "phone", model: "Pixel 8", osVersion: "15" }, - platform: "android", - }, - { - address: { udid: "00008030-000270803C28802E" }, - asked: { model: "iPhone 15 Pro", platform: "ios" }, - extra: { device: "iPhone 15 Pro" }, - id: "00008030-000270803C28802E", - phone: { class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" }, - platform: "ios", - }, - { - address: { udid: "00008030-000270803C28802E" }, - asked: { osVersion: ">=18", platform: "ios" }, - extra: { osVersion: ">=18" }, - id: "00008030-000270803C28802E", - phone: { class: "phone", model: "iPhone 15 Pro", osVersion: "18.4" }, - platform: "ios", - }, - ] as const; - for (const { address, asked, extra, id, phone, platform } of cases) { - const rig = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: ["35"], - clock, - knownModels: [phone.model], - physicalDevices: [{ ids: [id], ...phone }], - platform, - }), - ], - }); - await rig.daemon.dispatch("device.add", { id, platform }, adminSession); - rig.driver.setPhysicalPresence(id, true); - const answer = await envelopeOf( - await allocate(rig, 1, { platform, ...extra, physical: true }), - ); - expect(answer.result).toMatchObject({ - data: { - lease: { - device: { ...address, model: phone.model, osVersion: phone.osVersion, physical: true }, - }, - }, - ok: true, - }); - expect(answer.result).not.toHaveProperty("data.lease.device.deviceMode"); - expect(await requestedSpecs(rig, hex(1))).toMatchObject([{ ...asked, physical: true }]); - expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toMatchObject([ - { params: { leaseId: hex(1), ...address } }, - ]); - // why: agent-device's daemon refuses any spec key it receives, so `physical` stays with simlock. - expect(rig.phone.rpcRequests("agent_device.lease.allocate")[0]?.params).not.toHaveProperty( - "physical", - ); - expect(allocatesReceived(rig)).toEqual([]); - if (platform === "ios") { - // why: a physical iPhone has no `device.rotate`, so its `orientation auto` must reach agent-device, not be refused as an unknown name. - await command(rig, "orientation", 1, {}, { positionals: ["auto"] }); - expect(rig.phone.rpcRequests("agent_device.command")).toMatchObject([ - { params: { command: "orientation", positionals: ["auto"] } }, - ]); - } - } - }); + expect(carried?.[1]).toBe(`Bearer ${secrets.one}`); - it("allocate with `physical: true` that simlock refuses (an `imageTag`, a model no enrolled phone has, an OS range none fits) answers `refused by simlock: ` and leaves no lease", async () => { - const rig = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - availableOsVersions: ["35"], - clock, - knownModels: ["Pixel 8"], - physicalDevices: [ - { class: "phone", ids: ["R58Y61A60HN"], model: "Pixel 8", osVersion: "15" }, - ], - platform: "android", - }), - ], + const put = await fetch(ticket.url, { + body: "artifact bytes", + headers: ticket.headers, + method: "PUT", }); - await rig.daemon.dispatch( - "device.add", - { id: "R58Y61A60HN", platform: "android" }, - adminSession, - ); - rig.driver.setPhysicalPresence("R58Y61A60HN", true); - const cases = [ - [{ imageTag: "default" }, "imageTag: `imageTag` has no meaning for a physical device"], - [{ device: "Pixel 9" }, "no enrolled physical device can match android model Pixel 9"], - [ - { osVersion: ">=16" }, - "no enrolled physical device can match android class phone os in the requested range", - ], - ] as const; - - for (const [index, [extra, message]] of cases.entries()) { - const answer = await envelopeOf( - await allocate(rig, index + 1, { platform: "android", ...extra, physical: true }), - ); - - expect(answer.error).toMatchObject(refusal(`refused by simlock: ${message}`)); - } - expect(await leases(rig)).toEqual([]); - expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toEqual([]); - }); - - it("allocate with `physical: true` while the phone daemon is not running answers `agent-device is not available on this simlock: phone daemon: ` and releases the lease; a virtual allocate is granted, a command on a live phone lease answers the same, and with the simulator daemon down a physical allocate answers the simulator daemon's reason", async () => { - const down = [ - ["starting", "phone daemon: starting", true], - ["adb not found", "phone daemon: adb not found", false], - ] as const; - for (const [reason, message, retriable] of down) { - const rig = await setUpWithPhone({ maxRunning: 4 }); - rig.phone.setUnavailable(reason, retriable); - - const answer = await envelopeOf(await allocatePhone(rig, 1)); - - expect(answer.error).toMatchObject(refusal(`${notAvailable}${message}`, "COMMAND_FAILED")); - expect(answer.error?.data).toMatchObject({ - details: { reason: "SIMLOCK_UNAVAILABLE" }, - retriable, - }); - await expect.poll(() => listed(rig)).toEqual([]); - await granted(rig, 2, { platform: "android" }); - expect(await listed(rig)).toEqual([hex(2)]); - } - - const live = await setUpWithPhone(); - await granted(live, 1, { platform: "android", physical: true }); - live.phone.setUnavailable("starting"); - const liveCommand = await envelopeOf(await command(live, "snapshot", 1)); - expect(liveCommand.error).toMatchObject( - refusal(`${notAvailable}phone daemon: starting`, "COMMAND_FAILED"), - ); - const beforeRules = await envelopeOf( - await runCommand(live, "snapshot", 1, { flags: { serial: "emulator-9999" } }), - ); - expect(beforeRules.error).toMatchObject( - refusal(`${notAvailable}phone daemon: starting`, "COMMAND_FAILED"), - ); - const simulatorsDown = await setUpWithPhone(); - simulatorsDown.fake.setUnavailable("found 0.20.3"); - const simulatorsDownAnswer = await envelopeOf(await allocatePhone(simulatorsDown, 1)); - expect(simulatorsDownAnswer.error).toMatchObject( - refusal(`${notAvailable}found 0.20.3`, "COMMAND_FAILED"), - ); - expect(await listed(simulatorsDown)).toEqual([]); + expect(put.status).toBe(200); + expect(rig.fake.uploads).toEqual(["artifact bytes"]); }); - it("a command, a heartbeat, a release and a `devices` naming a phone lease reach the phone daemon, those naming a simulator lease reach the simulator daemon, and `devices` with one lease of each answers both devices, each from its daemon", async () => { + it("on a worker, an upload preflight and an artifact download follow the `x-agent-device-lease-id` header to the daemon of the lease it names, go to the simulator daemon with no header, and a header naming another token's lease is refused 403 `refused: lease is not one of yours`, and a malformed header is refused `the x-agent-device-lease-id header is not a lease id`", async () => { const rig = await setUpWithPhone(); await granted(rig, 1, { physical: true, platform: "android" }); await granted(rig, 2, { platform: "android" }); - rig.phone.listDevices([{ id: phoneSerial }, { id: "emulator-5600" }]); - rig.fake.listDevices([{ id: idOf(rig, 2) }, { id: "emulator-5601" }]); - const reached = (fake: FakeAgentDevice, method: string) => fake.rpcRequests(method).length; - const calls = [ - ["agent_device.command", (n: number) => command(rig, "snapshot", n)], - ["agent_device.lease.heartbeat", (n: number) => heartbeat(rig, n)], - ["agent_device.lease.release", (n: number) => release(rig, n)], + const named = (n: number) => ({ "x-agent-device-lease-id": hex(n) }); + const malformed = { "x-agent-device-lease-id": "NOT-A-LEASE-ID".repeat(4) }; + const reaching = (fake: FakeAgentDevice, path: string) => + fake.requests.filter((request) => request.path === path).length; + const routes = [ + ["/upload/preflight", { body: "{}" }], + ["/artifacts/a1", {}], + ] as const; + const cases = [ + [named(1), secrets.one, "phone"], + [named(2), secrets.one, "simulators"], + [{}, secrets.one, "simulators"], + [named(1), secrets.two, "forbidden"], + [malformed, secrets.one, "refused"], ] as const; - for (const [method, send] of calls) { - const [phoneBefore, simulatorBefore] = [ - reached(rig.phone, method), - reached(rig.fake, method), - ]; - await send(1); - expect([reached(rig.phone, method), reached(rig.fake, method)]).toEqual([ - phoneBefore + 1, - simulatorBefore, - ]); - await send(2); - expect([reached(rig.phone, method), reached(rig.fake, method)]).toEqual([ - phoneBefore + 1, - simulatorBefore + 1, - ]); - } - - await granted(rig, 3, { physical: true, platform: "android" }); - await granted(rig, 4, { platform: "android" }); - rig.fake.listDevices([{ id: idOf(rig, 4) }, { id: "emulator-5601" }]); - const answer = await envelopeOf(await runCommand(rig, "devices", undefined)); - const data = answer.result?.["data"] as { devices: { id: string }[] } | undefined; - expect(data?.devices.map((device) => device.id).sort()).toEqual( - [phoneSerial, idOf(rig, 4)].sort(), - ); - }); - - it("a client that sends the instance headers of the health it read, as the agent-device CLI does, is granted a phone and its later calls on that phone lease reach the phone daemon", async () => { - const rig = await setUpWithPhone(); - const health = (await (await send(rig, "/health")).json()) as { instanceId: string }; - const pinned = { - headers: { - "x-agent-device-instance": health.instanceId, - "x-agent-device-upstream-instance": "upstream-from-health", - }, - }; - - const allocated = await envelopeOf( - await allocate(rig, 1, { physical: true, platform: "android" }, pinned), - ); - const later = await envelopeOf(await command(rig, "snapshot", 1, pinned)); - const beat = await envelopeOf(await heartbeat(rig, 1, {}, pinned)); - - expect(allocated.result).toMatchObject({ ok: true }); - expect(later.error).toBeUndefined(); - expect(beat.error).toBeUndefined(); - expect(rig.phone.rpcRequests("agent_device.command")).toHaveLength(1); - expect(rig.phone.rpcRequests("agent_device.lease.heartbeat")).toHaveLength(1); - }); - - it("when simlock ends a phone lease (presence lost, an operator's release of all) the release reaches the phone daemon, not the simulator daemon, and the agent's next heartbeat answers ended by simlock", async () => { - const endings = [ - async (rig: Rig) => { - rig.driver.setPhysicalPresence(phoneSerial, false); - await advance(rig, 120_000); - }, - async (rig: Rig) => { - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - }, - ]; - for (const end of endings) { - const rig = await setUpWithPhone(); - await granted(rig, 1, { physical: true, platform: "android" }); + for (const [path, body] of routes) { + for (const [headers, token, reaches] of cases) { + const [phoneBefore, simulatorsBefore] = [ + reaching(rig.phone, path), + reaching(rig.fake, path), + ]; - await end(rig); + const answer = await send(rig, path, { ...body, headers, token }); - await expect.poll(() => listed(rig)).toEqual([]); - await expect.poll(() => rig.phone.rpcRequests("agent_device.lease.release")).toHaveLength(1); - expect(rig.fake.rpcRequests("agent_device.lease.release")).toEqual([]); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + expect([ + reaching(rig.phone, path) - phoneBefore, + reaching(rig.fake, path) - simulatorsBefore, + ]).toEqual([reaches === "phone" ? 1 : 0, reaches === "simulators" ? 1 : 0]); + if (reaches === "forbidden") { + expect(answer.status).toBe(403); + expect(await answer.text()).toContain(`refused: lease ${hex(1)} is not one of yours`); + } + if (reaches === "refused") { + expect(answer.status).toBe(400); + expect(await answer.text()).toContain( + "the x-agent-device-lease-id header is not a lease id", + ); + } + } } }); - it("a virtual allocate whose address is also its driver id reaches the simulator daemon, not the phone daemon", async () => { - const rig = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - addressIsDeviceId: true, - availableOsVersions: ["35"], - clock, - defaultModels: { phone: ["Pixel 8"] }, - knownModels: ["Pixel 8"], - modelClasses: { "Pixel 8": "phone" }, - physicalDevices: [ - { class: "phone", ids: [phoneSerial], model: "Pixel 8", osVersion: "15" }, - ], - platform: "android", + it("an `/rpc` body over 1 MiB, one that is not JSON, a JSON array, and an object with no `method` are each refused with their message and `id: null`, also while agent-device is unavailable, and neither simlock nor the agent-device daemon sees them", async () => { + const rig = await setUp(); + expect((await command(rig, "snapshot", undefined)).status).toBe(200); + const oneMiB = 1024 * 1024; + const bodies: readonly [string, string][] = [ + [ + JSON.stringify({ + id: 1, + jsonrpc: "2.0", + method: "agent_device.command", + params: { pad: "x".repeat(oneMiB) }, }), + "refused: request body over 1 MiB", ], - }); - await rig.daemon.dispatch("device.add", { id: phoneSerial, platform: "android" }, adminSession); - rig.driver.setPhysicalPresence(phoneSerial, true); - - await granted(rig, 1, { platform: "android" }); - - expect(rig.fake.rpcRequests("agent_device.lease.allocate")).toMatchObject([ - { params: { leaseId: hex(1), serial: expect.stringMatching(/^fake-android-/) } }, - ]); - expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toEqual([]); - }); - - it("after a phone lease ends and its lease id has left the ended-id window, an allocate that reuses it for a virtual device reaches the simulator daemon", async () => { - const rig = await setUpWithPhone(); - await granted(rig, 1, { physical: true, platform: "android" }); - await release(rig, 1); - await expect.poll(() => listed(rig)).toEqual([]); - rig.clock.advance(25 * 60 * 60_000); - - await granted(rig, 1, { platform: "android" }); - - expect(allocatesReceived(rig, 1)).toHaveLength(1); - expect(rig.phone.rpcRequests("agent_device.lease.allocate")).toHaveLength(1); - }); - - it("allocate with `physical: false` and `physical: null` are granted", async () => { - const rig = await setUp({ maxRunning: 8 }); - - await granted(rig, 2, { physical: false }); - await granted(rig, 3, { physical: null }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2), hex(3)]); - }); - - it('allocate with an unmapped `target`, "watch" or "", alone or beside `device: "iPhone 17 Pro"`, answers `refused: unknown --target ; use mobile, tv or desktop` and leaves no lease', async () => { - const rig = await setUp(); - const cases = [ - [{ target: "watch" }, "watch"], - [{ target: "" }, ""], - [{ device: "iPhone 17 Pro", target: "watch" }, "watch"], - ] as const; - - for (const [index, [extra, value]] of cases.entries()) { - const answer = await envelopeOf(await allocate(rig, index + 1, extra)); + ["{ not json", "refused: send one JSON-RPC request"], + ["[]", "refused: send one JSON-RPC request"], + [JSON.stringify({ id: 3, jsonrpc: "2.0" }), "refused: send one JSON-RPC request"], + ]; - expect(answer.error).toMatchObject( - refusal(`refused: unknown --target ${value}; use mobile, tv or desktop`), - ); + for (const unavailable of [false, true]) { + rig.fake.setUnavailable(unavailable ? "starting" : undefined); + const receivedBefore = rig.fake.requests.length; + for (const [body, message] of bodies) { + const answer = await envelopeOf(await send(rig, "/rpc", { body })); + expect(answer).toMatchObject({ error: refusal(message), id: null, jsonrpc: "2.0" }); + } + expect(rig.fake.requests).toHaveLength(receivedBefore); } + expect(await waiting(rig)).toEqual([]); expect(await leases(rig)).toEqual([]); - expect(allocatesReceived(rig)).toEqual([]); - }); - - it("the simlock lease's TTL is the CLI's `ttlMs`, 5 s when the CLI sent less, and `lease.defaultTtlMs` when it sent none; the allocate the agent-device daemon receives carries `ttlMs` equal to the ceiling in each case", async () => { - const rig = await setUp(); - - await granted(rig, 1, { ttlMs: 120_000 }); - await granted(rig, 2, { ttlMs: 1_000 }); - await granted(rig, 3, { ttlMs: undefined }); - - const byId = new Map((await leases(rig)).map((lease) => [lease.id, lease.ttlMs])); - expect(byId.get(hex(1))).toBe(120_000); - expect(byId.get(hex(2))).toBe(5_000); - expect(byId.get(hex(3))).toBe(defaultTtlMs); - expect(allocatesReceived(rig).map((request) => request.params?.["ttlMs"])).toEqual([ - rig.fake.leaseMaxTtlMs, - rig.fake.leaseMaxTtlMs, - rig.fake.leaseMaxTtlMs, - ]); - }); - - it('allocate `--target mobile`, `tv` and `desktop` ask simlock for class `phone`, `tv` and `desktop`; allocate `--device "iPhone 17 Pro"`, alone and with `--target tv`, asks for that model and no class', async () => { - const rig = await setUp({ maxRunning: 8 }); - - await granted(rig, 1, { target: "mobile" }); - await granted(rig, 2, { target: "tv" }); - await granted(rig, 3, { target: "desktop" }); - await granted(rig, 4, { device: "iPhone 17 Pro" }); - await granted(rig, 5, { device: "iPhone 17 Pro", target: "tv" }); - - expect(await requestedSpecs(rig, hex(1))).toEqual([{ class: "phone", platform: "ios" }]); - expect(await requestedSpecs(rig, hex(2))).toEqual([{ class: "tv", platform: "ios" }]); - expect(await requestedSpecs(rig, hex(3))).toEqual([{ class: "desktop", platform: "ios" }]); - expect(await requestedSpecs(rig, hex(4))).toEqual([ - { model: "iPhone 17 Pro", platform: "ios" }, - ]); - expect(await requestedSpecs(rig, hex(5))).toEqual([ - { model: "iPhone 17 Pro", platform: "ios" }, - ]); - }); - - it("an allocate whose `ttlMs` is above `lease.maxTtlMs` answers `refused by simlock:` with simlock's `BAD_REQUEST` message and leaves no simlock lease; one with `ttlMs: null` gets `lease.defaultTtlMs`", async () => { - const rig = await setUp(); - expect((await envelopeOf(await allocate(rig, 1, { ttlMs: null }))).result).toMatchObject({ - ok: true, - }); - expect((await leases(rig)).map((lease) => lease.ttlMs)).toEqual([defaultTtlMs]); - - const over = maxTtlMs + 1; - const expected = await simlockMessage(rig, { leaseId: hex(9), platform: "ios", ttlMs: over }); - const answer = await envelopeOf(await allocate(rig, 2, { ttlMs: over })); - - expect(answer.error?.message).toBe(`refused by simlock: ${expected}`); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); }); - it('allocate `--device "My test phone"` answers `refused by simlock:` with simlock\'s message, and leaves no simlock lease', async () => { - const rig = await setUp(); - expect( - (await envelopeOf(await allocate(rig, 1, { device: "iPhone 17 Pro" }))).result, - ).toMatchObject({ ok: true }); - const expected = await simlockMessage(rig, { - leaseId: hex(9), - model: "My test phone", - platform: "ios", - }); + it("`config.get` returns `capabilities.agent-device.leaseTtlCapMs` when it is set, and no `agentDevice` key", async () => { + const rig = await setUp({ leaseTtlCapMs: 10_000 }); - const answer = await envelopeOf(await allocate(rig, 2, { device: "My test phone" })); - - expect(answer.error).toMatchObject(refusal(`refused by simlock: ${expected}`)); - expect(answer.error?.data).toMatchObject({ - details: { reason: "REFUSED_BY_SIMLOCK" }, - retriable: false, - }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - }); - - describe("each input refused before simlock", () => { - const tooOld = "refused: this agent-device is too old for simlock; upgrade it"; - const ttlRefusal = "refused: ttlMs must be a positive number of milliseconds"; - const plainOnly = "refused: simlock leases only plain devices"; - const lifecycle = "refused: send lease calls as lease RPCs"; - const cases: readonly { - readonly title: string; - readonly message: string; - readonly method?: string; - readonly params: (n: number) => Record; - }[] = [ - { - message: "refused: only iOS and Android", - params: (n) => allocateParams(n, { platform: "macos" }), - title: "--platform macos", - }, - { - message: "refused: simlock picks the device", - params: (n) => allocateParams(n, { udid: "ABC" }), - title: "--udid", - }, - { - message: "refused: simlock picks the device", - params: (n) => allocateParams(n, { serial: "emulator-5554" }), - title: "--serial", - }, - { - message: "refused: name --platform ios or android", - params: (n) => allocateParams(n, { platform: undefined }), - title: "no platform", - }, - { - message: tooOld, - params: (n) => allocateParams(n, { leaseId: undefined }), - title: "no lease ID", - }, - { - message: tooOld, - params: (n) => allocateParams(n, { leaseId: hex(n).toUpperCase().replace(/[0-9]/g, "A") }), - title: "a lease ID that is not 32 lowercase hex", - }, - { - message: ttlRefusal, - params: (n) => allocateParams(n, { ttlMs: "abc" }), - title: 'a ttlMs of "abc"', - }, - { - message: ttlRefusal, - params: (n) => allocateParams(n, { ttlMs: -1 }), - title: "a ttlMs of -1", - }, - { - message: ttlRefusal, - params: (n) => allocateParams(n, { ttlMs: 0 }), - title: "a ttlMs of 0", - }, - { - message: plainOnly, - params: (n) => allocateParams(n, { backend: "cloud" }), - title: "backend", - }, - { - message: plainOnly, - params: (n) => allocateParams(n, { backend: "android-instance" }), - title: "the Android backend on an iOS allocate", - }, - { - message: plainOnly, - params: (n) => allocateParams(n, { providerName: "x" }), - title: "a provider* key", - }, - { - message: plainOnly, - params: (n) => allocateParams(n, { leaseProvider: "aws" }), - title: 'leaseProvider: "aws"', - }, - ...["agent_device.command", "agent-device.command"].flatMap((method) => - ["lease_allocate", "lease_heartbeat", "lease_release"].map((name) => ({ - message: lifecycle, - method, - params: (n: number) => ({ command: name, meta: { leaseId: hex(n) } }), - title: `${method} carrying ${name}`, - })), - ), - ]; - - it.each(cases)( - "$title answers its message, simlock stores no lease request, and the agent-device daemon receives nothing", - async ({ message, method, params }) => { - const rig = await setUp(); - const accepted = await allocate(rig, 1); - expect((await envelopeOf(accepted)).result).toMatchObject({ ok: true }); - const heldBefore = (await leases(rig)).map((lease) => lease.id); - const receivedBefore = rig.fake.requests.length; - - const answer = await rpc(rig, method ?? "agent_device.lease.allocate", params(2)); - - expect((await envelopeOf(answer)).error).toMatchObject(refusal(message)); - expect(await requestedSpecs(rig, hex(2))).toEqual([]); - expect(await waiting(rig)).toEqual([]); - expect((await leases(rig)).map((lease) => lease.id)).toEqual(heldBefore); - expect(rig.fake.requests).toHaveLength(receivedBefore); - }, - ); - }); - - it("an allocate in the shape connect proxy sends, with backend ios-instance, is granted and keeps the backend toward the agent-device daemon", async () => { - const rig = await setUp(); - - const answer = await envelopeOf( - await allocate(rig, 1, { - backend: "ios-instance", - runId: "proxy-1", - session: "adc-1", - tenantId: "proxy", - }), - ); - - expect(answer.result).toMatchObject({ ok: true }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - expect(allocatesReceived(rig, 1).map((received) => received.params?.["backend"])).toEqual([ - "ios-instance", - ]); - }); - - it("allocate and release sent as `agent-device.lease.allocate` and `agent-device.lease.release` act exactly as the `agent_device.lease.*` spelling", async () => { - const rig = await setUp(); - - const allocated = await rpc(rig, "agent-device.lease.allocate", allocateParams(1)); - - expect((await envelopeOf(allocated)).result).toMatchObject({ ok: true }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - expect(rig.fake.leases.has(hex(1))).toBe(true); - - const released = await rpc(rig, "agent-device.lease.release", { ...scope, leaseId: hex(1) }); - - expect((await envelopeOf(released)).result).toMatchObject({ ok: true }); - expect(await leases(rig)).toEqual([]); - expect(rig.fake.leases.has(hex(1))).toBe(false); - }); - - it("after the grant, the allocate the agent-device daemon receives names the granted device's address by `udid` on iOS and by `serial` on Android (the emulator serial, not the AVD name), and carries no `device`, `deviceKey` or `target`; the CLI gets the daemon's reply with the same lease ID", async () => { - const ios = await setUp(); - const iosAnswer = await envelopeOf( - await allocate(ios, 1, { device: "iPhone 17 Pro", deviceKey: "saved", target: "mobile" }), - ); - const iosDevice = await grantedDevice(ios); - const iosParams = receivedAllocate(ios, 1); - - expect(iosAnswer.result).toMatchObject({ data: { lease: { leaseId: hex(1) } }, ok: true }); - expect(iosParams["udid"]).toBe(iosDevice.address); - expect(iosParams).not.toHaveProperty("serial"); - for (const dropped of ["device", "deviceKey", "target"]) { - expect(iosParams).not.toHaveProperty(dropped); - } - - const android = await setUp({ drivers: (clock) => [androidDriver(clock)] }); - const androidAnswer = await envelopeOf( - await allocate(android, 2, { device: "Pixel 8", platform: "android" }), - ); - const androidDevice = await grantedDevice(android); - const androidParams = receivedAllocate(android, 2); - - expect(androidAnswer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); - expect(androidParams["serial"]).toBe(androidDevice.address); - expect(androidParams["serial"]).not.toBe(androidDevice.driverDeviceId); - expect(androidParams).not.toHaveProperty("udid"); - }); - - it("with every matching device leased, allocate waits, and is granted and answered once that device is released and reclaimed", async () => { - const rig = await setUp({ maxRunning: 1 }); - expect((await envelopeOf(await allocate(rig, 1))).result).toMatchObject({ ok: true }); - - const second = track(allocate(rig, 2, {}, { token: secrets.two })); - await expect - .poll(async () => (await waiting(rig)).map((request) => request.requesterId)) - .toEqual([hex(2)]); - expect(second.done()).toBe(false); - - await release(rig, 1); - const answer = await envelopeOf(await second.promise); - - expect(answer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2)]); - }); - - it("a CLI that hangs up while its allocate waits, and one that hangs up before simlock admitted the request, leave simlock's queue, and neither request is granted when a device frees", async () => { - const rig = await setUp({ maxRunning: 1 }); - await granted(rig, 1); - const hangUp = new AbortController(); - const second = track(allocate(rig, 2, {}, { signal: hangUp.signal, token: secrets.two })); - await sleep(50); - expect(second.done()).toBe(false); - await expect.poll(async () => (await waiting(rig)).length).toBe(1); - - hangUp.abort(); - await second.promise.catch(() => undefined); - await expect.poll(async () => (await waiting(rig)).length).toBe(0); - await release(rig, 1); - await sleep(50); - - expect(await leases(rig)).toEqual([]); - expect(allocatesReceived(rig, 2)).toEqual([]); - - const slow = await setUp({ - drivers: (clock) => [ - new FakeDriver({ - ...phoneModels, - availableOsVersions: ["26.5"], - clock, - knownModels: ["iPhone 16"], - latencyMs: { listManaged: 30_000 }, - platform: "ios", - }), - ], - slowStart: true, - }); - const early = new AbortController(); - const parked = track( - fetch(`${slow.origin}/v1/agent-device/rpc`, { - body: rpcBody("agent_device.lease.allocate", allocateParams(3)), - headers: { authorization: `Bearer ${secrets.one}`, "content-type": "application/json" }, - method: "POST", - signal: early.signal, - }), - ); - await sleep(100); - expect(parked.done()).toBe(false); - early.abort(); - await parked.promise.catch(() => undefined); - await sleep(50); - slow.clock.advance(30_000); - const ready = { ...slow, daemon: await slow.started }; - await sleep(100); - - expect(await leases(ready)).toEqual([]); - expect(await requestedSpecs(ready, hex(3))).toEqual([]); - expect(allocatesReceived(ready, 3)).toEqual([]); - }); - - it("a CLI that hangs up after device work started for it gets no agent-device lease: the grant that lands later is released at once and never reaches the agent-device daemon; when that device work fails and the request goes back to the queue, it is never granted", async () => { - const slowDevice = (clock: FakeClock) => [ - new FakeDriver({ - ...phoneModels, - availableOsVersions: ["26.5"], - clock, - knownModels: ["iPhone 16"], - latencyMs: { makeReady: 10_000 }, - platform: "ios", - }), - ]; - const rig = await setUp({ drivers: slowDevice }); - const hangUp = new AbortController(); - const first = track(allocate(rig, 1, {}, { signal: hangUp.signal })); - await sleep(50); - expect(first.done()).toBe(false); - await expect - .poll(() => rig.driver.calls.some((call) => call.operation === "makeReady")) - .toBe(true); - - hangUp.abort(); - await first.promise.catch(() => undefined); - await advance(rig, 12_000); - await expect.poll(async () => (await leases(rig)).length).toBe(0); - - expect(allocatesReceived(rig, 1)).toEqual([]); - // why: the released device is deleted and the pool boots another on the fake clock, which the daemon's stop waits for. - await advance(rig, 12_000); - - const failing = await setUp({ drivers: slowDevice }); - failing.driver.failOn("makeReady", 1, new Error("boot failed")); - const abort = new AbortController(); - const second = track(allocate(failing, 2, {}, { signal: abort.signal })); - await sleep(50); - expect(second.done()).toBe(false); - await expect - .poll(() => failing.driver.calls.some((call) => call.operation === "makeReady")) - .toBe(true); - abort.abort(); - await second.promise.catch(() => undefined); - - await advance(failing, 15_000); - - await expect.poll(async () => (await waiting(failing)).length).toBe(0); - expect(await leases(failing)).toEqual([]); - expect(allocatesReceived(failing, 2)).toEqual([]); - }); - - it("`DELETE /v1/lease-requests/:id` for the front's waiting request answers the allocate `refused by simlock:` with simlock's message, and nothing is leased", async () => { - const rig = await setUp({ maxRunning: 1 }); - await granted(rig, 1, {}, { token: secrets.two }); - const second = track(allocate(rig, 2)); - await expect.poll(async () => (await waiting(rig)).length).toBe(1); - const [request] = await waiting(rig); - - const cancelled = await fetch(`${rig.origin}/v1/lease-requests/${request?.id}`, { - headers: { authorization: `Bearer ${secrets.one}` }, - method: "DELETE", - }); - const answer = await envelopeOf(await second.promise); - - expect(cancelled.status).toBe(204); - expect(answer.error?.message).toMatch(/^refused by simlock: .+/); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - expect(await waiting(rig)).toEqual([]); - }); - - it("stopping the daemon while an allocate waits ends with no simlock lease, and the lease request it waited on is cancelled, not open, in state.db", async () => { - const rig = await setUp({ maxRunning: 1 }); - await granted(rig, 1, {}, { token: secrets.two }); - const second = track(allocate(rig, 2)); - await expect.poll(async () => (await waiting(rig)).length).toBe(1); - - await rig.daemon.stop("test"); - await second.promise.catch(() => undefined); - - const database = await new NodeSqliteOpener().open(rig.stateDbPath, { readOnly: true }); - try { - const leaseIds = database.prepare("SELECT id FROM leases").all(); - const requests = database.prepare("SELECT lease_id, state FROM lease_requests").all(); - expect(second.done()).toBe(true); - expect(leaseIds).toEqual([{ id: hex(1) }]); - expect(requests.filter((row) => row.lease_id === hex(2))).toEqual([ - expect.objectContaining({ state: "cancelled" }), - ]); - expect(requests.some((row) => row.state === "open")).toBe(false); - } finally { - database.close(); - } - }); - - it("an allocate still waiting 330 s answers `timed out waiting for a device; nothing was leased`, and lease request failures carry their retry classification", async () => { - const rig = await setUp({ maxRunning: 1 }); - await granted(rig, 1, { ttlMs: maxTtlMs }, { token: secrets.two }); - const second = track(allocate(rig, 2)); - await expect.poll(async () => (await waiting(rig)).length).toBe(1); - - await advance(rig, 329_000); - expect(second.done()).toBe(false); - await advance(rig, 1_000); - expect(second.done()).toBe(true); - const answer = await envelopeOf(await second.promise); - - expect(answer.error).toMatchObject(refusal(queueTimeoutMessage, "COMMAND_FAILED")); - expect(answer.error?.data).toMatchObject({ - details: { reason: "NO_DEVICE_IN_TIME" }, - retriable: true, - }); - expect(await waiting(rig)).toEqual([]); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - - const tap = tapDispatch(rig); - for (const [code, reason] of [ - ["NO_CAPACITY", "SIMLOCK_UNAVAILABLE"], - ["DAEMON_STOPPING", "SIMLOCK_UNAVAILABLE"], - ["DAEMON_CONNECTION_LOST", "SIMLOCK_UNAVAILABLE"], - ["WORKER_UNREACHABLE", "WORKER_UNREACHABLE"], - ["INVALID_ARGS", "REFUSED_BY_SIMLOCK"], - ] as const) { - tap.rejectNext("lease.request", new DispatchError(code, `failure ${code}`)); - const response = await allocate(rig, 3); - expect(response.status).toBe(400); - expect((await envelopeOf(response)).error).toMatchObject({ - message: `refused by simlock: failure ${code}`, - data: { code: "INVALID_ARGS", details: { reason }, retriable: code !== "INVALID_ARGS" }, - }); - } - }); - - it("an agent-device allocate that answers an error, and a grant that names no device address, release the simlock lease, and the CLI gets the error", async () => { - const rig: Rig = await setUp(); - rig.fake.failNextAllocate({ - code: "NO_DEVICE", - message: "no such device", - details: { reason: "AGENT_DEVICE_REASON" }, - }); - - const answer = await envelopeOf(await allocate(rig, 1)); - - expect(answer.error).toEqual({ - code: -32000, - data: { - code: "NO_DEVICE", - message: "no such device", - details: { reason: "AGENT_DEVICE_REASON" }, - }, - message: "no such device", - }); - await expect.poll(async () => (await leases(rig)).length).toBe(0); - - class NoAddressDriver extends FakeDriver { - override async makeReady(...arguments_: Parameters) { - const ready = await super.makeReady(...arguments_); - return { ...ready, address: undefined as unknown as string }; - } - } - const bare = await setUp({ - drivers: (clock) => [ - new NoAddressDriver({ - ...phoneModels, - availableOsVersions: ["26.5"], - clock, - knownModels: ["iPhone 16"], - platform: "ios", - }), - ], - }); - - const noAddress = await envelopeOf(await allocate(bare, 2)); - - expect(noAddress.error?.data).toMatchObject({ - code: "COMMAND_FAILED", - details: { reason: "SIMLOCK_FAILED" }, - retriable: false, - }); - await expect.poll(async () => (await leases(bare)).length).toBe(0); - expect(allocatesReceived(bare, 2)).toEqual([]); - }); - - it("an agent-device allocate with no usable reply (an unreachable daemon, a non-JSON body) releases the simlock lease and answers `COMMAND_FAILED`; when agent-device answers the follow-up release with a scope error, the simlock lease stays until its TTL, and when it answers `LEASE_NOT_FOUND`, the simlock lease is released", async () => { - const rig: Rig = await setUp(); - expect((await envelopeOf(await allocate(rig, 1))).result).toMatchObject({ ok: true }); - rig.fake.setUnreachable(true); - const unreachable = await envelopeOf(await allocate(rig, 2)); - rig.fake.setUnreachable(false); - - expect(unreachable.error?.data?.code).toBe("COMMAND_FAILED"); - await expect.poll(async () => (await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - - rig.fake.answerNextAllocateWith("bad gateway"); - const notJson = await envelopeOf(await allocate(rig, 3)); - - expect(notJson.error?.data?.code).toBe("COMMAND_FAILED"); - await expect.poll(async () => (await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - - rig.fake.answerNextAllocateWith("bad gateway"); - rig.fake.failNextRelease({ code: "UNAUTHORIZED", message: "lease scope mismatch" }); - const kept = await envelopeOf(await allocate(rig, 4, { ttlMs: 60_000 })); - - expect(kept.error?.data?.code).toBe("COMMAND_FAILED"); - expect((await leases(rig)).map((lease) => lease.id)).toContain(hex(4)); - await advance(rig, 61_000); - await expect - .poll(async () => (await leases(rig)).map((lease) => lease.id)) - .not.toContain(hex(4)); - - rig.fake.answerNextAllocateWith("bad gateway"); - rig.fake.failNextRelease({ code: "LEASE_NOT_FOUND", message: "no such lease" }); - const gone = await envelopeOf(await allocate(rig, 5, { ttlMs: 60_000 })); - - expect(gone.error?.data?.code).toBe("COMMAND_FAILED"); - await expect - .poll(async () => (await leases(rig)).map((lease) => lease.id)) - .not.toContain(hex(5)); - }); - - it("the follow-up release after an allocate with no usable reply carries the allocate's runId, leaseProvider and clientId and not its deviceKey", async () => { - const rig: Rig = await setUp(); - rig.fake.answerNextAllocateWith("bad gateway"); - - const answer = await envelopeOf(await allocate(rig, 1, { deviceKey: "saved-device" })); - - expect(answer.error?.data?.code).toBe("COMMAND_FAILED"); - const params = rig.fake.rpcRequests("agent_device.lease.release")[0]?.params; - expect(params).toMatchObject({ ...scope, leaseId: hex(1) }); - expect(params).not.toHaveProperty("deviceKey"); - }); - - it("a lease whose allocate reply names device key `ios:mobile:` keeps it: a command naming that lease with that `deviceKey` reaches agent-device", async () => { - const rig: Rig = await setUp(); - const key = "ios:mobile:00008110-001A2B3C4D5E"; - rig.fake.nameDeviceKeyInNextReply(key); - await granted(rig, 1); - - await forwarded(rig, () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: key } })); - }); - - it("an allocate reply whose lease `deviceKey` is over 128 characters still succeeds and keeps the lease, with the device key left unset: no release is sent and a command naming the lease with a `deviceKey` is refused", async () => { - const rig: Rig = await setUp(); - rig.fake.nameDeviceKeyInNextReply("x".repeat(129)); - - const answer = await envelopeOf(await allocate(rig, 1)); - - expect(answer.error).toBeUndefined(); - expect(rig.fake.rpcRequests("agent_device.lease.release")).toHaveLength(0); - expect((await leases(rig)).map((lease) => lease.id)).toContain(hex(1)); - await refused( - rig, - () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: "x".repeat(129) } }), - simlockPicks, - ); - }); - - it("a CLI that hangs up while agent-device makes its lease leaves neither lease: the agent-device lease is released with the reply's scope, then the simlock lease; when that release fails, the simlock lease stays until its TTL", async () => { - const rig: Rig = await setUp(); - rig.fake.holdAllocate(); - rig.fake.nameDeviceKeyInNextReply("device-key-1"); - const hangUp = new AbortController(); - const first = track(allocate(rig, 1, {}, { signal: hangUp.signal })); - await sleep(50); - expect(first.done()).toBe(false); - await expect.poll(() => allocatesReceived(rig, 1).length).toBe(1); - - hangUp.abort(); - await first.promise.catch(() => undefined); - await sleep(50); - rig.fake.letAllocateGo(); - - await expect.poll(async () => (await leases(rig)).length).toBe(0); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expect(rig.fake.rpcRequests("agent_device.lease.release")[0]?.params).toMatchObject({ - ...scope, - deviceKey: "device-key-1", - }); - - rig.fake.holdAllocate(); - rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "release failed" }); - const abort = new AbortController(); - const second = track(allocate(rig, 2, { ttlMs: 60_000 }, { signal: abort.signal })); - await sleep(50); - expect(second.done()).toBe(false); - await expect.poll(() => allocatesReceived(rig, 2).length).toBe(1); - abort.abort(); - await second.promise.catch(() => undefined); - await sleep(50); - rig.fake.letAllocateGo(); - await expect.poll(() => rig.fake.rpcRequests("agent_device.lease.release").length).toBe(2); - - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(2)]); - await advance(rig, 61_000); - await expect.poll(async () => (await leases(rig)).length).toBe(0); - }); - - it("release goes to the agent-device daemon first, then frees the simlock lease and the device; a release after `close`, which already freed the simlock lease, reaches the daemon with no lease held and is answered `released: false`", async () => { - const rig = await setUp(); - let heldAtDaemonRelease: readonly string[] = []; - rig.fake.onRelease = async () => { - heldAtDaemonRelease = (await leases(rig)).map((lease) => lease.id); - }; - await granted(rig, 1); - - const released = await envelopeOf(await release(rig, 1)); - - expect(released.result).toMatchObject({ ok: true, released: true }); - expect(heldAtDaemonRelease).toEqual([hex(1)]); - expect(await leases(rig)).toEqual([]); - await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); - - await granted(rig, 2); - await command(rig, "close", 2); - await expect.poll(async () => leases(rig)).toEqual([]); - const afterClose = await envelopeOf(await release(rig, 2)); - - expect(afterClose.result).toMatchObject({ ok: true, released: false }); - expect(heldAtDaemonRelease).toEqual([]); - expect(await leases(rig)).toEqual([]); - await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); - }); - - it("a runner allocates with retainOnClose, closes, and its lease stays until release", async () => { - const cases = [ - (rig: Rig) => command(rig, "close", 1), - (rig: Rig) => command(rig, "close", 1, {}, { positionals: ["com.example.app"] }), - (rig: Rig) => batch(rig, 1, [{ command: "snapshot" }, { command: "close" }]), - ]; - for (const close of cases) { - const rig: Rig = await setUp(); - expect( - (await envelopeOf(await allocate(rig, 1, { retainOnClose: true }))).result, - ).toMatchObject({ ok: true, data: { lease: { retainOnClose: true } } }); - expect((await envelopeOf(await close(rig))).result).toMatchObject({ ok: true }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); - expect((await envelopeOf(await command(rig, "snapshot", 1))).result).toMatchObject({ - ok: true, - }); - expect((await envelopeOf(await release(rig, 1))).result).toMatchObject({ - ok: true, - released: true, - }); - expect(await leases(rig)).toEqual([]); - await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); - } - }); - - const ended = [{ leaseId: hex(1), outcome: "released" }]; - it.each([ - [ - "a `close` that succeeds, in `data`", - { result: { data: { endedLeases: ended } } }, - (rig: Rig) => command(rig, "close", 1), - ], - [ - "a `close` that fails, in `error.details`", - { error: { data: { details: { endedLeases: ended } } } }, - (rig: Rig) => { - rig.fake.failNextCommand({ code: "COMMAND_FAILED", message: "close failed" }); - return command(rig, "close", 1); - }, - ], - [ - "a `batch` with a close step, in `data` beside `results`", - { result: { data: { endedLeases: ended, results: [] } } }, - (rig: Rig) => batch(rig, 1, [{ command: "snapshot" }, { command: "close" }]), - ], - ] as const)( - "%s: a command that ends the agent-device lease and names it in `endedLeases` releases the simlock lease and frees the device though the CLI sends no release, and a later heartbeat answers `LEASE_NOT_FOUND`", - async (_case, reply, send) => { - const rig = await setUp(); - await granted(rig, 1); - - expect(await envelopeOf(await send(rig))).toMatchObject(reply); - await expect.poll(async () => listed(rig)).toEqual([]); - await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); - expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); - }, - ); - - // why: agent-device 0.21.28 streams a reply when the CLI runs without --json; the JSON-RPC answer is the `response` of a `type: "response"` line, not the line itself. - it("a close streamed in agent-device's own line shape releases the simlock lease", async () => { - const rig = await setUp(); - await granted(rig, 1); - const chunks = [ - `${JSON.stringify({ event: { kind: "command", message: "Closing" }, type: "progress" })}\n`, - `${JSON.stringify({ - response: { - id: "x", - jsonrpc: "2.0", - result: { data: { endedLeases: ended, session: "t:adc-1" }, ok: true }, - }, - type: "response", - })}\n`, - ]; - rig.fake.streamNextCommand(chunks); - - const streamed = await command(rig, "close", 1); - - expect(await streamed.text()).toBe(chunks.join("")); - await expect.poll(async () => listed(rig), { timeout: 3_000 }).toEqual([]); - await expect.poll(async () => liveStates(rig)).toEqual(["ready"]); - }); - - it("a release the agent-device daemon fails keeps the simlock lease and answers the daemon's error", async () => { - const rig = await setUp(); - await granted(rig, 1); - rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon could not release" }); - - const answer = await envelopeOf(await release(rig, 1)); - - expect(answer.error).toMatchObject({ - data: { code: "COMMAND_FAILED" }, - message: "daemon could not release", - }); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - }); - - it("a release from a second token naming the first token's lease ID leaves the first token's simlock lease in place", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1); - - const answer = await envelopeOf(await release(rig, 1, { token: secrets.two })); - - expect(answer.error).toBeDefined(); - expect(rig.fake.rpcRequests("agent_device.lease.release")).toHaveLength(1); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - expect(rig.fake.leases.has(hex(1))).toBe(true); - - rig.fake.letAnyTenantRelease(); - const crossed = await envelopeOf(await release(rig, 1, { token: secrets.two })); - - expect(crossed.result).toMatchObject({ ok: true }); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expect((await leases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - }); - - it("a command's reply, a streamed progress body included, reaches the CLI unchanged, and a CLI that hangs up during it aborts the request the agent-device daemon was answering", async () => { - const rig = await setUp(); - const chunks = [ - '{"type":"progress","event":{"kind":"command","message":"one"}}\n', - '{"type":"progress","event":{"kind":"command","message":"two"}}\n', - '{"type":"response","response":{"jsonrpc":"2.0","id":"x","result":{"ok":true,"data":{}}}}\n', - ]; - rig.fake.streamNextCommand(chunks); - - const streamed = await command( - rig, - "install", - undefined, - {}, - { meta: { uploadedArtifactId: "art-1" } }, - ); - - expect(await streamed.text()).toBe(chunks.join("")); - - const plain = await envelopeOf(await command(rig, "snapshot", undefined)); - expect(plain.result).toMatchObject({ command: "snapshot", ok: true }); - - rig.fake.holdCommand(); - const hangUp = new AbortController(); - const held = track(command(rig, "snapshot", undefined, { signal: hangUp.signal })); - await sleep(50); - expect(held.done()).toBe(false); - await expect.poll(() => rig.fake.rpcRequests("agent_device.command").length).toBe(3); - hangUp.abort(); - await held.promise.catch(() => undefined); - - await expect.poll(() => rig.fake.abortedCommands).toEqual(["snapshot"]); - }); - - it("an upload preflight answers a ticket URL under `/v1/agent-device/upload/direct/` on simlock's own origin, carrying the agent's own simlock token, and a `PUT` to it reaches the agent-device daemon", async () => { - const rig = await setUp(); - - const preflight = await send(rig, "/upload/preflight", { body: "{}" }); - expect(preflight.status).toBe(200); - const ticket = ( - (await preflight.json()) as { upload: { url: string; headers: Record } } - ).upload; - - expect(ticket.url.startsWith(`${rig.origin}/v1/agent-device/upload/direct/`)).toBe(true); - const carried = Object.entries(ticket.headers).find( - ([name]) => name.toLowerCase() === "authorization", - ); - expect(carried?.[1]).toBe(`Bearer ${secrets.one}`); - - const put = await fetch(ticket.url, { - body: "artifact bytes", - headers: ticket.headers, - method: "PUT", - }); - - expect(put.status).toBe(200); - expect(rig.fake.uploads).toEqual(["artifact bytes"]); - }); - - it("on a worker, an upload preflight and an artifact download follow the `x-agent-device-lease-id` header to the daemon of the lease it names, go to the simulator daemon with no header, and a header naming another token's lease is refused 403 `refused: lease is not one of yours`, and a malformed header is refused `the x-agent-device-lease-id header is not a lease id`", async () => { - const rig = await setUpWithPhone(); - await granted(rig, 1, { physical: true, platform: "android" }); - await granted(rig, 2, { platform: "android" }); - const named = (n: number) => ({ "x-agent-device-lease-id": hex(n) }); - const malformed = { "x-agent-device-lease-id": "NOT-A-LEASE-ID".repeat(4) }; - const reaching = (fake: FakeAgentDevice, path: string) => - fake.requests.filter((request) => request.path === path).length; - const routes = [ - ["/upload/preflight", { body: "{}" }], - ["/artifacts/a1", {}], - ] as const; - const cases = [ - [named(1), secrets.one, "phone"], - [named(2), secrets.one, "simulators"], - [{}, secrets.one, "simulators"], - [named(1), secrets.two, "forbidden"], - [malformed, secrets.one, "refused"], - ] as const; - - for (const [path, body] of routes) { - for (const [headers, token, reaches] of cases) { - const [phoneBefore, simulatorsBefore] = [ - reaching(rig.phone, path), - reaching(rig.fake, path), - ]; - - const answer = await send(rig, path, { ...body, headers, token }); - - expect([ - reaching(rig.phone, path) - phoneBefore, - reaching(rig.fake, path) - simulatorsBefore, - ]).toEqual([reaches === "phone" ? 1 : 0, reaches === "simulators" ? 1 : 0]); - if (reaches === "forbidden") { - expect(answer.status).toBe(403); - expect(await answer.text()).toContain(`refused: lease ${hex(1)} is not one of yours`); - } - if (reaches === "refused") { - expect(answer.status).toBe(400); - expect(await answer.text()).toContain( - "the x-agent-device-lease-id header is not a lease id", - ); - } - } - } - }); - - it("an `/rpc` body over 1 MiB, one that is not JSON, a JSON array, and an object with no `method` are each refused with their message and `id: null`, also while agent-device is unavailable, and neither simlock nor the agent-device daemon sees them", async () => { - const rig = await setUp(); - expect((await command(rig, "snapshot", undefined)).status).toBe(200); - const oneMiB = 1024 * 1024; - const bodies: readonly [string, string][] = [ - [ - JSON.stringify({ - id: 1, - jsonrpc: "2.0", - method: "agent_device.command", - params: { pad: "x".repeat(oneMiB) }, - }), - "refused: request body over 1 MiB", - ], - ["{ not json", "refused: send one JSON-RPC request"], - ["[]", "refused: send one JSON-RPC request"], - [JSON.stringify({ id: 3, jsonrpc: "2.0" }), "refused: send one JSON-RPC request"], - ]; - - for (const unavailable of [false, true]) { - rig.fake.setUnavailable(unavailable ? "starting" : undefined); - const receivedBefore = rig.fake.requests.length; - for (const [body, message] of bodies) { - const answer = await envelopeOf(await send(rig, "/rpc", { body })); - expect(answer).toMatchObject({ error: refusal(message), id: null, jsonrpc: "2.0" }); - } - expect(rig.fake.requests).toHaveLength(receivedBefore); - } - expect(await waiting(rig)).toEqual([]); - expect(await leases(rig)).toEqual([]); - }); - - it("a heartbeat reaches the agent-device daemon before simlock, carrying `ttlMs` equal to the ceiling, and once it answers, the simlock lease's deadline is now plus the CLI's `ttlMs`", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - const allocated = await deadlineOf(rig, 1); - let deadlineAtDaemon: number | undefined; - rig.fake.onHeartbeat = async () => { - deadlineAtDaemon = await deadlineOf(rig, 1); - }; - await advance(rig, 10_000); - - const answer = await envelopeOf( - await heartbeat(rig, 1, { ttlMs: 120_000 }, { token: secrets.one }), - ); - - expect(answer.result).toMatchObject({ ok: true }); - expect(deadlineAtDaemon).toBe(allocated); - expect(rig.fake.rpcRequests("agent_device.lease.heartbeat")[0]?.params).toMatchObject({ - ...scope, - leaseId: hex(1), - ttlMs: rig.fake.leaseMaxTtlMs, - }); - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 120_000); - }); - - it.each([ - { cap: undefined, expected: 300_000, row: "300000 with no cap", sent: 300_000 }, - { cap: 10_000, expected: 10_000, row: "300000 with cap 10000", sent: 300_000 }, - { cap: 600_000, expected: 300_000, row: "300000 with cap 600000", sent: 300_000 }, - { cap: 10_000, expected: 10_000, row: "none with cap 10000", sent: undefined }, - { cap: undefined, expected: defaultTtlMs, row: "none with no cap", sent: undefined }, - { cap: undefined, expected: 5_000, row: "1 with no cap", sent: 1 }, - ])( - "a CLI `ttlMs` of $row gives the simlock lease, its heartbeat renewal and its renewal interval a TTL of $expected ms", - async ({ cap, expected, sent }) => { - const rig = await setUp(cap === undefined ? {} : { leaseTtlCapMs: cap }); - const ttl = expected as number; - await granted(rig, 1, { ttlMs: sent }); - expect((await leases(rig))[0]?.ttlMs).toBe(ttl); - await advance(rig, 1_000); - - const answer = await envelopeOf( - await heartbeat(rig, 1, sent === undefined ? {} : { ttlMs: sent }), - ); - - expect(answer.result).toMatchObject({ ok: true }); - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + ttl); - const settled = await deadlineOf(rig, 1); - await runningCommand(rig, 1); - await advance(rig, Math.floor(ttl / 3) - 700, 500); - expect(await deadlineOf(rig, 1)).toBe(settled); - await advance(rig, 1_400, 500); - const renewed = (await deadlineOf(rig, 1)) as number; - expect(renewed).toBeGreaterThan(settled as number); - expect(renewed - (settled as number)).toBeLessThanOrEqual(ttl / 3 + 700); - }, - 60_000, - ); - - it("a heartbeat with a longer `ttlMs` than the allocate's moves the deadline to now plus that TTL, and a following heartbeat with no `ttlMs` renews with it too", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - await advance(rig, 5_000); - - await heartbeat(rig, 1, { ttlMs: 300_000 }); - - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 300_000); - await advance(rig, 20_000); - - await heartbeat(rig, 1); - - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 300_000); - expect((await leases(rig))[0]?.ttlMs).toBe(300_000); - }); - - it("after the agent's own `simlock lease renew` with a longer TTL, a heartbeat and a running command's renewals leave that later deadline in place", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - await rig.daemon.dispatch( - "lease.renew", - { leaseId: hex(1), ttlMs: 900_000 }, - sessionFor(principals.one), - ); - const later = await deadlineOf(rig, 1); - expect(later).toBe(rig.clock.now() + 900_000); - await advance(rig, 5_000); - - await heartbeat(rig, 1, { ttlMs: 120_000 }); - - expect(await deadlineOf(rig, 1)).toBe(later); - const running = await runningCommand(rig, 1); - await advance(rig, 130_000, 5_000); - expect(await deadlineOf(rig, 1)).toBe(later); - rig.fake.releaseCommand(); - await running.promise; - await expect.poll(() => running.done()).toBe(true); - expect(await deadlineOf(rig, 1)).toBe(later); - }); - - it("a heartbeat the agent-device daemon cannot answer returns that error and leaves the simlock lease's deadline where it was", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - const before = await deadlineOf(rig, 1); - await advance(rig, 10_000); - rig.fake.failNextHeartbeat({ code: "COMMAND_FAILED", message: "daemon is busy" }); - - const failed = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); - - expect(failed.error).toMatchObject({ - data: { code: "COMMAND_FAILED" }, - message: "daemon is busy", - }); - expect(await deadlineOf(rig, 1)).toBe(before); - - rig.fake.setUnreachable(true); - const unreachable = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); - - expect(unreachable.error).toBeDefined(); - expect(unreachable.result).toBeUndefined(); - expect(await deadlineOf(rig, 1)).toBe(before); - - rig.fake.setUnreachable(false); - await heartbeat(rig, 1, { ttlMs: 60_000 }); - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 60_000); - }); - - it("with no cap set, a heartbeat whose `ttlMs` is above `lease.maxTtlMs` answers `COMMAND_FAILED` with simlock's `BAD_REQUEST` message and leaves the deadline where it was, and a following heartbeat with a valid `ttlMs` renews the lease", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - const before = await deadlineOf(rig, 1); - await advance(rig, 10_000); - const message = await renewMessage(rig, hex(1), maxTtlMs + 1); - - const refused = await envelopeOf(await heartbeat(rig, 1, { ttlMs: maxTtlMs + 1 })); - - expect(refused.error).toMatchObject({ data: { code: "COMMAND_FAILED" }, message }); - expect(await deadlineOf(rig, 1)).toBe(before); - - const renewed = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); - - expect(renewed.result).toMatchObject({ ok: true }); - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 120_000); - expect((await leases(rig))[0]?.ttlMs).toBe(120_000); - }); - - it("a heartbeat for a lease the agent released through the front answers `LEASE_NOT_FOUND`, and a following allocate gets a new lease", async () => { - const rig = await setUp(); - await granted(rig, 1); - await release(rig, 1); - - const answer = await envelopeOf(await heartbeat(rig, 1)); - - expectLeaseGone(answer, "LEASE_NOT_FOUND"); - expect(await listed(rig)).toEqual([]); - - await granted(rig, 2); - - expect(await listed(rig)).toEqual([hex(2)]); - }); - - it("one TTL after the agent's release, a heartbeat for that lease answers `LEASE_ENDED_BY_HOST`", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - await release(rig, 1); - expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); - await advance(rig, 61_000, 5_000); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - }); - - it.each([ - [ - "`simlock release --all`", - 5_000, - async (rig: Rig) => { - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - }, - ], - [ - "expiry", - 5_000, - async (rig: Rig) => { - await advance(rig, 6_000); - }, - ], - [ - "the device is lost", - 600_000, - async (rig: Rig) => { - rig.driver.setManagedReality({ devices: [], processes: [] }); - await advance(rig, 120_000, 10_000); - }, - ], - ])( - "after %s, a heartbeat for the lease answers `LEASE_ENDED_BY_HOST` with the message and reason, and makes no new simlock lease", - async (_cause, ttl, end) => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: ttl }); - await end(rig); - await expect.poll(() => listed(rig)).toEqual([]); - - const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 5_000 })); - - expectEndedBySimlock(answer); - expect(await listed(rig)).toEqual([]); - }, - ); - - it.each([ - [ - "`LEASE_NOT_FOUND` with the simlock lease already released", - async (rig: Rig) => { - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - expect(await listed(rig)).toEqual([]); - }, - ], - [ - "`LEASE_EXPIRED`", - async (rig: Rig) => { - rig.fake.loseLease(hex(1), "LEASE_EXPIRED"); - expect(await listed(rig)).toEqual([hex(1)]); - }, - ], - [ - "`LEASE_REVOKED`", - async (rig: Rig) => { - rig.fake.loseLease(hex(1), "LEASE_REVOKED"); - expect(await listed(rig)).toEqual([hex(1)]); - }, - ], - ] as const)( - "a heartbeat the agent-device daemon answers with %s, for a lease that really ended, releases the simlock lease and answers `LEASE_ENDED_BY_HOST`", - async (_case, end) => { - const rig: Rig = await setUp(); - await granted(rig, 1); - await end(rig); - - const answer = await envelopeOf(await heartbeat(rig, 1)); - - expectEndedBySimlock(answer); - expect(await listed(rig)).toEqual([]); - }, - ); - - it("a heartbeat naming a lease ID the front map lacks, which simlock kept while agent-device still holds it: it is released with the heartbeat's scope first, then the simlock lease; when agent-device is unreachable, neither lease is touched; each answers as in the table", async () => { - const rig: Rig = await setUp(); - await keptWhileAgentDeviceHolds(rig, 2); - await keptWhileAgentDeviceHolds(rig, 3); - expect(await listed(rig)).toEqual([hex(2), hex(3)]); - - let heldAtDaemonRelease: readonly string[] = []; - rig.fake.onRelease = async () => { - heldAtDaemonRelease = await listed(rig); - }; - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 2))); - expect(heldAtDaemonRelease).toEqual([hex(2), hex(3)]); - expect(rig.fake.rpcRequests("agent_device.lease.release")[0]?.params).toMatchObject(scope); - expect(rig.fake.leases.has(hex(2))).toBe(false); - expect(await listed(rig)).toEqual([hex(3)]); - - const before = await deadlineOf(rig, 3); - await advance(rig, 10_000); - rig.fake.setUnreachable(true); - const unreachable = await envelopeOf(await heartbeat(rig, 3)); - rig.fake.setUnreachable(false); - - expect(unreachable.error).toBeDefined(); - expect(unreachable.error?.data?.code).not.toBe("UNAUTHORIZED"); - expect(await listed(rig)).toEqual([hex(3)]); - expect(await deadlineOf(rig, 3)).toBe(before); - expect(rig.fake.leases.has(hex(3))).toBe(true); - }); - - it("a heartbeat from a second token naming the first token's lease ID answers `LEASE_ENDED_BY_HOST`, and the first token's lease is neither released nor renewed", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - const before = await deadlineOf(rig, 1); - await advance(rig, 10_000); - - const answer = await envelopeOf( - await heartbeat(rig, 1, { ttlMs: 60_000 }, { token: secrets.two }), - ); - - expectEndedBySimlock(answer); - expect(await listed(rig)).toEqual([hex(1)]); - expect(await deadlineOf(rig, 1)).toBe(before); - expect(rig.fake.leases.has(hex(1))).toBe(true); - - const own = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); - - expect(own.result).toMatchObject({ ok: true }); - expect(await deadlineOf(rig, 1)).toBe(rig.clock.now() + 60_000); - }); - - it("a heartbeat naming a lease ID the front map lacks, when the agent-device daemon fails to release its lease, answers that error and leaves the simlock lease in place", async () => { - const rig: Rig = await setUp(); - await keptWhileAgentDeviceHolds(rig, 1); - rig.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon cannot release" }); - - const failed = await envelopeOf(await heartbeat(rig, 1)); - - expect(failed.error).toMatchObject({ - data: { code: "COMMAND_FAILED" }, - message: "daemon cannot release", - }); - expect(await listed(rig)).toEqual([hex(1)]); - }); - - it("a heartbeat for a lease the agent-device daemon lost, when simlock cannot release its lease, answers `COMMAND_FAILED` with simlock's message and marks nothing, and the next heartbeat releases it and answers `LEASE_ENDED_BY_HOST`", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1); - rig.fake.loseLease(hex(1), "LEASE_EXPIRED"); - const tap = tapDispatch(rig); - tap.rejectNext("lease.release"); - - const failed = await envelopeOf(await heartbeat(rig, 1)); - - expect(failed.error).toMatchObject({ - data: { code: "COMMAND_FAILED" }, - message: "simlock could not do that", - }); - expect(await listed(rig)).toEqual([hex(1)]); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - expect(await listed(rig)).toEqual([]); - }); - - it.each([0, -5, "soon", true])( - "a heartbeat with a `ttlMs` of %j is refused with `ttlMs must be a positive number of milliseconds`, and neither daemon is reached", - async (ttlMs) => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - const before = await deadlineOf(rig, 1); - const received = heartbeatsReceived(rig); - await advance(rig, 10_000); - - const refused = await envelopeOf(await heartbeat(rig, 1, { ttlMs })); - - expect(refused.error).toMatchObject( - refusal("refused: ttlMs must be a positive number of milliseconds"), - ); - expect(heartbeatsReceived(rig)).toBe(received); - expect(await deadlineOf(rig, 1)).toBe(before); - }, - ); - - it("after a heartbeat answered `LEASE_ENDED_BY_HOST` because simlock ended the lease, the next heartbeat answers it again and the agent-device daemon receives nothing for it", async () => { - const rig = await setUp(); - await granted(rig, 1); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - const received = rig.fake.requests.length; - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - - expect(rig.fake.requests).toHaveLength(received); - }); - - it("a lease released through the front while a command runs stays ended by the agent: a renewal that fires between the simlock release and the end of the agent's release, and the command's final renewal, mark nothing, and the next heartbeat still answers `LEASE_NOT_FOUND`", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - const running = await runningCommand(rig, 1); - const tap = tapDispatch(rig); - tap.afterNext("lease.release", async () => { - await advance(rig, 12_000, 4_000); - }); - - rig.fake.answerReleaseWhileCommandsRun(); - - const released = await envelopeOf(await release(rig, 1)); - rig.fake.releaseCommand(); - await running.promise; - - expect(released.result).toMatchObject({ ok: true, released: true }); - expect(await listed(rig)).toEqual([]); - expectLeaseGone(await envelopeOf(await heartbeat(rig, 1)), "LEASE_NOT_FOUND"); - }); - - it("after `simlock release --all`, the agent's release answers success, a heartbeat for the old ID then answers `LEASE_ENDED_BY_HOST` (ended by simlock, not `LEASE_NOT_FOUND`), and the next allocate gets a new lease; an entry found unlisted by another allocate's check answers its next heartbeat `LEASE_ENDED_BY_HOST` without reaching the agent-device daemon", async () => { - const rig = await setUp(); - await granted(rig, 1); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - - const released = await envelopeOf(await release(rig, 1)); - - expect(released.result).toMatchObject({ ok: true, released: false }); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - await granted(rig, 2); - expect(await listed(rig)).toEqual([hex(2)]); - - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - await advance(rig, 125_000, 25_000); - await granted(rig, 3); - const received = heartbeatsReceived(rig); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 2))); - expect(heartbeatsReceived(rig)).toBe(received); - }); - - it("after `simlock release --all` and the agent's release, a heartbeat answers `LEASE_ENDED_BY_HOST` without reaching the agent-device daemon, and a running command's renewals stop", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await runningCommand(rig, 1); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - rig.fake.answerReleaseWhileCommandsRun(); - await release(rig, 1); - const received = rig.fake.requests.length; - const tap = tapDispatch(rig); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - await advance(rig, 120_000, 10_000); - - expect(rig.fake.requests).toHaveLength(received); - rig.fake.releaseCommand(); - expect(tap.operations).not.toContain("lease.list"); - expect(tap.operations).not.toContain("lease.renew"); - }); - - it("a release for a lease simlock ended answers the agent-device daemon's reply as a success, and the agent-device lease is released", async () => { - const rig = await setUp(); - await granted(rig, 1); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - const tap = tapDispatch(rig); - - const answer = await envelopeOf(await release(rig, 1)); - - expect(answer.result).toMatchObject({ ok: true, released: false }); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expect(tap.operations).not.toContain("lease.release"); - }); - - it("while a command for the lease runs longer than its TTL, the simlock lease is renewed every TTL/3 and does not expire", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await runningCommand(rig, 1); - const tap = tapDispatch(rig); - const deadlines: (number | undefined)[] = []; - - for (let tick = 0; tick < 9; tick += 1) { - await advance(rig, 10_000, 5_000); - deadlines.push(await deadlineOf(rig, 1)); - } - - expect(deadlines.every((deadline) => deadline !== undefined)).toBe(true); - expect(deadlines.at(-1)).toBeGreaterThan(rig.clock.now()); - expect(new Set(deadlines).size).toBeGreaterThan(5); - const checks = tap.operations.filter((name) => name === "lease.list").length; - expect(checks).toBeGreaterThanOrEqual(8); - expect(checks).toBeLessThanOrEqual(9); - }); - - it("a command that runs on past `lease.maxTtlMs` stops counting as running: its renewals stop, and the lease is no longer kept", async () => { - const rig = await setUp(); - await granted(rig, 1); - const tap = tapDispatch(rig); - await runningCommand(rig, 1); - - await advance(rig, maxTtlMs + 400_000, 100_000); - const calls = frontCalls(tap).length; - await advance(rig, 1_000_000, 100_000); - - expect(calls).toBeGreaterThan(0); - expect(frontCalls(tap)).toHaveLength(calls); - expect(await listed(rig)).toEqual([]); - }, 60_000); - - it("when the command ends, the lease is renewed once more, and its deadline does not move again over the next TTL", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - const running = await runningCommand(rig, 1); - await advance(rig, 25_000, 5_000); - rig.fake.releaseCommand(); - await running.promise; - - await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); - const final = await deadlineOf(rig, 1); - await advance(rig, 29_000, 5_000); - - expect(await deadlineOf(rig, 1)).toBe(final); - }); - - it("a command agent-device answers with a 5xx reaches the CLI unchanged", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1); - rig.fake.failNextCommandWithStatus(500, "snapshot runner crashed"); - - const response = await command(rig, "snapshot", 1); - - expect(response.status).toBe(500); - expect(await response.text()).toBe("snapshot runner crashed"); - }); - - it("a command that fails, and one whose CLI hangs up, stop the renewals the same way", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await granted(rig, 2, { ttlMs: 30_000 }); - rig.fake.failNextCommand({ code: "COMMAND_FAILED", message: "the command failed" }); - const failing = await runningCommand(rig, 1); - await advance(rig, 15_000, 5_000); - rig.fake.releaseCommand(); - const failure = await envelopeOf(await failing.promise); - expect(failure.error).toMatchObject({ message: "the command failed" }); - await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); - const afterFailure = await deadlineOf(rig, 1); - - const hangUp = new AbortController(); - const hanging = await runningCommand(rig, 2, { signal: hangUp.signal }); - await advance(rig, 15_000, 5_000); - hangUp.abort(); - await hanging.promise.catch(() => undefined); - await expect.poll(() => deadlineOf(rig, 2)).toBe(rig.clock.now() + 30_000); - const afterHangUp = await deadlineOf(rig, 2); - - await advance(rig, 14_000, 2_000); - expect(await deadlineOf(rig, 1)).toBe(afterFailure); - expect(await deadlineOf(rig, 2)).toBe(afterHangUp); - }); - - it("with two overlapping commands on one lease, renewals go on until the second one ends", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - const firstHangUp = new AbortController(); - await runningCommand(rig, 1, { signal: firstHangUp.signal }); - const second = await runningCommand(rig, 1); - await advance(rig, 20_000, 5_000); - firstHangUp.abort(); - - await advance(rig, 40_000, 5_000); - - expect(await deadlineOf(rig, 1)).toBeGreaterThan(rig.clock.now()); - rig.fake.releaseCommand(); - await second.promise; - await expect.poll(() => deadlineOf(rig, 1)).toBe(rig.clock.now() + 30_000); - const final = await deadlineOf(rig, 1); - await advance(rig, 29_000, 5_000); - expect(await deadlineOf(rig, 1)).toBe(final); - }); - - it("a renewal that fails because simlock ended the lease mid-command stops the renewals, and the command's reply still reaches the CLI; a renewal refused for another reason is tried again at the next tick", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - const tap = tapDispatch(rig); - const running = await runningCommand(rig, 1); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - await advance(rig, 15_000, 5_000); - const calls = frontCalls(tap).length; - - await advance(rig, 60_000, 5_000); - - expect(frontCalls(tap)).toHaveLength(calls); - rig.fake.releaseCommand(); - const reply = await envelopeOf(await running.promise); - expect(reply.result).toMatchObject({ command: "snapshot", ok: true }); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - - await granted(rig, 2, { ttlMs: 30_000 }); - const started = await deadlineOf(rig, 2); - await runningCommand(rig, 2); - tap.rejectNext("lease.renew"); - await advance(rig, 11_000, 1_000); - expect(await deadlineOf(rig, 2)).toBe(started); - - await advance(rig, 10_000, 1_000); - - expect(await deadlineOf(rig, 2)).toBeGreaterThan(started as number); - }); - - it('an agent release whose `lease.release` simlock refuses with another error clears "release in progress": the simlock lease stays, and a later renewal that finds it gone marks it ended by simlock', async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - const tap = tapDispatch(rig); - tap.rejectNext("lease.release"); - - const refused = await envelopeOf(await release(rig, 1)); - - expect(refused.result).toMatchObject({ ok: true }); - expect(await listed(rig)).toEqual([hex(1)]); - await rig.daemon.dispatch("lease.release-all", {}, adminSession); - - const answer = await envelopeOf(await heartbeat(rig, 1)); - - expectEndedBySimlock(answer); - const received = heartbeatsReceived(rig); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - expect(heartbeatsReceived(rig)).toBe(received); - }); - - it("a heartbeat lowering the TTL from 300 s to 5 s while a command runs re-arms the renewals, and the lease does not expire before the command ends", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 300_000 }); - await runningCommand(rig, 1); - await advance(rig, 1_000); - - await heartbeat(rig, 1, { ttlMs: 5_000 }); - - await advance(rig, 360_000, 2_000); - expect(await listed(rig)).toEqual([hex(1)]); - expect(await deadlineOf(rig, 1)).toBeGreaterThan(rig.clock.now()); - }, 60_000); - - it("a command naming a lease ID the front map does not hold for its token, or the lease of an ended entry, starts no renewals", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await granted(rig, 2, { ttlMs: 30_000 }); - await release(rig, 2); - const tap = tapDispatch(rig); - await runningCommand(rig, 1, { token: secrets.two }); - await runningCommand(rig, 2); - - await advance(rig, 40_000, 5_000); - - expect(frontCalls(tap)).toEqual([]); - expect(await listed(rig)).toEqual([]); - }); - - it("a release through the front while a command runs stops that lease's renewals", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await runningCommand(rig, 1); - await advance(rig, 5_000, 5_000); - rig.fake.answerReleaseWhileCommandsRun(); - await release(rig, 1); - const tap = tapDispatch(rig); - - await advance(rig, 60_000, 5_000); - - expect(frontCalls(tap)).toEqual([]); - rig.fake.releaseCommand(); - }); - - it("a lease whose CLI stops sending heartbeats expires one TTL after its last heartbeat, and its device can be leased again", async () => { - const rig = await setUp({ maxRunning: 1 }); - await granted(rig, 1, { ttlMs: 30_000 }); - await advance(rig, 10_000, 5_000); - await heartbeat(rig, 1, { ttlMs: 30_000 }); - await advance(rig, 29_000, 5_000); - expect(await listed(rig)).toEqual([hex(1)]); - - await advance(rig, 3_000, 1_000); - - expect(await listed(rig)).toEqual([]); - await granted(rig, 2); - expect(await listed(rig)).toEqual([hex(2)]); - }); - - it("after the same restart (same `FakeDriver`, lease still listed), a release for a lease made before the restart answers the agent-device daemon's reply as a success and releases that simlock lease", async () => { - const first = await setUp(); - await granted(first, 1); - const rig = await first.restart(); - expect(await listed(rig)).toEqual([hex(1)]); - - const answer = await envelopeOf(await release(rig, 1)); - - expect(answer.result).toMatchObject({ ok: true, released: true }); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expect(await listed(rig)).toEqual([]); - }); - - it("a release with no map entry from a second token, naming the first token's lease ID, answers the agent-device daemon's reply and leaves the first token's simlock lease in place", async () => { - const first = await setUp(); - await granted(first, 1); - const rig: Rig = await first.restart(); - rig.fake.letAnyTenantRelease(); - - const answer = await envelopeOf(await release(rig, 1, { token: secrets.two })); - - expect(answer.result).toMatchObject({ ok: true, released: true }); - expect(rig.fake.leases.has(hex(1))).toBe(false); - expect(await listed(rig)).toEqual([hex(1)]); - }); - - it("stopping the daemon while a command runs renews nothing once the stop began: the lease's deadline, read after a restart on the same data directory and the same `FakeDriver` (the lease still listed), is the one it had before the stop", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 30_000 }); - await runningCommand(rig, 1); - await advance(rig, 12_000, 4_000); - const before = await deadlineOf(rig, 1); - const tap = tapDispatch(rig); - - const stopping = rig.daemon.stop("test"); - await advance(rig, 3_000, 1_000).catch(() => undefined); - await stopping; - expect(frontCalls(tap)).toEqual([]); - const restarted = await rig.restart(); - - expect(await listed(restarted)).toEqual([hex(1)]); - expect(await deadlineOf(restarted, 1)).toBe(before); - }); - - it("`config.get` returns `capabilities.agent-device.leaseTtlCapMs` when it is set, and no `agentDevice` key", async () => { - const rig = await setUp({ leaseTtlCapMs: 10_000 }); - - const config = (await rig.daemon.dispatch("config.get", {}, adminSession)) as { - readonly capabilities: Record; - }; + const config = (await rig.daemon.dispatch("config.get", {}, adminSession)) as { + readonly capabilities: Record; + }; expect(config.capabilities["agent-device"]?.leaseTtlCapMs).toBe(10_000); expect(config).not.toHaveProperty("agentDevice"); }); }); - -describe("FakeAgentDevice as agent-device behaves", () => { - const udid = "00000000-0000-0000-0000-000000000001"; - - function call( - fake: FakeAgentDevice, - method: string, - params: Record, - tenant: string | null = "tok_a", - ): Promise { - return fake.fetch( - new Request(`${fake.baseUrl}/rpc`, { - body: JSON.stringify({ id: 1, jsonrpc: "2.0", method, params }), - headers: { - authorization: `Bearer ${fake.token}`, - "content-type": "application/json", - ...(tenant === null ? {} : { "x-agent-device-tenant": tenant }), - }, - method: "POST", - }), - ); - } - - const allocation = (n: number, extra: Record = {}) => ({ - ...scope, - deviceKey: "key-1", - leaseId: hex(n), - platform: "ios", - udid, - ...extra, - }); - - it("refuses a release missing the tenant, the run ID, the lease provider, the client ID or the device key with LEASE_SCOPE_REQUIRED, and one with a different client ID with LEASE_SCOPE_MISMATCH, as agent-device does", async () => { - const fake = new FakeAgentDevice(); - await call(fake, "agent_device.lease.allocate", allocation(1)); - const full = { ...allocation(1), tenantId: "tok_a" }; - delete (full as Record)["platform"]; - delete (full as Record)["udid"]; - - const accepted = (await ( - await call(fake, "agent_device.lease.release", full) - ).json()) as Envelope; - expect(accepted.result).toMatchObject({ released: true }); - await call(fake, "agent_device.lease.allocate", allocation(1)); - - const reasonOf = async (params: Record, tenant: string | null = "tok_a") => { - const answer = (await ( - await call(fake, "agent_device.lease.release", params, tenant) - ).json()) as { - readonly error?: { readonly data?: { readonly details?: { readonly reason?: string } } }; - }; - return answer.error?.data?.details?.reason; - }; - const { tenantId: _tenantId, ...untenanted } = full; - expect(await reasonOf(untenanted, null)).toBe("LEASE_SCOPE_REQUIRED"); - for (const missing of ["runId", "leaseProvider", "clientId", "deviceKey"]) { - const partial = { ...full } as Record; - delete partial[missing]; - expect(await reasonOf(partial), missing).toBe("LEASE_SCOPE_REQUIRED"); - } - expect(await reasonOf({ ...full, clientId: "client-2" })).toBe("LEASE_SCOPE_MISMATCH"); - }); - - it("answers an allocate for a device a live lease holds with DEVICE_IN_USE, and a release of a lease with a held command only after the command is let go", async () => { - const fake = new FakeAgentDevice(); - const codeOf = async (response: Response) => - ((await response.json()) as Envelope).error?.data?.code; - expect( - await codeOf(await call(fake, "agent_device.lease.allocate", allocation(1))), - ).toBeUndefined(); - expect( - await codeOf( - await call(fake, "agent_device.lease.allocate", allocation(2, { udid: "other-device" })), - ), - ).toBeUndefined(); - - expect(await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3)))).toBe( - "DEVICE_IN_USE", - ); - - fake.holdCommand(); - const running = track( - call(fake, "agent_device.command", { command: "snapshot", meta: { leaseId: hex(1) } }), - ); - await sleep(20); - const full = { ...allocation(1), tenantId: "tok_a" }; - const releasing = track(call(fake, "agent_device.lease.release", full)); - await sleep(50); - - expect(releasing.done()).toBe(false); - expect(await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3)))).toBe( - "DEVICE_IN_USE", - ); - fake.releaseCommand(); - await running.promise; - expect(((await (await releasing.promise).json()) as Envelope).result).toMatchObject({ - released: true, - }); - expect( - await codeOf(await call(fake, "agent_device.lease.allocate", allocation(3))), - ).toBeUndefined(); - }); -}); diff --git a/src/agent-device/test-edge-front.ts b/src/agent-device/test-edge-front.ts new file mode 100644 index 000000000..e268d731e --- /dev/null +++ b/src/agent-device/test-edge-front.ts @@ -0,0 +1,33 @@ +import { expect } from "vitest"; +import { adminSession, type Envelope, type Rig } from "./test-support.js"; + +export async function waiting( + rig: Rig, +): Promise { + return (await rig.daemon.dispatch("list.get", { kind: "requests" }, adminSession)) as never; +} + +export const phoneModels = { + defaultModels: { phone: ["iPhone 16"] }, + modelClasses: { "iPhone 16": "phone" }, +} as const; + +export const endedMessage = "lease ended by simlock; run disconnect, then open"; + +export function expectEndedBySimlock(answer: Envelope): void { + expect(answer.result).toBeUndefined(); + expect(answer.error).toMatchObject({ + data: { + code: "UNAUTHORIZED", + details: { reason: "LEASE_ENDED_BY_HOST" }, + message: endedMessage, + retriable: false, + }, + }); +} + +export function expectLeaseGone(answer: Envelope, reason: string): void { + expect(answer.error).toMatchObject({ + data: { code: "UNAUTHORIZED", details: { reason } }, + }); +} From 6d4b737a5891f81b7eb28cec036287af0b1d68f6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:28:19 +0100 Subject: [PATCH 47/83] test(agent-device): split gateway-front.test.ts by topic --- .../gateway-front-lease-header.test.ts | 441 ++++++ src/agent-device/gateway-front-lease.test.ts | 487 ++++++ .../gateway-front-reconnect.test.ts | 520 ++++++ src/agent-device/gateway-front.test.ts | 1387 +---------------- src/agent-device/test-gateway-front.ts | 64 + 5 files changed, 1513 insertions(+), 1386 deletions(-) create mode 100644 src/agent-device/gateway-front-lease-header.test.ts create mode 100644 src/agent-device/gateway-front-lease.test.ts create mode 100644 src/agent-device/gateway-front-reconnect.test.ts create mode 100644 src/agent-device/test-gateway-front.ts diff --git a/src/agent-device/gateway-front-lease-header.test.ts b/src/agent-device/gateway-front-lease-header.test.ts new file mode 100644 index 000000000..b5435ec7c --- /dev/null +++ b/src/agent-device/gateway-front-lease-header.test.ts @@ -0,0 +1,441 @@ +import { describe, expect, it } from "vitest"; +import { + secrets, + notAvailable, + holderCannot, + tooOld, + adminSession, + setUp, + turn, + send, + rpc, + envelopeOf, + hex, + grantedOnGateway, + release, + heartbeat, + command, + fleetLeases, + tenantOf, + track, + expectEndedBySimlock, + tapDispatch, + heartbeatsReceived, + first, + leaveOutOfNextListing, +} from "./test-gateway.js"; +import { + LEASE_HEADER, + notYoursOnGateway, + leased, + holderOf, + listedByGateway, + twoLeasesOnTwoWorkers, + addressOfLease, + pathsReceived, + listedNames, +} from "./test-gateway-front.js"; + +const notALeaseId = "refused: the x-agent-device-lease-id header is not a lease id"; + +describe("the agent-device front, through a gateway: the lease header", { timeout: 60_000 }, () => { + it("through the gateway, an upload preflight with the header naming the token's lease on W2 reaches W2's agent-device daemon and not W1's", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const target = holderOf(rig, 2); + const other = holderOf(rig, 1); + const before = pathsReceived(other); + + const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + + expect(preflight.status).toBe(200); + expect(pathsReceived(target)).toContain("/upload/preflight"); + expect(target.fake.requests.find((seen) => seen.path === "/upload/preflight")?.tenant).toBe( + await tenantOf(target, 2), + ); + expect(pathsReceived(other)).toEqual(before); + }); + + it("the preflight's ticket URL names the gateway's origin, its `upload.headers` carry the caller's own `authorization`, W2 received its own daemon's token in `authorization` and `x-agent-device-token` and none of the caller's credentials, and a `PUT` to it with the lease header and those headers reaches W2", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const target = holderOf(rig, 2); + + const preflight = await send(rig, "/upload/preflight", { + body: "{}", + headers: { ...leased(2), "x-agent-device-token": "caller-token" }, + }); + + const ticket = ( + (await preflight.json()) as { upload: { url: string; headers: Record } } + ).upload; + expect(ticket.url.startsWith(`${rig.origin}/v1/agent-device/upload/direct/`)).toBe(true); + const named = (name: string) => + Object.entries(ticket.headers).find(([key]) => key.toLowerCase() === name)?.[1]; + expect(named("authorization")).toBe(`Bearer ${secrets.one}`); + expect(named("x-agent-device-token")).toBe("caller-token"); + const seen = target.fake.preflightHeaders[0] ?? {}; + expect(seen["authorization"]).toBe(`Bearer ${target.fake.token}`); + // why: the proxy sends its own daemon token in both headers, never the caller's. + expect(seen["x-agent-device-token"]).toBe(target.fake.token); + expect(Object.values(seen).join("\n")).not.toContain(secrets.one); + expect(Object.values(seen).join("\n")).not.toContain("caller-token"); + + const put = await fetch(ticket.url, { + body: "artifact bytes", + headers: { ...ticket.headers, ...leased(2) }, + method: "PUT", + }); + + expect(put.status).toBe(200); + expect(target.fake.uploads).toEqual(["artifact bytes"]); + expect(holderOf(rig, 1).fake.uploads).toEqual([]); + }); + + it("a finalize, a `POST /upload`, a `GET artifacts/` and a diagnostics `GET`, each with the header, reach the lease's worker", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const target = holderOf(rig, 2); + const other = holderOf(rig, 1); + const before = pathsReceived(other); + + await send(rig, "/upload/finalize", { body: "{}", headers: leased(2) }); + // why: the proxy confines diagnostics to sessions under the caller's tenant, which the gateway names. + const tenant = target.fake.requests.at(-1)?.tenant; + expect(tenant).toBeDefined(); + const diagnostics = `/sessions/${String(tenant)}:s1/requests/r1/diagnostics`; + await send(rig, "/upload", { body: "{}", headers: leased(2) }); + await send(rig, "/artifacts/a.png", { headers: leased(2) }); + await send(rig, diagnostics, { headers: leased(2) }); + + const paths = pathsReceived(target); + expect(paths).toContain("/upload/finalize"); + expect(paths).toContain("/upload"); + expect(paths).toContain("/artifacts/a.png"); + expect(paths).toContain(diagnostics); + expect(pathsReceived(other)).toEqual(before); + }); + + it("through the gateway, the same requests without the header, or with an empty one, and a `GET artifacts` list, are refused with `refused: this agent-device is too old to use through a gateway; update it`", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const diagnostics = `/sessions/${await tenantOf(holderOf(rig, 2), 2)}:s1/requests/r1/diagnostics`; + const calls: readonly { path: string; body?: string }[] = [ + { body: "{}", path: "/upload/preflight" }, + { body: "{}", path: "/upload/finalize" }, + { body: "{}", path: "/upload" }, + { path: "/artifacts/a.png" }, + { path: diagnostics }, + ]; + for (const call of calls) { + const accepted = await send(rig, call.path, { + ...(call.body === undefined ? {} : { body: call.body }), + headers: leased(2), + }); + expect(await accepted.text()).not.toContain(tooOld); + expect(pathsReceived(holderOf(rig, 2))).toContain(call.path); + } + const before = rig.members.map((member) => member.fake.requests.length); + + for (const call of calls) { + for (const headers of [{}, { [LEASE_HEADER]: "" }]) { + const answer = await send(rig, call.path, { + ...(call.body === undefined ? {} : { body: call.body }), + headers, + }); + expect(await answer.text()).toContain(tooOld); + } + } + const list = await send(rig, "/artifacts"); + expect(await list.text()).toContain(tooOld); + + expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); + }); + + it("through the gateway, a header naming another token's lease is refused `refused: lease is not one of yours on this gateway` with status 403, and no agent-device daemon receives it", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const own = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + expect(own.status).toBe(200); + const before = rig.members.map((member) => member.fake.requests.length); + + const other = await send(rig, "/upload/preflight", { + body: "{}", + headers: leased(2), + token: secrets.two, + }); + + expect(other.status).toBe(403); + expect(await other.text()).toContain(notYoursOnGateway(2)); + expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); + }); + + it("through the gateway, a header of 40 characters, and one with upper-case hex, are refused `refused: the x-agent-device-lease-id header is not a lease id` with status 400, and the reply does not contain the value", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const accepted = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + expect(accepted.status).toBe(200); + const before = rig.members.map((member) => member.fake.requests.length); + + for (const value of ["a".repeat(40), `ABCDEF${"0".repeat(26)}`]) { + const answer = await send(rig, "/upload/preflight", { + body: "{}", + headers: { [LEASE_HEADER]: value }, + }); + const text = await answer.text(); + expect(answer.status).toBe(400); + expect(text).toContain(notALeaseId); + expect(text).not.toContain(value); + } + + expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); + }); + + it("through the gateway, a header naming the token's lease whose worker stopped being capable answers the notAvailable reply naming that reason", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const accepted = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + expect(accepted.status).toBe(200); + const target = holderOf(rig, 2); + await turn(rig, target, "unavailable"); + const before = pathsReceived(target); + + const answer = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + + expect(await answer.text()).toContain(`${notAvailable}${holderCannot}`); + expect(pathsReceived(target)).toEqual(before); + }); + + it("through the gateway, `/health` with a header naming another token's lease answers the proxy's health", async () => { + const rig = await twoLeasesOnTwoWorkers(); + + const health = await send(rig, "/health", { headers: leased(2), token: secrets.two }); + + expect(health.status).toBe(200); + expect(await health.json()).toMatchObject({ service: "agent-device-proxy" }); + }); + + it("through the gateway, an upload naming the token's lease after `simlock release --all` reaches that lease's worker", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const target = holderOf(rig, 2); + await rig.gateway.dispatch("lease.release-all", {}, adminSession); + await expect.poll(async () => (await fleetLeases(rig)).length).toBe(0); + const before = pathsReceived(target).filter((path) => path === "/upload/preflight").length; + + const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); + + expect(await preflight.text()).not.toContain(notYoursOnGateway(2)); + expect(pathsReceived(target).filter((path) => path === "/upload/preflight").length).toBe( + before + 1, + ); + }); + + it("through the gateway, `devices` with no lease held answers an empty list and no agent-device daemon receives it", async () => { + const rig = await setUp({ workers: [{ label: "w1" }, { label: "w2" }] }); + for (const member of rig.members) member.fake.listDevices([{ id: "stray", name: "stray" }]); + + const answer = await envelopeOf(await command(rig, "devices", undefined)); + + expect(answer.result).toMatchObject({ data: { devices: [] }, ok: true }); + for (const member of rig.members) { + expect(member.fake.rpcRequests("agent_device.command")).toEqual([]); + } + }); + + it("through the gateway, `devices` with leases on W1 and W2 answers both devices, each once, and no device of another token", async () => { + const rig = await twoLeasesOnTwoWorkers(); + for (const [index, n] of [1, 2].entries()) { + holderOf(rig, n).fake.listDevices([ + { id: addressOfLease(rig, n), name: `mine-${String(index)}` }, + { id: "another-token-device", name: "foreign" }, + ]); + } + + expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ + "mine-0", + "mine-1", + ]); + }); + + it("through the gateway, `devices` with leases on W1 and W2 whose emulators are both `emulator-5554` answers both, each from its own worker", async () => { + const rig = await twoLeasesOnTwoWorkers(); + expect(addressOfLease(rig, 1)).toBe(addressOfLease(rig, 2)); + holderOf(rig, 1).fake.listDevices([{ id: addressOfLease(rig, 1), name: "from-first" }]); + holderOf(rig, 2).fake.listDevices([{ id: addressOfLease(rig, 2), name: "from-second" }]); + + expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ + "from-first", + "from-second", + ]); + }); + + it("through the gateway, `devices` with two leases on one worker answers both devices, each once", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); + await grantedOnGateway(rig, 1); + await grantedOnGateway(rig, 2); + await listedByGateway(rig, 2); + const member = first(rig); + member.fake.listDevices([ + { id: addressOfLease(rig, 1), name: "first" }, + { id: addressOfLease(rig, 2), name: "second" }, + { id: "another-token-device", name: "foreign" }, + ]); + + expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ + "first", + "second", + ]); + }); + + it("through the gateway, `devices` with two leases on one worker, sent with a `meta.requestId` as the agent-device CLI does, answers both devices", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); + await grantedOnGateway(rig, 1); + await grantedOnGateway(rig, 2); + await listedByGateway(rig, 2); + const member = first(rig); + member.fake.listDevices([ + { id: addressOfLease(rig, 1), name: "first" }, + { id: addressOfLease(rig, 2), name: "second" }, + ]); + member.fake.holdCommand(); + const answer = track( + command(rig, "devices", undefined, {}, { meta: { requestId: "req-devices-1" } }), + ); + await expect.poll(() => member.fake.rpcRequests("agent_device.command").length).toBe(2); + member.fake.releaseCommand(); + + expect(await listedNames(await answer.promise)).toEqual(["first", "second"]); + }); + + it("through the gateway, `devices` leaves out a lease that was released, and a lease the gateway's `lease.list` omits from that one answer while still holding it", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 3 }] }); + for (const n of [1, 2, 3]) await grantedOnGateway(rig, n); + await listedByGateway(rig, 3); + first(rig).fake.listDevices( + [1, 2, 3].map((n) => ({ id: addressOfLease(rig, n), name: `lease-${String(n)}` })), + ); + expect((await envelopeOf(await release(rig, 1))).result).toMatchObject({ ok: true }); + const listing = leaveOutOfNextListing(rig.gateway, hex(2)); + + expect(await listedNames(await command(rig, "devices", undefined))).toEqual(["lease-3"]); + expect(listing.cut()).toBe(1); + expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ + "lease-2", + "lease-3", + ]); + }); + + it("through the gateway, `devices` after `simlock release --all` on the gateway leaves the ended lease out and ends it for the front, so the next heartbeat answers the ended-by-simlock reply without reaching the worker", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + first(rig).fake.listDevices([{ id: addressOfLease(rig, 1), name: "lease-1" }]); + await rig.gateway.dispatch("lease.release-all", {}, adminSession); + + expect(await listedNames(await command(rig, "devices", undefined))).toEqual([]); + const received = heartbeatsReceived(first(rig)); + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + expect(heartbeatsReceived(first(rig))).toBe(received); + }); + + it("through the gateway, `devices` when W2's agent-device answers an error passes that error on", async () => { + const rig = await twoLeasesOnTwoWorkers(); + holderOf(rig, 1).fake.listDevices([{ id: addressOfLease(rig, 1), name: "first" }]); + const second = holderOf(rig, 2).fake; + second.answerDevicesWith( + JSON.stringify({ + error: { + code: -32000, + data: { code: "COMMAND_FAILED", message: "inventory down" }, + message: "inventory down", + }, + id: 1, + jsonrpc: "2.0", + }), + ); + + const failed = await envelopeOf(await command(rig, "devices", undefined)); + + expect(failed.error?.message).toBe("inventory down"); + }); + + it("through the gateway, `devices` whose lease check fails answers `could not check your leases; try again`, not an empty list, and no daemon receives the command", async () => { + const rig = await twoLeasesOnTwoWorkers(); + for (const n of [1, 2]) { + holderOf(rig, n).fake.listDevices([ + { id: addressOfLease(rig, n), name: `mine-${String(n)}` }, + ]); + } + const tap = tapDispatch(rig.gateway); + tap.rejectNext("lease.list"); + const before = rig.members.map( + (member) => member.fake.rpcRequests("agent_device.command").length, + ); + + const answer = await envelopeOf(await command(rig, "devices", undefined)); + + expect(answer.error?.data).toMatchObject({ + details: { reason: "SIMLOCK_FAILED" }, + retriable: true, + }); + expect(answer.error?.message).toBe("could not check your leases; try again"); + expect( + rig.members.map((member) => member.fake.rpcRequests("agent_device.command").length), + ).toEqual(before); + }); + + it("through the gateway, `devices` when W2 answers a body with no list aborts the forward still waiting on W1", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const slow = holderOf(rig, 1).fake; + slow.holdCommand(); + holderOf(rig, 2).fake.answerDevicesWith( + JSON.stringify({ id: 1, jsonrpc: "2.0", result: { data: {}, ok: true } }), + ); + + const answer = await command(rig, "devices", undefined); + + expect(answer.status).toBe(502); + expect((await envelopeOf(answer)).error?.message).toBe( + `could not read the device list from the worker holding lease ${hex(2)}`, + ); + await expect.poll(() => slow.abortedCommands).toEqual(["devices"]); + }); + + it("through the gateway, a CLI that hangs up during `devices` aborts every forward", async () => { + const rig = await twoLeasesOnTwoWorkers(); + for (const member of rig.members) member.fake.holdCommand(); + const hangUp = new AbortController(); + const held = track(command(rig, "devices", undefined, { signal: hangUp.signal })); + await expect + .poll(() => + rig.members.map((member) => member.fake.rpcRequests("agent_device.command").length), + ) + .toEqual([1, 1]); + + hangUp.abort(); + await held.promise.catch(() => undefined); + + await expect + .poll(() => rig.members.map((member) => member.fake.abortedCommands)) + .toEqual([["devices"], ["devices"]]); + }); + + it("through the gateway, `release_materialized_paths` and an `install_from_source` naming the token's lease are refused `refused: unknown method `, and no worker sees either", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const before = rig.members.map((member) => member.fake.requests.length); + + const released = await envelopeOf( + await rpc(rig, "agent_device.release_materialized_paths", { + materializationId: "m-1", + }), + ); + const installed = await envelopeOf( + await rpc(rig, "agent_device.install_from_source", { + meta: { leaseId: hex(2) }, + platform: "ios", + source: { kind: "url", url: "https://example.com/App.zip" }, + }), + ); + + expect(released.error?.message).toBe( + "refused: unknown method agent_device.release_materialized_paths", + ); + expect(installed.error?.message).toBe( + "refused: unknown method agent_device.install_from_source", + ); + expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); + }); +}); diff --git a/src/agent-device/gateway-front-lease.test.ts b/src/agent-device/gateway-front-lease.test.ts new file mode 100644 index 000000000..bb0ade65f --- /dev/null +++ b/src/agent-device/gateway-front-lease.test.ts @@ -0,0 +1,487 @@ +import { describe, expect, it } from "vitest"; +import { DispatchError } from "../daemon/dispatch.js"; +import { readHistoryEvents } from "../history/testing.js"; +import { + runningDaemons, + secrets, + principals, + notAvailable, + holderCannot, + tooOld, + workerCeilingMs, + scope, + adminSession, + sessionFor, + Member, + Rig, + sleep, + setUp, + turn, + advance, + send, + envelopeOf, + hex, + allocate, + grantedOnGateway, + release, + heartbeat, + command, + workerLeases, + fleetLeases, + waiting, + workerLeaseFor, + deadlineOf, + tenantOf, + grantedDevice, + deviceStates, + track, + expectEndedBySimlock, + tapDispatch, + interceptRelay, + relayFailsWith, + wipes, + heartbeatsReceived, + allocatesReceived, + runningCommand, + first, + leaveOutOfNextListing, +} from "./test-gateway.js"; +import { listedByGateway } from "./test-gateway-front.js"; + +describe("the agent-device front, through a gateway", { timeout: 60_000 }, () => { + it("with every matching device leased, allocate waits and is granted once one is released", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); + await grantedOnGateway(rig, 1); + + const second = track(allocate(rig, 2, {}, { token: secrets.two })); + await expect.poll(() => waiting(rig)).toEqual([hex(2)]); + expect(second.done()).toBe(false); + + await release(rig, 1); + const answer = await envelopeOf(await second.promise); + + expect(answer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); + await expect + .poll(async () => (await fleetLeases(rig, principals.two)).map((lease) => lease.id)) + .toEqual([hex(2)]); + }); + + it("a CLI that hangs up while its allocate waits leaves the gateway's queue, and nothing is granted later", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); + const member = first(rig); + await grantedOnGateway(rig, 1); + const hangUp = new AbortController(); + const second = track(allocate(rig, 2, {}, { signal: hangUp.signal, token: secrets.two })); + await expect.poll(() => waiting(rig)).toEqual([hex(2)]); + + hangUp.abort(); + await second.promise.catch(() => undefined); + await expect.poll(() => waiting(rig)).toEqual([]); + await release(rig, 1); + await sleep(100); + + expect(await fleetLeases(rig, principals.two)).toEqual([]); + expect(await workerLeases(member)).toEqual([]); + expect(allocatesReceived(member, 2)).toEqual([]); + }); + + it("a command for the lease reaches that worker's agent-device daemon with the prefixed tenant, and its streamed reply reaches the CLI unchanged", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + const chunks = [ + '{"type":"progress","event":{"kind":"command","message":"one"}}\n', + '{"type":"progress","event":{"kind":"command","message":"two"}}\n', + '{"type":"response","response":{"jsonrpc":"2.0","id":"x","result":{"ok":true,"data":{}}}}\n', + ]; + member.fake.streamNextCommand(chunks); + + const streamed = await command(rig, "snapshot", 1); + + expect(await streamed.text()).toBe(chunks.join("")); + expect( + member.fake.rpcRequests("agent_device.command").map((request) => request.tenant), + ).toEqual([await tenantOf(member, 1)]); + }); + + it("a heartbeat reaches the worker's agent-device daemon first, then renews the simlock lease to now plus its `ttlMs`", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); + const allocated = await deadlineOf(member, 1); + let deadlineAtDaemon: number | undefined; + member.fake.onHeartbeat = async () => { + deadlineAtDaemon = await deadlineOf(member, 1); + }; + await advance(rig, 10_000, 5_000); + + const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); + + expect(answer.result).toMatchObject({ ok: true }); + expect(deadlineAtDaemon).toBe(allocated); + expect(member.fake.rpcRequests("agent_device.lease.heartbeat")[0]?.params).toMatchObject({ + ...scope, + leaseId: hex(1), + ttlMs: workerCeilingMs, + }); + expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); + }); + + it("a heartbeat for a lease whose worker stopped being capable answers the notAvailable reply naming `the worker holding this lease cannot serve agent-device`", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); + await turn(rig, member, "unavailable"); + const received = heartbeatsReceived(member); + + const answer = await envelopeOf(await heartbeat(rig, 1)); + + expect(answer.error?.message).toBe(`${notAvailable}${holderCannot}`); + expect(heartbeatsReceived(member)).toBe(received); + }); + + it("while a command runs longer than the TTL, the simlock lease is renewed every TTL/3 through the gateway and does not expire", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); + await runningCommand(rig, member, 1); + const tap = tapDispatch(rig.gateway); + const deadlines: (number | undefined)[] = []; + + for (let tick = 0; tick < 9; tick += 1) { + await advance(rig, 10_000, 5_000); + await sleep(30); + deadlines.push(await deadlineOf(member, 1)); + } + + expect(deadlines.every((deadline) => deadline !== undefined)).toBe(true); + expect(deadlines.at(-1)).toBeGreaterThan(rig.clock.now()); + expect(new Set(deadlines).size).toBeGreaterThan(4); + expect(tap.operations.filter((name) => name === "lease.renew").length).toBeGreaterThanOrEqual( + 7, + ); + member.fake.releaseCommand(); + }); + + it("release goes to the worker's agent-device daemon first, then releases the simlock lease, and the device is deleted and the pool makes a ready one", async () => { + const rig = await setUp(); + const member = first(rig); + let heldAtDaemonRelease: readonly string[] = []; + member.fake.onRelease = async () => { + heldAtDaemonRelease = (await workerLeases(member)).map((lease) => lease.requesterId); + }; + await grantedOnGateway(rig, 1); + + const released = await envelopeOf(await release(rig, 1)); + + expect(released.result).toMatchObject({ ok: true, released: true }); + expect(heldAtDaemonRelease).toHaveLength(1); + expect(await workerLeases(member)).toEqual([]); + await expect.poll(() => deviceStates(member)).toEqual(["ready"]); + expect(await fleetLeases(rig)).toEqual([]); + }); + + it("a release the agent-device daemon fails keeps the simlock lease", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + member.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon could not release" }); + + const answer = await envelopeOf(await release(rig, 1)); + + expect(answer.error).toMatchObject({ + data: { code: "COMMAND_FAILED" }, + message: "daemon could not release", + }); + expect(await workerLeases(member)).toHaveLength(1); + expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + }); + + it("after gateway release or expiry, boot is refused locally; release reaches agent-device before the wipe and heartbeat answers `LEASE_ENDED_BY_HOST`", async () => { + const rig = await setUp(); + const member = first(rig); + const wipesAtRelease: number[] = []; + member.fake.onRelease = async () => { + wipesAtRelease.push(wipes(member)); + }; + await grantedOnGateway(rig, 1); + const before = wipes(member); + + await rig.gateway.dispatch("lease.release-all", {}, adminSession); + + await expect.poll(() => member.fake.rpcRequests("agent_device.lease.release").length).toBe(1); + expect(wipesAtRelease).toEqual([before]); + const received = heartbeatsReceived(member); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + + expect(heartbeatsReceived(member)).toBe(received + 1); + + const bootRig = await setUp(); + const bootMember = first(bootRig); + await grantedOnGateway(bootRig, 1); + await listedByGateway(bootRig, 1); + expect((await envelopeOf(await command(bootRig, "boot", 1))).result).toMatchObject({ + ok: true, + }); + await bootRig.gateway.dispatch("lease.release-all", {}, adminSession); + const commands = bootMember.fake.rpcRequests("agent_device.command").length; + expectEndedBySimlock(await envelopeOf(await command(bootRig, "boot", 1))); + expect(bootMember.fake.rpcRequests("agent_device.command")).toHaveLength(commands); + + await grantedOnGateway(bootRig, 2, { ttlMs: 5_000 }); + await listedByGateway(bootRig, 1); + const address = (await grantedDevice(bootMember)).address; + expect((await envelopeOf(await command(bootRig, "boot", 2))).result).toMatchObject({ + ok: true, + }); + await advance(bootRig, 6_000, 1_000); + await expect.poll(() => fleetLeases(bootRig)).toEqual([]); + const afterExpiry = bootMember.fake.rpcRequests("agent_device.command").length; + expectEndedBySimlock( + await envelopeOf(await command(bootRig, "boot", 2, {}, { flags: { udid: address } })), + ); + expect(bootMember.fake.rpcRequests("agent_device.command")).toHaveLength(afterExpiry); + }); + + it("a command with no device selector naming a lease id another token holds, and one the gateway does not know, answer `refused: lease is not one of yours on this gateway` with status 403, and no worker receives it; with `--udid`, the same command answers `refused: simlock picks the device`, as on a worker", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + const own = await envelopeOf(await command(rig, "snapshot", 1)); + expect(own.result).toMatchObject({ ok: true }); + const received = member.fake.rpcRequests("agent_device.command").length; + + const other = await command(rig, "snapshot", 1, { token: secrets.two }); + // why: within 60 s of the gateway starting, a lease it does not know answers "try again" (#517); past that, a refusal. + rig.clock.advance(60_000); + const unknown = await command(rig, "snapshot", 9); + + expect(other.status).toBe(403); + expect((await envelopeOf(other)).error?.message).toBe( + `refused: lease ${hex(1)} is not one of yours on this gateway`, + ); + expect(unknown.status).toBe(403); + expect((await envelopeOf(unknown)).error?.message).toBe( + `refused: lease ${hex(9)} is not one of yours on this gateway`, + ); + const picked = await envelopeOf( + await command(rig, "snapshot", 1, { token: secrets.two }, { flags: { udid: "ABC" } }), + ); + expect(picked.error?.message).toBe("refused: simlock picks the device"); + expect(member.fake.rpcRequests("agent_device.command").length).toBe(received); + }); + + it("after `simlock release --all`, once the fake clock has moved the gateway's `lease.maxTtlMs` plus 1 s past the index's removal of the lease, the link's memory no longer holds it, and a heartbeat for it answers `refused: lease is not one of yours on this gateway`; one second before that, the heartbeat is still relayed", async () => { + const options = { defaultTtlMs: 60_000, maxTtlMs: 120_000 } as const; + const early = await setUp(options); + const earlyMember = first(early); + await grantedOnGateway(early, 1, { ttlMs: 30_000 }); + await early.gateway.dispatch("lease.release-all", {}, adminSession); + await expect.poll(async () => (await fleetLeases(early)).length).toBe(0); + await advance(early, 119_000, 10_000); + const received = heartbeatsReceived(earlyMember); + + expectEndedBySimlock(await envelopeOf(await heartbeat(early, 1))); + expect(heartbeatsReceived(earlyMember)).toBe(received + 1); + + const late = await setUp(options); + const lateMember = first(late); + await grantedOnGateway(late, 1, { ttlMs: 30_000 }); + await late.gateway.dispatch("lease.release-all", {}, adminSession); + await expect.poll(async () => (await fleetLeases(late)).length).toBe(0); + await advance(late, 121_000, 10_000); + const lateReceived = heartbeatsReceived(lateMember); + + const answer = await envelopeOf(await heartbeat(late, 1)); + + expect(answer.error?.message).toBe( + `refused: lease ${hex(1)} is not one of yours on this gateway`, + ); + expect(heartbeatsReceived(lateMember)).toBe(lateReceived); + }); + + it("a request with no lease id other than `/health` is refused with `refused: this agent-device is too old to use through a gateway; update it`", async () => { + const rig = await setUp(); + const member = first(rig); + expect((await send(rig, "/health")).status).toBe(200); + const before = member.fake.requests.length; + + const bare = await envelopeOf(await command(rig, "snapshot", undefined)); + const upload = await send(rig, "/upload/preflight", { body: "{}" }); + + expect(bare.error?.message).toBe(tooOld); + expect(await upload.text()).toContain(tooOld); + expect(member.fake.requests.length).toBe(before); + }); + + it("`capabilities.agent-device.leaseTtlCapMs` on the gateway caps the TTL of a lease made through it", async () => { + const rig = await setUp({ leaseTtlCapMs: 10_000 }); + const member = first(rig); + + await grantedOnGateway(rig, 1, { ttlMs: 300_000 }); + + expect((await workerLeaseFor(member, 1)).ttlMs).toBe(10_000); + await expect.poll(async () => (await fleetLeases(rig))[0]?.ttlMs).toBe(10_000); + }); + + it("stopping the gateway while an allocate waits ends its lease request `cancelled`, not `DAEMON_STOPPING`, and a running command's renewals stop", async () => { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 30_000 }, { token: secrets.two }); + await runningCommand(rig, member, 1, { token: secrets.two }); + const tap = tapDispatch(rig.gateway); + await advance(rig, 20_000, 5_000); + const renewedBefore = tap.operations.filter((name) => name === "lease.renew").length; + const second = track(allocate(rig, 2)); + await expect.poll(() => waiting(rig)).toEqual([hex(2)]); + + await rig.gateway.stop("test"); + await second.promise.catch(() => undefined); + const afterStop = tap.operations.length; + await advance(rig, 60_000, 10_000); + + expect(renewedBefore).toBeGreaterThan(0); + expect(tap.operations.length).toBe(afterStop); + const lines = (await readHistoryEvents(rig.directory)) as Array<{ + event?: string; + payload?: Record; + }>; + expect( + lines.filter( + (entry) => + entry.event === "lease.rejected" && + entry.payload?.["requester"] === hex(2) && + entry.payload["reason"] === "cancelled", + ), + ).toHaveLength(1); + member.fake.releaseCommand(); + }); + + it("a heartbeat, a command, a release and an allocate whose relay fails with `RELAY_TARGET_MISSING` answer the notAvailable reply with that message, and the failed allocate leaves no lease", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + const missing = `${notAvailable}no relay target agent-device on this worker`; + expect((await envelopeOf(await command(rig, "snapshot", 1))).result).toMatchObject({ + ok: true, + }); + relayFailsWith(rig, "RELAY_TARGET_MISSING", "no relay target agent-device on this worker"); + + const beat = await heartbeat(rig, 1); + const commanded = await command(rig, "snapshot", 1); + const released = await release(rig, 1); + const made = await allocate(rig, 2); + + expect((await envelopeOf(beat)).error?.message).toBe(missing); + expect(commanded.status).toBe(503); + expect((await envelopeOf(commanded)).error?.message).toBe(missing); + expect((await envelopeOf(released)).error?.message).toBe(missing); + expect((await envelopeOf(made)).error?.message).toBe(missing); + await expect.poll(async () => (await workerLeases(member)).length).toBe(1); + }); + + it("a heartbeat for a lease the front's memory lacks, whose release relay then fails with `RELAY_TARGET_MISSING`, answers the notAvailable reply with that message and leaves the simlock lease", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + const held = member.fake.leases.get(hex(1)); + expect(held).toBeDefined(); + member.fake.leases.set(hex(2), { + ...(held as NonNullable), + leaseId: hex(2), + tenant: (held?.tenant ?? "").replace(hex(1), hex(2)), + }); + // why: a lease whose requester is not its own id is never adopted, so the front's memory lacks it however soon the worker's catalog arrives. + await rig.gateway.dispatch( + "lease.request", + { leaseId: hex(2), platform: "ios", requesterId: "someone-else", requires: ["agent-device"] }, + sessionFor(principals.one), + ); + interceptRelay(rig, async ({ request }) => { + const body = (await request.clone().json()) as { method?: string }; + if (body.method === "agent_device.lease.release") { + throw new DispatchError( + "RELAY_TARGET_MISSING", + "no relay target agent-device on this worker", + ); + } + }); + + const beat = await envelopeOf(await heartbeat(rig, 2)); + + expect(beat.error?.message).toBe(`${notAvailable}no relay target agent-device on this worker`); + expect((await fleetLeases(rig)).map((lease) => lease.id)).toContain(hex(2)); + }); + + it("a lease the gateway's `lease.list` leaves out during devices, allocate, boot, renewal or heartbeat is not ended, and the next heartbeat renews it", async () => { + const during: Record = {}; + const triggers: Record Promise> = { + allocate: async (rig) => { + await grantedOnGateway(rig, 2); + }, + devices: async (rig) => { + await command(rig, "devices", undefined); + }, + heartbeat: async (rig) => { + const beat = await envelopeOf(await heartbeat(rig, 1)); + during.heartbeat = beat.result ?? beat.error?.data; + }, + boot: async (rig) => { + const answer = await envelopeOf(await command(rig, "boot", 1)); + during.boot = answer.error?.data; + }, + renewal: async (rig, member) => { + const running = await runningCommand(rig, member, 1); + await advance(rig, 10_000, 5_000); + member.fake.releaseCommand(); + await running.promise; + }, + }; + const outcomes: Record = {}; + + for (const [name, trigger] of Object.entries(triggers)) { + const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); + await listedByGateway(rig, 1); + // why: the front's own record of the deadline is only the grant's, so an allocate re-reads lease 1 only once that has passed; a heartbeat keeps the simlock lease alive meanwhile. + await advance(rig, 20_000, 5_000); + expect((await envelopeOf(await heartbeat(rig, 1, { ttlMs: 30_000 }))).result).toMatchObject({ + ok: true, + }); + await advance(rig, 15_000, 5_000); + const listing = leaveOutOfNextListing(rig.gateway, hex(1)); + + await trigger(rig, member); + expect(listing.cut()).toBe(1); + await advance(rig, 1_000, 1_000); + const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); + + outcomes[name] = { + answer: answer.result ?? answer.error?.data, + cut: listing.cut(), + leftMs: ((await deadlineOf(member, 1)) ?? Number.NaN) - rig.clock.now(), + }; + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + } + + const live = { answer: expect.objectContaining({ ok: true }), cut: 1, leftMs: 60_000 }; + expect(outcomes).toEqual({ + allocate: live, + boot: live, + devices: live, + heartbeat: live, + renewal: live, + }); + // why: the command already ran on the worker, so the heartbeat that met the omission still answers its reply. + expect(during).toEqual({ + boot: expect.objectContaining({ + message: "could not check your leases; try again", + retriable: true, + }), + heartbeat: expect.objectContaining({ ok: true }), + }); + }); +}); diff --git a/src/agent-device/gateway-front-reconnect.test.ts b/src/agent-device/gateway-front-reconnect.test.ts new file mode 100644 index 000000000..5ba89a21e --- /dev/null +++ b/src/agent-device/gateway-front-reconnect.test.ts @@ -0,0 +1,520 @@ +import { describe, expect, it, vi } from "vitest"; +import { + commandsReceived, + secrets, + principals, + notAvailable, + slowCatalogMs, + adminSession, + sessionFor, + Envelope, + Member, + Rig, + sleep, + setUp, + viewOf, + untilRead, + advance, + send, + envelopeOf, + hex, + allocate, + grantedOnGateway, + release, + heartbeat, + command, + fleetLeases, + deadlineOf, + track, + expectEndedBySimlock, + tapDispatch, + heartbeatsReceived, + allocatesReceived, + runningCommand, + first, + restarted, +} from "./test-gateway.js"; +import { + notYoursOnGateway, + leased, + holderOf, + listedByGateway, + twoLeasesOnTwoWorkers, + addressOfLease, + pathsReceived, + listedNames, + picksTheDevice, +} from "./test-gateway-front.js"; + +const unreachableReason = + "the worker holding this device is not connected to the gateway; try again"; + +const unreachableMessage = `${notAvailable}${unreachableReason}`; + +const gatewayWindowMs = 60_000; + +const hourMs = 3_600_000; + +function expectUnreachable(answer: Envelope): void { + expect(answer.result).toBeUndefined(); + expect(answer.error).toMatchObject({ + data: { + details: { reason: "WORKER_UNREACHABLE" }, + message: unreachableMessage, + retriable: true, + }, + message: unreachableMessage, + }); +} + +async function expectUnreachablePlain(response: Response): Promise { + expect(response.status).toBe(503); + const body = (await response.json()) as Record; + expect(body["error"]).toBe(unreachableMessage); + expect(body).toMatchObject({ reason: "WORKER_UNREACHABLE", retriable: true }); +} + +async function expectUnreachableEverywhere(rig: Rig, n: number): Promise { + for (const response of [ + await heartbeat(rig, n), + await command(rig, "snapshot", n), + await release(rig, n), + ]) { + expect(response.status).toBe(503); + expectUnreachable(await envelopeOf(response)); + } + await expectUnreachablePlain( + await send(rig, "/upload/preflight", { body: "{}", headers: leased(n) }), + ); +} + +// why: the gateway closes the worker's socket as a network drop does; the worker redials only when its clock moves. +async function dropUplink(rig: Rig, member: Member): Promise { + const view = await viewOf(rig, member.label); + expect(view?.id).toBeDefined(); + await rig.uplinks.closeWorkerLink(view?.id as string); + await vi.waitFor( + async () => expect((await viewOf(rig, member.label))?.connection).toBe("disconnected"), + { timeout: 10_000 }, + ); +} + +async function untilConnected(rig: Rig, member: Member): Promise { + await advance(rig, 2_000, 500); + await untilRead(rig, member); +} + +async function commandWorks(rig: Rig, member: Member, n: number): Promise { + const before = commandsReceived(member); + const answer = await envelopeOf(await command(rig, "snapshot", n)); + expect(answer.error).toBeUndefined(); + expect(answer.result).toBeDefined(); + expect(commandsReceived(member)).toBe(before + 1); +} + +describe( + "the agent-device front, through a gateway: an unreachable worker", + { + timeout: 90_000, + }, + () => { + it("with the device's worker disconnected, a command, a heartbeat and a release each answer 503 with `details.reason` `WORKER_UNREACHABLE` and the `workerUnreachable` message, and no agent-device daemon receives them", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await dropUplink(rig, member); + const received = member.fake.requests.length; + + for (const response of [ + await command(rig, "snapshot", 1), + await heartbeat(rig, 1), + await release(rig, 1), + ]) { + expect(response.status).toBe(503); + expectUnreachable(await envelopeOf(response)); + } + + expect(member.fake.requests.length).toBe(received); + }); + + it("with the device's worker disconnected, an upload with the lease header answers 503 with that message, reason and retry flag, and `devices` answers the RPC reply", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await dropUplink(rig, member); + + await expectUnreachablePlain( + await send(rig, "/upload/preflight", { body: "{}", headers: leased(1) }), + ); + const devices = await command(rig, "devices", undefined); + + expect(devices.status).toBe(503); + expectUnreachable(await envelopeOf(devices)); + }); + + it("with one of the token's two workers reconnected but not yet read and the other connected, `devices` answers `workerUnreachable()` and asks neither worker", async () => { + const rig = await twoLeasesOnTwoWorkers(); + const gone = holderOf(rig, 1); + const kept = holderOf(rig, 2); + await listedByGateway(rig, 2); + await dropUplink(rig, gone); + gone.latency["listCatalog"] = slowCatalogMs; + await advance(rig, 2_000, 500); + await vi.waitFor( + async () => { + const view = await viewOf(rig, gone.label); + expect(view?.connection).toBe("connected"); + expect(view?.catalogReadAt).toBeUndefined(); + }, + { timeout: 10_000 }, + ); + const received = [gone, kept].map((member) => member.fake.requests.length); + + const devices = await command(rig, "devices", undefined); + + expect(devices.status).toBe(503); + expectUnreachable(await envelopeOf(devices)); + expect([gone, kept].map((member) => member.fake.requests.length)).toEqual(received); + }); + + it("while the worker is disconnected, the simlock lease is not released, and the front's entry is not marked ended", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + const tap = tapDispatch(rig.gateway); + await dropUplink(rig, member); + + expectUnreachable(await envelopeOf(await heartbeat(rig, 1))); + expectUnreachable(await envelopeOf(await release(rig, 1))); + + expect(tap.operations).not.toContain("lease.release"); + expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + await untilConnected(rig, member); + const beat = await envelopeOf(await heartbeat(rig, 1)); + expect(beat.error).toBeUndefined(); + expect(beat.result).toMatchObject({ ok: true }); + }); + + it("a command answered WORKER_UNREACHABLE after an uplink drop stops renewing its lease after reconnect without releasing it", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); + await listedByGateway(rig, 1); + member.fake.keepCommandsAfterAbort(); + const running = await runningCommand(rig, member, 1); + const before = await deadlineOf(member, 1); + expect(before).toBeTypeOf("number"); + const tap = tapDispatch(rig.gateway); + + try { + await dropUplink(rig, member); + + await vi.waitFor(() => expect(running.done()).toBe(true), { timeout: 5_000 }); + const answer = await running.promise; + expect(answer.status).toBe(503); + expectUnreachable(await envelopeOf(answer)); + await untilConnected(rig, member); + await advance(rig, 20_000, 1_000); + + expect(tap.operations).not.toContain("lease.release"); + expect(await deadlineOf(member, 1)).toBe(before); + } finally { + member.fake.releaseCommand(); + } + }); + + it("right after the worker reconnects and before its view is refreshed, a command answers `workerUnreachable()`; once it is refreshed, within the TTL, the next command reaches it, and the next heartbeat renews the lease", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); + await listedByGateway(rig, 1); + await dropUplink(rig, member); + member.latency["listCatalog"] = slowCatalogMs; + await advance(rig, 2_000, 500); + await vi.waitFor( + async () => { + const view = await viewOf(rig, member.label); + expect(view?.connection).toBe("connected"); + expect(view?.catalogReadAt).toBeUndefined(); + }, + { timeout: 10_000 }, + ); + + expectUnreachable(await envelopeOf(await command(rig, "snapshot", 1))); + + await advance(rig, 9_000, 1_000); + await untilRead(rig, member); + await commandWorks(rig, member, 1); + const beat = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); + expect(beat.result).toMatchObject({ ok: true }); + expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); + }); + + it("a worker that stays disconnected past the TTL: once it reconnects, the next heartbeat answers `LEASE_ENDED_BY_HOST`", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await dropUplink(rig, member); + + // why: one jump fires the worker's redial and the lease's expiry together, before the new socket is up. + await sleep(50); + rig.clock.advance(130_000); + await untilConnected(rig, member); + + expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); + }); + + it("a lease known only from the link's memory (after `simlock release --all`) whose worker then disconnects answers `workerUnreachable()`", async () => { + // why: a worker clock of its own, so moving the gateway's clock past the start window does not make the worker redial. + const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); + const member = first(rig); + await grantedOnGateway(rig, 1); + await rig.gateway.dispatch("lease.release-all", {}, adminSession); + await expect.poll(async () => (await fleetLeases(rig)).length).toBe(0); + await dropUplink(rig, member); + // why: past the 60 s start window an unplaced lease is not unreachable, so only the link's memory can place this one. + rig.clock.advance(70_000); + + expectUnreachable(await envelopeOf(await heartbeat(rig, 1))); + }); + + it("an unreachable command's final renewal fails while disconnected; after reconnect only a new heartbeat renews the retained lease", async () => { + // why: a worker clock of its own, so moving the gateway's clock does not make the worker redial. + const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); + await listedByGateway(rig, 1); + const running = await runningCommand(rig, member, 1); + const tap = tapDispatch(rig.gateway); + const before = await deadlineOf(member, 1); + rig.clock.advance(1_000); + await dropUplink(rig, member); + expectUnreachable(await envelopeOf(await running.promise)); + + rig.clock.advance(10_000); + await sleep(50); + + expect(tap.operations).toContain("lease.renew"); + expect(tap.operations).not.toContain("lease.release"); + expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); + await untilConnected(rig, member); + await advance(rig, 10_000, 5_000); + await sleep(50); + expect(before).toBe(31_000); + expect(await deadlineOf(member, 1)).toBe(before); + expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); + expect(await deadlineOf(member, 1)).toBe(member.clock.now() + 30_000); + member.fake.releaseCommand(); + }); + + it("an allocate by the same token while its other lease's worker is disconnected does not mark that lease ended", async () => { + const rig = await setUp({ + workers: [ + { label: "w1", maxRunning: 1 }, + { label: "w2", maxRunning: 1 }, + ], + }); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + const holder = holderOf(rig, 1); + await dropUplink(rig, holder); + + await grantedOnGateway(rig, 2); + expect((await fleetLeases(rig)).map((lease) => lease.id)).toContain(hex(1)); + await untilConnected(rig, holder); + + const beat = await envelopeOf(await heartbeat(rig, 1)); + expect(beat.error).toBeUndefined(); + expect(beat.result).toMatchObject({ ok: true }); + }); + }, +); + +describe( + "the agent-device front, through a gateway: after a gateway restart", + { + timeout: 90_000, + }, + () => { + it("after a gateway restart and the worker's reconnect, the next heartbeat for the lease reaches the worker's agent-device daemon and renews the simlock lease, and a following command works", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); + await listedByGateway(rig, 1); + await restarted(rig, member, 1); + const received = heartbeatsReceived(member); + + const beat = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); + + expect(beat.result).toMatchObject({ ok: true }); + expect(heartbeatsReceived(member)).toBe(received + 1); + expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); + await commandWorks(rig, member, 1); + }); + + it("after a gateway restart, a command, a release, `devices` and an upload with the lease header, each as the token's first request, work on the adopted lease", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + + await restarted(rig, member, 1); + await commandWorks(rig, member, 1); + + await restarted(rig, member, 1); + member.fake.listDevices([{ id: addressOfLease(rig, 1), name: "adopted" }]); + expect(await listedNames(await command(rig, "devices", undefined))).toEqual(["adopted"]); + + await restarted(rig, member, 1); + const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(1) }); + expect(preflight.status).toBe(200); + expect(pathsReceived(member)).toContain("/upload/preflight"); + + await restarted(rig, member, 1); + const released = await envelopeOf(await release(rig, 1)); + expect(released.result).toMatchObject({ ok: true, released: true }); + expect(await fleetLeases(rig)).toEqual([]); + }); + + it("after a gateway restart, an adopted lease on a TV model reads target `tv`, and `test --target tv` sharding uses it", async () => { + const rig = await setUp({ workers: [{ label: "w1", modelClass: "tv" }] }); + const member = first(rig); + await grantedOnGateway(rig, 1, { target: "tv" }); + await listedByGateway(rig, 1); + await restarted(rig, member, 1); + + const answer = await envelopeOf( + await command(rig, "test", 1, {}, { flags: { shardAll: 1, target: "tv" } }), + ); + + expect(answer.error).toBeUndefined(); + const sent = member.fake.rpcRequests("agent_device.command").at(-1)?.params; + const flags = (sent?.["flags"] ?? {}) as Record; + expect(flags["device"]).toBe(`simlock-${addressOfLease(rig, 1)}`); + }); + + it("after a gateway restart, a command naming a `deviceKey` on the adopted lease is accepted and sets it; a later command naming another `deviceKey` is refused `refused: simlock picks the device`", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await restarted(rig, member, 1); + + const accepted = await envelopeOf( + await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-a" } }), + ); + expect(accepted.error).toBeUndefined(); + const received = commandsReceived(member); + const other = await envelopeOf( + await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-b" } }), + ); + + expect(other.error?.message).toBe(picksTheDevice); + expect(commandsReceived(member)).toBe(received); + const same = await envelopeOf( + await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-a" } }), + ); + expect(same.error).toBeUndefined(); + }); + + it("within 60 s of a gateway restart, before the worker reconnects, a heartbeat, a command, a release and an upload with the lease header for the lease each answer `workerUnreachable()`; after 60 s, with another capable worker connected, a heartbeat for it answers `refused: lease is not one of yours on this gateway`", async () => { + // why: a worker clock of its own, so the worker does not redial until the test moves it. + const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await rig.restart(); + const received = member.fake.requests.length; + + await expectUnreachableEverywhere(rig, 1); + rig.clock.advance(gatewayWindowMs - 1_000); + await expectUnreachableEverywhere(rig, 1); + + expect(member.fake.requests.length).toBe(received); + const other = await rig.addWorker({ label: "w2" }); + await other.daemon; + await untilRead(rig, other); + rig.clock.advance(2_000); + const beat = await envelopeOf(await heartbeat(rig, 1)); + expect(beat.error?.message).toBe(notYoursOnGateway(1)); + }); + + it("after a gateway restart, a lease of another token is not adopted, and a heartbeat naming it is refused", async () => { + const rig = await setUp(); + const member = first(rig); + await grantedOnGateway(rig, 1, {}, { token: secrets.two }); + await expect.poll(async () => (await fleetLeases(rig, principals.two)).length).toBe(1); + await rig.restart(); + await untilConnected(rig, member); + await expect.poll(async () => (await fleetLeases(rig, principals.two)).length).toBe(1); + const received = commandsReceived(member); + + const beat = await envelopeOf(await heartbeat(rig, 1)); + const devices = await envelopeOf(await command(rig, "devices", undefined)); + + expect(beat.error?.message).toBe(notYoursOnGateway(1)); + expect(devices.result).toMatchObject({ data: { devices: [] }, ok: true }); + expect(commandsReceived(member)).toBe(received); + }); + + it("after a gateway restart, a lease of this token whose requester is not its own id is not adopted", async () => { + const rig = await setUp(); + const member = first(rig); + await rig.gateway.dispatch( + "lease.request", + { platform: "ios", requesterId: "someone-else", requires: ["agent-device"] }, + sessionFor(principals.one), + ); + await restarted(rig, member, 1); + const received = commandsReceived(member); + + const devices = await envelopeOf(await command(rig, "devices", undefined)); + + expect(devices.result).toMatchObject({ data: { devices: [] }, ok: true }); + expect(commandsReceived(member)).toBe(received); + }); + + it("a lease this gateway process asked for whose allocate failed after the grant (the uplink dropped mid-allocate) is never adopted, and stays out of `devices`, also when the worker's clock is an hour behind or ahead of the gateway's", async () => { + for (const skew of [0, -hourMs, hourMs]) { + const rig = await setUp({ workers: [{ clockSkewMs: skew, label: "w1" }] }); + const member = first(rig); + member.fake.holdAllocate(); + const running = track(allocate(rig, 1)); + await expect.poll(() => allocatesReceived(member, 1).length).toBe(1); + await dropUplink(rig, member); + member.fake.letAllocateGo(); + await running.promise; + await untilConnected(rig, member); + const received = commandsReceived(member); + + const devices = await envelopeOf(await command(rig, "devices", undefined)); + + expect(devices.result, `skew ${String(skew)}`).toMatchObject({ + data: { devices: [] }, + ok: true, + }); + expect(commandsReceived(member)).toBe(received); + } + }); + + it("after a gateway restart, a lease made before it is adopted when the worker's clock is an hour behind or ahead of the gateway's", async () => { + for (const skew of [-hourMs, hourMs]) { + const rig = await setUp({ workers: [{ clockSkewMs: skew, label: "w1" }] }); + const member = first(rig); + await grantedOnGateway(rig, 1); + await listedByGateway(rig, 1); + await restarted(rig, member, 1); + + const beat = await envelopeOf(await heartbeat(rig, 1)); + + expect(beat.error, `skew ${String(skew)}`).toBeUndefined(); + expect(beat.result).toMatchObject({ ok: true }); + await commandWorks(rig, member, 1); + } + }); + }, +); diff --git a/src/agent-device/gateway-front.test.ts b/src/agent-device/gateway-front.test.ts index 6f7c11047..3ca1c9c5d 100644 --- a/src/agent-device/gateway-front.test.ts +++ b/src/agent-device/gateway-front.test.ts @@ -1,10 +1,5 @@ import { describe, expect, it, vi } from "vitest"; - -import { DispatchError } from "../daemon/dispatch.js"; -import { readHistoryEvents } from "../history/testing.js"; - import { - commandsReceived, runningDaemons, secrets, principals, @@ -12,54 +7,35 @@ import { noWorker, holderCannot, missingMessage, - tooOld, - slowCatalogMs, phoneSerial, gatewayMaxTtlMs, workerCeilingMs, - scope, adminSession, sessionFor, - Envelope, - Member, - Rig, - sleep, setUp, viewOf, - untilRead, turn, advance, send, rpcBody, - rpc, envelopeOf, hex, allocate, grantedOnGateway, release, - heartbeat, command, workerLeases, fleetLeases, - waiting, - workerLeaseFor, - deadlineOf, tenantOf, grantedDevice, - deviceStates, - track, - expectEndedBySimlock, tapDispatch, interceptRelay, relayFailsWith, - wipes, - heartbeatsReceived, allocatesReceived, - runningCommand, first, - leaveOutOfNextListing, restarted, } from "./test-gateway.js"; +import { listedByGateway, listedNames, picksTheDevice } from "./test-gateway-front.js"; describe("the agent-device front, through a gateway", { timeout: 60_000 }, () => { it("with capabilities.agent-device.command set on a gateway, the gateway logs that it ignores it and starts; with capabilities.agent-device.leaseTtlCapMs above lease.maxTtlMs on a gateway, the gateway fails to start, as a worker does", async () => { @@ -444,1365 +420,4 @@ describe("the agent-device front, through a gateway", { timeout: 60_000 }, () => await expect.poll(async () => (await workerLeases(member)).length).toBe(0); expect(await fleetLeases(rig)).toEqual([]); }); - - it("with every matching device leased, allocate waits and is granted once one is released", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); - await grantedOnGateway(rig, 1); - - const second = track(allocate(rig, 2, {}, { token: secrets.two })); - await expect.poll(() => waiting(rig)).toEqual([hex(2)]); - expect(second.done()).toBe(false); - - await release(rig, 1); - const answer = await envelopeOf(await second.promise); - - expect(answer.result).toMatchObject({ data: { lease: { leaseId: hex(2) } }, ok: true }); - await expect - .poll(async () => (await fleetLeases(rig, principals.two)).map((lease) => lease.id)) - .toEqual([hex(2)]); - }); - - it("a CLI that hangs up while its allocate waits leaves the gateway's queue, and nothing is granted later", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); - const member = first(rig); - await grantedOnGateway(rig, 1); - const hangUp = new AbortController(); - const second = track(allocate(rig, 2, {}, { signal: hangUp.signal, token: secrets.two })); - await expect.poll(() => waiting(rig)).toEqual([hex(2)]); - - hangUp.abort(); - await second.promise.catch(() => undefined); - await expect.poll(() => waiting(rig)).toEqual([]); - await release(rig, 1); - await sleep(100); - - expect(await fleetLeases(rig, principals.two)).toEqual([]); - expect(await workerLeases(member)).toEqual([]); - expect(allocatesReceived(member, 2)).toEqual([]); - }); - - it("a command for the lease reaches that worker's agent-device daemon with the prefixed tenant, and its streamed reply reaches the CLI unchanged", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - const chunks = [ - '{"type":"progress","event":{"kind":"command","message":"one"}}\n', - '{"type":"progress","event":{"kind":"command","message":"two"}}\n', - '{"type":"response","response":{"jsonrpc":"2.0","id":"x","result":{"ok":true,"data":{}}}}\n', - ]; - member.fake.streamNextCommand(chunks); - - const streamed = await command(rig, "snapshot", 1); - - expect(await streamed.text()).toBe(chunks.join("")); - expect( - member.fake.rpcRequests("agent_device.command").map((request) => request.tenant), - ).toEqual([await tenantOf(member, 1)]); - }); - - it("a heartbeat reaches the worker's agent-device daemon first, then renews the simlock lease to now plus its `ttlMs`", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); - const allocated = await deadlineOf(member, 1); - let deadlineAtDaemon: number | undefined; - member.fake.onHeartbeat = async () => { - deadlineAtDaemon = await deadlineOf(member, 1); - }; - await advance(rig, 10_000, 5_000); - - const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); - - expect(answer.result).toMatchObject({ ok: true }); - expect(deadlineAtDaemon).toBe(allocated); - expect(member.fake.rpcRequests("agent_device.lease.heartbeat")[0]?.params).toMatchObject({ - ...scope, - leaseId: hex(1), - ttlMs: workerCeilingMs, - }); - expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); - }); - - it("a heartbeat for a lease whose worker stopped being capable answers the notAvailable reply naming `the worker holding this lease cannot serve agent-device`", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); - await turn(rig, member, "unavailable"); - const received = heartbeatsReceived(member); - - const answer = await envelopeOf(await heartbeat(rig, 1)); - - expect(answer.error?.message).toBe(`${notAvailable}${holderCannot}`); - expect(heartbeatsReceived(member)).toBe(received); - }); - - it("while a command runs longer than the TTL, the simlock lease is renewed every TTL/3 through the gateway and does not expire", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); - await runningCommand(rig, member, 1); - const tap = tapDispatch(rig.gateway); - const deadlines: (number | undefined)[] = []; - - for (let tick = 0; tick < 9; tick += 1) { - await advance(rig, 10_000, 5_000); - await sleep(30); - deadlines.push(await deadlineOf(member, 1)); - } - - expect(deadlines.every((deadline) => deadline !== undefined)).toBe(true); - expect(deadlines.at(-1)).toBeGreaterThan(rig.clock.now()); - expect(new Set(deadlines).size).toBeGreaterThan(4); - expect(tap.operations.filter((name) => name === "lease.renew").length).toBeGreaterThanOrEqual( - 7, - ); - member.fake.releaseCommand(); - }); - - it("release goes to the worker's agent-device daemon first, then releases the simlock lease, and the device is deleted and the pool makes a ready one", async () => { - const rig = await setUp(); - const member = first(rig); - let heldAtDaemonRelease: readonly string[] = []; - member.fake.onRelease = async () => { - heldAtDaemonRelease = (await workerLeases(member)).map((lease) => lease.requesterId); - }; - await grantedOnGateway(rig, 1); - - const released = await envelopeOf(await release(rig, 1)); - - expect(released.result).toMatchObject({ ok: true, released: true }); - expect(heldAtDaemonRelease).toHaveLength(1); - expect(await workerLeases(member)).toEqual([]); - await expect.poll(() => deviceStates(member)).toEqual(["ready"]); - expect(await fleetLeases(rig)).toEqual([]); - }); - - it("a release the agent-device daemon fails keeps the simlock lease", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - member.fake.failNextRelease({ code: "COMMAND_FAILED", message: "daemon could not release" }); - - const answer = await envelopeOf(await release(rig, 1)); - - expect(answer.error).toMatchObject({ - data: { code: "COMMAND_FAILED" }, - message: "daemon could not release", - }); - expect(await workerLeases(member)).toHaveLength(1); - expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - }); - - it("after gateway release or expiry, boot is refused locally; release reaches agent-device before the wipe and heartbeat answers `LEASE_ENDED_BY_HOST`", async () => { - const rig = await setUp(); - const member = first(rig); - const wipesAtRelease: number[] = []; - member.fake.onRelease = async () => { - wipesAtRelease.push(wipes(member)); - }; - await grantedOnGateway(rig, 1); - const before = wipes(member); - - await rig.gateway.dispatch("lease.release-all", {}, adminSession); - - await expect.poll(() => member.fake.rpcRequests("agent_device.lease.release").length).toBe(1); - expect(wipesAtRelease).toEqual([before]); - const received = heartbeatsReceived(member); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - - expect(heartbeatsReceived(member)).toBe(received + 1); - - const bootRig = await setUp(); - const bootMember = first(bootRig); - await grantedOnGateway(bootRig, 1); - await listedByGateway(bootRig, 1); - expect((await envelopeOf(await command(bootRig, "boot", 1))).result).toMatchObject({ - ok: true, - }); - await bootRig.gateway.dispatch("lease.release-all", {}, adminSession); - const commands = bootMember.fake.rpcRequests("agent_device.command").length; - expectEndedBySimlock(await envelopeOf(await command(bootRig, "boot", 1))); - expect(bootMember.fake.rpcRequests("agent_device.command")).toHaveLength(commands); - - await grantedOnGateway(bootRig, 2, { ttlMs: 5_000 }); - await listedByGateway(bootRig, 1); - const address = (await grantedDevice(bootMember)).address; - expect((await envelopeOf(await command(bootRig, "boot", 2))).result).toMatchObject({ - ok: true, - }); - await advance(bootRig, 6_000, 1_000); - await expect.poll(() => fleetLeases(bootRig)).toEqual([]); - const afterExpiry = bootMember.fake.rpcRequests("agent_device.command").length; - expectEndedBySimlock( - await envelopeOf(await command(bootRig, "boot", 2, {}, { flags: { udid: address } })), - ); - expect(bootMember.fake.rpcRequests("agent_device.command")).toHaveLength(afterExpiry); - }); - - it("a command with no device selector naming a lease id another token holds, and one the gateway does not know, answer `refused: lease is not one of yours on this gateway` with status 403, and no worker receives it; with `--udid`, the same command answers `refused: simlock picks the device`, as on a worker", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - const own = await envelopeOf(await command(rig, "snapshot", 1)); - expect(own.result).toMatchObject({ ok: true }); - const received = member.fake.rpcRequests("agent_device.command").length; - - const other = await command(rig, "snapshot", 1, { token: secrets.two }); - // why: within 60 s of the gateway starting, a lease it does not know answers "try again" (#517); past that, a refusal. - rig.clock.advance(60_000); - const unknown = await command(rig, "snapshot", 9); - - expect(other.status).toBe(403); - expect((await envelopeOf(other)).error?.message).toBe( - `refused: lease ${hex(1)} is not one of yours on this gateway`, - ); - expect(unknown.status).toBe(403); - expect((await envelopeOf(unknown)).error?.message).toBe( - `refused: lease ${hex(9)} is not one of yours on this gateway`, - ); - const picked = await envelopeOf( - await command(rig, "snapshot", 1, { token: secrets.two }, { flags: { udid: "ABC" } }), - ); - expect(picked.error?.message).toBe("refused: simlock picks the device"); - expect(member.fake.rpcRequests("agent_device.command").length).toBe(received); - }); - - it("after `simlock release --all`, once the fake clock has moved the gateway's `lease.maxTtlMs` plus 1 s past the index's removal of the lease, the link's memory no longer holds it, and a heartbeat for it answers `refused: lease is not one of yours on this gateway`; one second before that, the heartbeat is still relayed", async () => { - const options = { defaultTtlMs: 60_000, maxTtlMs: 120_000 } as const; - const early = await setUp(options); - const earlyMember = first(early); - await grantedOnGateway(early, 1, { ttlMs: 30_000 }); - await early.gateway.dispatch("lease.release-all", {}, adminSession); - await expect.poll(async () => (await fleetLeases(early)).length).toBe(0); - await advance(early, 119_000, 10_000); - const received = heartbeatsReceived(earlyMember); - - expectEndedBySimlock(await envelopeOf(await heartbeat(early, 1))); - expect(heartbeatsReceived(earlyMember)).toBe(received + 1); - - const late = await setUp(options); - const lateMember = first(late); - await grantedOnGateway(late, 1, { ttlMs: 30_000 }); - await late.gateway.dispatch("lease.release-all", {}, adminSession); - await expect.poll(async () => (await fleetLeases(late)).length).toBe(0); - await advance(late, 121_000, 10_000); - const lateReceived = heartbeatsReceived(lateMember); - - const answer = await envelopeOf(await heartbeat(late, 1)); - - expect(answer.error?.message).toBe( - `refused: lease ${hex(1)} is not one of yours on this gateway`, - ); - expect(heartbeatsReceived(lateMember)).toBe(lateReceived); - }); - - it("a request with no lease id other than `/health` is refused with `refused: this agent-device is too old to use through a gateway; update it`", async () => { - const rig = await setUp(); - const member = first(rig); - expect((await send(rig, "/health")).status).toBe(200); - const before = member.fake.requests.length; - - const bare = await envelopeOf(await command(rig, "snapshot", undefined)); - const upload = await send(rig, "/upload/preflight", { body: "{}" }); - - expect(bare.error?.message).toBe(tooOld); - expect(await upload.text()).toContain(tooOld); - expect(member.fake.requests.length).toBe(before); - }); - - it("`capabilities.agent-device.leaseTtlCapMs` on the gateway caps the TTL of a lease made through it", async () => { - const rig = await setUp({ leaseTtlCapMs: 10_000 }); - const member = first(rig); - - await grantedOnGateway(rig, 1, { ttlMs: 300_000 }); - - expect((await workerLeaseFor(member, 1)).ttlMs).toBe(10_000); - await expect.poll(async () => (await fleetLeases(rig))[0]?.ttlMs).toBe(10_000); - }); - - it("stopping the gateway while an allocate waits ends its lease request `cancelled`, not `DAEMON_STOPPING`, and a running command's renewals stop", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 1 }] }); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 30_000 }, { token: secrets.two }); - await runningCommand(rig, member, 1, { token: secrets.two }); - const tap = tapDispatch(rig.gateway); - await advance(rig, 20_000, 5_000); - const renewedBefore = tap.operations.filter((name) => name === "lease.renew").length; - const second = track(allocate(rig, 2)); - await expect.poll(() => waiting(rig)).toEqual([hex(2)]); - - await rig.gateway.stop("test"); - await second.promise.catch(() => undefined); - const afterStop = tap.operations.length; - await advance(rig, 60_000, 10_000); - - expect(renewedBefore).toBeGreaterThan(0); - expect(tap.operations.length).toBe(afterStop); - const lines = (await readHistoryEvents(rig.directory)) as Array<{ - event?: string; - payload?: Record; - }>; - expect( - lines.filter( - (entry) => - entry.event === "lease.rejected" && - entry.payload?.["requester"] === hex(2) && - entry.payload["reason"] === "cancelled", - ), - ).toHaveLength(1); - member.fake.releaseCommand(); - }); - - it("a heartbeat, a command, a release and an allocate whose relay fails with `RELAY_TARGET_MISSING` answer the notAvailable reply with that message, and the failed allocate leaves no lease", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - const missing = `${notAvailable}no relay target agent-device on this worker`; - expect((await envelopeOf(await command(rig, "snapshot", 1))).result).toMatchObject({ - ok: true, - }); - relayFailsWith(rig, "RELAY_TARGET_MISSING", "no relay target agent-device on this worker"); - - const beat = await heartbeat(rig, 1); - const commanded = await command(rig, "snapshot", 1); - const released = await release(rig, 1); - const made = await allocate(rig, 2); - - expect((await envelopeOf(beat)).error?.message).toBe(missing); - expect(commanded.status).toBe(503); - expect((await envelopeOf(commanded)).error?.message).toBe(missing); - expect((await envelopeOf(released)).error?.message).toBe(missing); - expect((await envelopeOf(made)).error?.message).toBe(missing); - await expect.poll(async () => (await workerLeases(member)).length).toBe(1); - }); - - it("a heartbeat for a lease the front's memory lacks, whose release relay then fails with `RELAY_TARGET_MISSING`, answers the notAvailable reply with that message and leaves the simlock lease", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - const held = member.fake.leases.get(hex(1)); - expect(held).toBeDefined(); - member.fake.leases.set(hex(2), { - ...(held as NonNullable), - leaseId: hex(2), - tenant: (held?.tenant ?? "").replace(hex(1), hex(2)), - }); - // why: a lease whose requester is not its own id is never adopted, so the front's memory lacks it however soon the worker's catalog arrives. - await rig.gateway.dispatch( - "lease.request", - { leaseId: hex(2), platform: "ios", requesterId: "someone-else", requires: ["agent-device"] }, - sessionFor(principals.one), - ); - interceptRelay(rig, async ({ request }) => { - const body = (await request.clone().json()) as { method?: string }; - if (body.method === "agent_device.lease.release") { - throw new DispatchError( - "RELAY_TARGET_MISSING", - "no relay target agent-device on this worker", - ); - } - }); - - const beat = await envelopeOf(await heartbeat(rig, 2)); - - expect(beat.error?.message).toBe(`${notAvailable}no relay target agent-device on this worker`); - expect((await fleetLeases(rig)).map((lease) => lease.id)).toContain(hex(2)); - }); - - it("a lease the gateway's `lease.list` leaves out during devices, allocate, boot, renewal or heartbeat is not ended, and the next heartbeat renews it", async () => { - const during: Record = {}; - const triggers: Record Promise> = { - allocate: async (rig) => { - await grantedOnGateway(rig, 2); - }, - devices: async (rig) => { - await command(rig, "devices", undefined); - }, - heartbeat: async (rig) => { - const beat = await envelopeOf(await heartbeat(rig, 1)); - during.heartbeat = beat.result ?? beat.error?.data; - }, - boot: async (rig) => { - const answer = await envelopeOf(await command(rig, "boot", 1)); - during.boot = answer.error?.data; - }, - renewal: async (rig, member) => { - const running = await runningCommand(rig, member, 1); - await advance(rig, 10_000, 5_000); - member.fake.releaseCommand(); - await running.promise; - }, - }; - const outcomes: Record = {}; - - for (const [name, trigger] of Object.entries(triggers)) { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); - await listedByGateway(rig, 1); - // why: the front's own record of the deadline is only the grant's, so an allocate re-reads lease 1 only once that has passed; a heartbeat keeps the simlock lease alive meanwhile. - await advance(rig, 20_000, 5_000); - expect((await envelopeOf(await heartbeat(rig, 1, { ttlMs: 30_000 }))).result).toMatchObject({ - ok: true, - }); - await advance(rig, 15_000, 5_000); - const listing = leaveOutOfNextListing(rig.gateway, hex(1)); - - await trigger(rig, member); - expect(listing.cut()).toBe(1); - await advance(rig, 1_000, 1_000); - const answer = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 60_000 })); - - outcomes[name] = { - answer: answer.result ?? answer.error?.data, - cut: listing.cut(), - leftMs: ((await deadlineOf(member, 1)) ?? Number.NaN) - rig.clock.now(), - }; - await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); - } - - const live = { answer: expect.objectContaining({ ok: true }), cut: 1, leftMs: 60_000 }; - expect(outcomes).toEqual({ - allocate: live, - boot: live, - devices: live, - heartbeat: live, - renewal: live, - }); - // why: the command already ran on the worker, so the heartbeat that met the omission still answers its reply. - expect(during).toEqual({ - boot: expect.objectContaining({ - message: "could not check your leases; try again", - retriable: true, - }), - heartbeat: expect.objectContaining({ ok: true }), - }); - }); -}); - -const LEASE_HEADER = "x-agent-device-lease-id"; -const notYoursOnGateway = (n: number): string => - `refused: lease ${hex(n)} is not one of yours on this gateway`; -const notALeaseId = "refused: the x-agent-device-lease-id header is not a lease id"; - -function leased(n: number): Record { - return { [LEASE_HEADER]: hex(n) }; -} - -function holderOf(rig: Rig, n: number): Member { - const member = rig.members.find((candidate) => candidate.fake.leases.has(hex(n))); - if (member === undefined) throw new Error(`no worker holds the agent-device lease ${hex(n)}`); - return member; -} - -// why: the gateway lists a lease only once its worker's next view reaches it, which can trail the grant. -async function listedByGateway(rig: Rig, count: number): Promise { - await expect.poll(async () => (await fleetLeases(rig)).length).toBe(count); -} - -// why: a worker with room for one lease each puts two leases on two workers. -async function twoLeasesOnTwoWorkers(): Promise { - const rig = await setUp({ - workers: [ - { label: "w1", maxRunning: 1 }, - { label: "w2", maxRunning: 1 }, - ], - }); - await grantedOnGateway(rig, 1); - await grantedOnGateway(rig, 2); - await listedByGateway(rig, 2); - expect(holderOf(rig, 1)).not.toBe(holderOf(rig, 2)); - return rig; -} - -function addressOfLease(rig: Rig, n: number): string { - const address = holderOf(rig, n).fake.leases.get(hex(n))?.address; - expect(address).toBeDefined(); - return address as string; -} - -function pathsReceived(member: Member): string[] { - return member.fake.requests.map((request) => request.path); -} - -async function listedNames(response: Response): Promise { - const answer = await envelopeOf(response); - const data = answer.result?.["data"] as { devices?: { name?: string }[] } | undefined; - expect(Array.isArray(data?.devices), "expected a devices list in result.data").toBe(true); - return (data?.devices ?? []).map((device) => String(device.name)).sort(); -} - -describe("the agent-device front, through a gateway: the lease header", { timeout: 60_000 }, () => { - it("through the gateway, an upload preflight with the header naming the token's lease on W2 reaches W2's agent-device daemon and not W1's", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const target = holderOf(rig, 2); - const other = holderOf(rig, 1); - const before = pathsReceived(other); - - const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - - expect(preflight.status).toBe(200); - expect(pathsReceived(target)).toContain("/upload/preflight"); - expect(target.fake.requests.find((seen) => seen.path === "/upload/preflight")?.tenant).toBe( - await tenantOf(target, 2), - ); - expect(pathsReceived(other)).toEqual(before); - }); - - it("the preflight's ticket URL names the gateway's origin, its `upload.headers` carry the caller's own `authorization`, W2 received its own daemon's token in `authorization` and `x-agent-device-token` and none of the caller's credentials, and a `PUT` to it with the lease header and those headers reaches W2", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const target = holderOf(rig, 2); - - const preflight = await send(rig, "/upload/preflight", { - body: "{}", - headers: { ...leased(2), "x-agent-device-token": "caller-token" }, - }); - - const ticket = ( - (await preflight.json()) as { upload: { url: string; headers: Record } } - ).upload; - expect(ticket.url.startsWith(`${rig.origin}/v1/agent-device/upload/direct/`)).toBe(true); - const named = (name: string) => - Object.entries(ticket.headers).find(([key]) => key.toLowerCase() === name)?.[1]; - expect(named("authorization")).toBe(`Bearer ${secrets.one}`); - expect(named("x-agent-device-token")).toBe("caller-token"); - const seen = target.fake.preflightHeaders[0] ?? {}; - expect(seen["authorization"]).toBe(`Bearer ${target.fake.token}`); - // why: the proxy sends its own daemon token in both headers, never the caller's. - expect(seen["x-agent-device-token"]).toBe(target.fake.token); - expect(Object.values(seen).join("\n")).not.toContain(secrets.one); - expect(Object.values(seen).join("\n")).not.toContain("caller-token"); - - const put = await fetch(ticket.url, { - body: "artifact bytes", - headers: { ...ticket.headers, ...leased(2) }, - method: "PUT", - }); - - expect(put.status).toBe(200); - expect(target.fake.uploads).toEqual(["artifact bytes"]); - expect(holderOf(rig, 1).fake.uploads).toEqual([]); - }); - - it("a finalize, a `POST /upload`, a `GET artifacts/` and a diagnostics `GET`, each with the header, reach the lease's worker", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const target = holderOf(rig, 2); - const other = holderOf(rig, 1); - const before = pathsReceived(other); - - await send(rig, "/upload/finalize", { body: "{}", headers: leased(2) }); - // why: the proxy confines diagnostics to sessions under the caller's tenant, which the gateway names. - const tenant = target.fake.requests.at(-1)?.tenant; - expect(tenant).toBeDefined(); - const diagnostics = `/sessions/${String(tenant)}:s1/requests/r1/diagnostics`; - await send(rig, "/upload", { body: "{}", headers: leased(2) }); - await send(rig, "/artifacts/a.png", { headers: leased(2) }); - await send(rig, diagnostics, { headers: leased(2) }); - - const paths = pathsReceived(target); - expect(paths).toContain("/upload/finalize"); - expect(paths).toContain("/upload"); - expect(paths).toContain("/artifacts/a.png"); - expect(paths).toContain(diagnostics); - expect(pathsReceived(other)).toEqual(before); - }); - - it("through the gateway, the same requests without the header, or with an empty one, and a `GET artifacts` list, are refused with `refused: this agent-device is too old to use through a gateway; update it`", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const diagnostics = `/sessions/${await tenantOf(holderOf(rig, 2), 2)}:s1/requests/r1/diagnostics`; - const calls: readonly { path: string; body?: string }[] = [ - { body: "{}", path: "/upload/preflight" }, - { body: "{}", path: "/upload/finalize" }, - { body: "{}", path: "/upload" }, - { path: "/artifacts/a.png" }, - { path: diagnostics }, - ]; - for (const call of calls) { - const accepted = await send(rig, call.path, { - ...(call.body === undefined ? {} : { body: call.body }), - headers: leased(2), - }); - expect(await accepted.text()).not.toContain(tooOld); - expect(pathsReceived(holderOf(rig, 2))).toContain(call.path); - } - const before = rig.members.map((member) => member.fake.requests.length); - - for (const call of calls) { - for (const headers of [{}, { [LEASE_HEADER]: "" }]) { - const answer = await send(rig, call.path, { - ...(call.body === undefined ? {} : { body: call.body }), - headers, - }); - expect(await answer.text()).toContain(tooOld); - } - } - const list = await send(rig, "/artifacts"); - expect(await list.text()).toContain(tooOld); - - expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); - }); - - it("through the gateway, a header naming another token's lease is refused `refused: lease is not one of yours on this gateway` with status 403, and no agent-device daemon receives it", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const own = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - expect(own.status).toBe(200); - const before = rig.members.map((member) => member.fake.requests.length); - - const other = await send(rig, "/upload/preflight", { - body: "{}", - headers: leased(2), - token: secrets.two, - }); - - expect(other.status).toBe(403); - expect(await other.text()).toContain(notYoursOnGateway(2)); - expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); - }); - - it("through the gateway, a header of 40 characters, and one with upper-case hex, are refused `refused: the x-agent-device-lease-id header is not a lease id` with status 400, and the reply does not contain the value", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const accepted = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - expect(accepted.status).toBe(200); - const before = rig.members.map((member) => member.fake.requests.length); - - for (const value of ["a".repeat(40), `ABCDEF${"0".repeat(26)}`]) { - const answer = await send(rig, "/upload/preflight", { - body: "{}", - headers: { [LEASE_HEADER]: value }, - }); - const text = await answer.text(); - expect(answer.status).toBe(400); - expect(text).toContain(notALeaseId); - expect(text).not.toContain(value); - } - - expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); - }); - - it("through the gateway, a header naming the token's lease whose worker stopped being capable answers the notAvailable reply naming that reason", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const accepted = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - expect(accepted.status).toBe(200); - const target = holderOf(rig, 2); - await turn(rig, target, "unavailable"); - const before = pathsReceived(target); - - const answer = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - - expect(await answer.text()).toContain(`${notAvailable}${holderCannot}`); - expect(pathsReceived(target)).toEqual(before); - }); - - it("through the gateway, `/health` with a header naming another token's lease answers the proxy's health", async () => { - const rig = await twoLeasesOnTwoWorkers(); - - const health = await send(rig, "/health", { headers: leased(2), token: secrets.two }); - - expect(health.status).toBe(200); - expect(await health.json()).toMatchObject({ service: "agent-device-proxy" }); - }); - - it("through the gateway, an upload naming the token's lease after `simlock release --all` reaches that lease's worker", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const target = holderOf(rig, 2); - await rig.gateway.dispatch("lease.release-all", {}, adminSession); - await expect.poll(async () => (await fleetLeases(rig)).length).toBe(0); - const before = pathsReceived(target).filter((path) => path === "/upload/preflight").length; - - const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(2) }); - - expect(await preflight.text()).not.toContain(notYoursOnGateway(2)); - expect(pathsReceived(target).filter((path) => path === "/upload/preflight").length).toBe( - before + 1, - ); - }); - - it("through the gateway, `devices` with no lease held answers an empty list and no agent-device daemon receives it", async () => { - const rig = await setUp({ workers: [{ label: "w1" }, { label: "w2" }] }); - for (const member of rig.members) member.fake.listDevices([{ id: "stray", name: "stray" }]); - - const answer = await envelopeOf(await command(rig, "devices", undefined)); - - expect(answer.result).toMatchObject({ data: { devices: [] }, ok: true }); - for (const member of rig.members) { - expect(member.fake.rpcRequests("agent_device.command")).toEqual([]); - } - }); - - it("through the gateway, `devices` with leases on W1 and W2 answers both devices, each once, and no device of another token", async () => { - const rig = await twoLeasesOnTwoWorkers(); - for (const [index, n] of [1, 2].entries()) { - holderOf(rig, n).fake.listDevices([ - { id: addressOfLease(rig, n), name: `mine-${String(index)}` }, - { id: "another-token-device", name: "foreign" }, - ]); - } - - expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ - "mine-0", - "mine-1", - ]); - }); - - it("through the gateway, `devices` with leases on W1 and W2 whose emulators are both `emulator-5554` answers both, each from its own worker", async () => { - const rig = await twoLeasesOnTwoWorkers(); - expect(addressOfLease(rig, 1)).toBe(addressOfLease(rig, 2)); - holderOf(rig, 1).fake.listDevices([{ id: addressOfLease(rig, 1), name: "from-first" }]); - holderOf(rig, 2).fake.listDevices([{ id: addressOfLease(rig, 2), name: "from-second" }]); - - expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ - "from-first", - "from-second", - ]); - }); - - it("through the gateway, `devices` with two leases on one worker answers both devices, each once", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); - await grantedOnGateway(rig, 1); - await grantedOnGateway(rig, 2); - await listedByGateway(rig, 2); - const member = first(rig); - member.fake.listDevices([ - { id: addressOfLease(rig, 1), name: "first" }, - { id: addressOfLease(rig, 2), name: "second" }, - { id: "another-token-device", name: "foreign" }, - ]); - - expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ - "first", - "second", - ]); - }); - - it("through the gateway, `devices` with two leases on one worker, sent with a `meta.requestId` as the agent-device CLI does, answers both devices", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); - await grantedOnGateway(rig, 1); - await grantedOnGateway(rig, 2); - await listedByGateway(rig, 2); - const member = first(rig); - member.fake.listDevices([ - { id: addressOfLease(rig, 1), name: "first" }, - { id: addressOfLease(rig, 2), name: "second" }, - ]); - member.fake.holdCommand(); - const answer = track( - command(rig, "devices", undefined, {}, { meta: { requestId: "req-devices-1" } }), - ); - await expect.poll(() => member.fake.rpcRequests("agent_device.command").length).toBe(2); - member.fake.releaseCommand(); - - expect(await listedNames(await answer.promise)).toEqual(["first", "second"]); - }); - - it("through the gateway, `devices` leaves out a lease that was released, and a lease the gateway's `lease.list` omits from that one answer while still holding it", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 3 }] }); - for (const n of [1, 2, 3]) await grantedOnGateway(rig, n); - await listedByGateway(rig, 3); - first(rig).fake.listDevices( - [1, 2, 3].map((n) => ({ id: addressOfLease(rig, n), name: `lease-${String(n)}` })), - ); - expect((await envelopeOf(await release(rig, 1))).result).toMatchObject({ ok: true }); - const listing = leaveOutOfNextListing(rig.gateway, hex(2)); - - expect(await listedNames(await command(rig, "devices", undefined))).toEqual(["lease-3"]); - expect(listing.cut()).toBe(1); - expect(await listedNames(await command(rig, "devices", undefined))).toEqual([ - "lease-2", - "lease-3", - ]); - }); - - it("through the gateway, `devices` after `simlock release --all` on the gateway leaves the ended lease out and ends it for the front, so the next heartbeat answers the ended-by-simlock reply without reaching the worker", async () => { - const rig = await setUp({ workers: [{ label: "w1", maxRunning: 2 }] }); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - first(rig).fake.listDevices([{ id: addressOfLease(rig, 1), name: "lease-1" }]); - await rig.gateway.dispatch("lease.release-all", {}, adminSession); - - expect(await listedNames(await command(rig, "devices", undefined))).toEqual([]); - const received = heartbeatsReceived(first(rig)); - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - expect(heartbeatsReceived(first(rig))).toBe(received); - }); - - it("through the gateway, `devices` when W2's agent-device answers an error passes that error on", async () => { - const rig = await twoLeasesOnTwoWorkers(); - holderOf(rig, 1).fake.listDevices([{ id: addressOfLease(rig, 1), name: "first" }]); - const second = holderOf(rig, 2).fake; - second.answerDevicesWith( - JSON.stringify({ - error: { - code: -32000, - data: { code: "COMMAND_FAILED", message: "inventory down" }, - message: "inventory down", - }, - id: 1, - jsonrpc: "2.0", - }), - ); - - const failed = await envelopeOf(await command(rig, "devices", undefined)); - - expect(failed.error?.message).toBe("inventory down"); - }); - - it("through the gateway, `devices` whose lease check fails answers `could not check your leases; try again`, not an empty list, and no daemon receives the command", async () => { - const rig = await twoLeasesOnTwoWorkers(); - for (const n of [1, 2]) { - holderOf(rig, n).fake.listDevices([ - { id: addressOfLease(rig, n), name: `mine-${String(n)}` }, - ]); - } - const tap = tapDispatch(rig.gateway); - tap.rejectNext("lease.list"); - const before = rig.members.map( - (member) => member.fake.rpcRequests("agent_device.command").length, - ); - - const answer = await envelopeOf(await command(rig, "devices", undefined)); - - expect(answer.error?.data).toMatchObject({ - details: { reason: "SIMLOCK_FAILED" }, - retriable: true, - }); - expect(answer.error?.message).toBe("could not check your leases; try again"); - expect( - rig.members.map((member) => member.fake.rpcRequests("agent_device.command").length), - ).toEqual(before); - }); - - it("through the gateway, `devices` when W2 answers a body with no list aborts the forward still waiting on W1", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const slow = holderOf(rig, 1).fake; - slow.holdCommand(); - holderOf(rig, 2).fake.answerDevicesWith( - JSON.stringify({ id: 1, jsonrpc: "2.0", result: { data: {}, ok: true } }), - ); - - const answer = await command(rig, "devices", undefined); - - expect(answer.status).toBe(502); - expect((await envelopeOf(answer)).error?.message).toBe( - `could not read the device list from the worker holding lease ${hex(2)}`, - ); - await expect.poll(() => slow.abortedCommands).toEqual(["devices"]); - }); - - it("through the gateway, a CLI that hangs up during `devices` aborts every forward", async () => { - const rig = await twoLeasesOnTwoWorkers(); - for (const member of rig.members) member.fake.holdCommand(); - const hangUp = new AbortController(); - const held = track(command(rig, "devices", undefined, { signal: hangUp.signal })); - await expect - .poll(() => - rig.members.map((member) => member.fake.rpcRequests("agent_device.command").length), - ) - .toEqual([1, 1]); - - hangUp.abort(); - await held.promise.catch(() => undefined); - - await expect - .poll(() => rig.members.map((member) => member.fake.abortedCommands)) - .toEqual([["devices"], ["devices"]]); - }); - - it("through the gateway, `release_materialized_paths` and an `install_from_source` naming the token's lease are refused `refused: unknown method `, and no worker sees either", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const before = rig.members.map((member) => member.fake.requests.length); - - const released = await envelopeOf( - await rpc(rig, "agent_device.release_materialized_paths", { - materializationId: "m-1", - }), - ); - const installed = await envelopeOf( - await rpc(rig, "agent_device.install_from_source", { - meta: { leaseId: hex(2) }, - platform: "ios", - source: { kind: "url", url: "https://example.com/App.zip" }, - }), - ); - - expect(released.error?.message).toBe( - "refused: unknown method agent_device.release_materialized_paths", - ); - expect(installed.error?.message).toBe( - "refused: unknown method agent_device.install_from_source", - ); - expect(rig.members.map((member) => member.fake.requests.length)).toEqual(before); - }); }); - -const unreachableReason = - "the worker holding this device is not connected to the gateway; try again"; -const unreachableMessage = `${notAvailable}${unreachableReason}`; -const picksTheDevice = "refused: simlock picks the device"; -const gatewayWindowMs = 60_000; -const hourMs = 3_600_000; - -function expectUnreachable(answer: Envelope): void { - expect(answer.result).toBeUndefined(); - expect(answer.error).toMatchObject({ - data: { - details: { reason: "WORKER_UNREACHABLE" }, - message: unreachableMessage, - retriable: true, - }, - message: unreachableMessage, - }); -} - -async function expectUnreachablePlain(response: Response): Promise { - expect(response.status).toBe(503); - const body = (await response.json()) as Record; - expect(body["error"]).toBe(unreachableMessage); - expect(body).toMatchObject({ reason: "WORKER_UNREACHABLE", retriable: true }); -} - -async function expectUnreachableEverywhere(rig: Rig, n: number): Promise { - for (const response of [ - await heartbeat(rig, n), - await command(rig, "snapshot", n), - await release(rig, n), - ]) { - expect(response.status).toBe(503); - expectUnreachable(await envelopeOf(response)); - } - await expectUnreachablePlain( - await send(rig, "/upload/preflight", { body: "{}", headers: leased(n) }), - ); -} - -// why: the gateway closes the worker's socket as a network drop does; the worker redials only when its clock moves. -async function dropUplink(rig: Rig, member: Member): Promise { - const view = await viewOf(rig, member.label); - expect(view?.id).toBeDefined(); - await rig.uplinks.closeWorkerLink(view?.id as string); - await vi.waitFor( - async () => expect((await viewOf(rig, member.label))?.connection).toBe("disconnected"), - { timeout: 10_000 }, - ); -} - -async function untilConnected(rig: Rig, member: Member): Promise { - await advance(rig, 2_000, 500); - await untilRead(rig, member); -} - -async function commandWorks(rig: Rig, member: Member, n: number): Promise { - const before = commandsReceived(member); - const answer = await envelopeOf(await command(rig, "snapshot", n)); - expect(answer.error).toBeUndefined(); - expect(answer.result).toBeDefined(); - expect(commandsReceived(member)).toBe(before + 1); -} - -describe( - "the agent-device front, through a gateway: an unreachable worker", - { - timeout: 90_000, - }, - () => { - it("with the device's worker disconnected, a command, a heartbeat and a release each answer 503 with `details.reason` `WORKER_UNREACHABLE` and the `workerUnreachable` message, and no agent-device daemon receives them", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await dropUplink(rig, member); - const received = member.fake.requests.length; - - for (const response of [ - await command(rig, "snapshot", 1), - await heartbeat(rig, 1), - await release(rig, 1), - ]) { - expect(response.status).toBe(503); - expectUnreachable(await envelopeOf(response)); - } - - expect(member.fake.requests.length).toBe(received); - }); - - it("with the device's worker disconnected, an upload with the lease header answers 503 with that message, reason and retry flag, and `devices` answers the RPC reply", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await dropUplink(rig, member); - - await expectUnreachablePlain( - await send(rig, "/upload/preflight", { body: "{}", headers: leased(1) }), - ); - const devices = await command(rig, "devices", undefined); - - expect(devices.status).toBe(503); - expectUnreachable(await envelopeOf(devices)); - }); - - it("with one of the token's two workers reconnected but not yet read and the other connected, `devices` answers `workerUnreachable()` and asks neither worker", async () => { - const rig = await twoLeasesOnTwoWorkers(); - const gone = holderOf(rig, 1); - const kept = holderOf(rig, 2); - await listedByGateway(rig, 2); - await dropUplink(rig, gone); - gone.latency["listCatalog"] = slowCatalogMs; - await advance(rig, 2_000, 500); - await vi.waitFor( - async () => { - const view = await viewOf(rig, gone.label); - expect(view?.connection).toBe("connected"); - expect(view?.catalogReadAt).toBeUndefined(); - }, - { timeout: 10_000 }, - ); - const received = [gone, kept].map((member) => member.fake.requests.length); - - const devices = await command(rig, "devices", undefined); - - expect(devices.status).toBe(503); - expectUnreachable(await envelopeOf(devices)); - expect([gone, kept].map((member) => member.fake.requests.length)).toEqual(received); - }); - - it("while the worker is disconnected, the simlock lease is not released, and the front's entry is not marked ended", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - const tap = tapDispatch(rig.gateway); - await dropUplink(rig, member); - - expectUnreachable(await envelopeOf(await heartbeat(rig, 1))); - expectUnreachable(await envelopeOf(await release(rig, 1))); - - expect(tap.operations).not.toContain("lease.release"); - expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - await untilConnected(rig, member); - const beat = await envelopeOf(await heartbeat(rig, 1)); - expect(beat.error).toBeUndefined(); - expect(beat.result).toMatchObject({ ok: true }); - }); - - it("a command answered WORKER_UNREACHABLE after an uplink drop stops renewing its lease after reconnect without releasing it", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); - await listedByGateway(rig, 1); - member.fake.keepCommandsAfterAbort(); - const running = await runningCommand(rig, member, 1); - const before = await deadlineOf(member, 1); - expect(before).toBeTypeOf("number"); - const tap = tapDispatch(rig.gateway); - - try { - await dropUplink(rig, member); - - await vi.waitFor(() => expect(running.done()).toBe(true), { timeout: 5_000 }); - const answer = await running.promise; - expect(answer.status).toBe(503); - expectUnreachable(await envelopeOf(answer)); - await untilConnected(rig, member); - await advance(rig, 20_000, 1_000); - - expect(tap.operations).not.toContain("lease.release"); - expect(await deadlineOf(member, 1)).toBe(before); - } finally { - member.fake.releaseCommand(); - } - }); - - it("right after the worker reconnects and before its view is refreshed, a command answers `workerUnreachable()`; once it is refreshed, within the TTL, the next command reaches it, and the next heartbeat renews the lease", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); - await listedByGateway(rig, 1); - await dropUplink(rig, member); - member.latency["listCatalog"] = slowCatalogMs; - await advance(rig, 2_000, 500); - await vi.waitFor( - async () => { - const view = await viewOf(rig, member.label); - expect(view?.connection).toBe("connected"); - expect(view?.catalogReadAt).toBeUndefined(); - }, - { timeout: 10_000 }, - ); - - expectUnreachable(await envelopeOf(await command(rig, "snapshot", 1))); - - await advance(rig, 9_000, 1_000); - await untilRead(rig, member); - await commandWorks(rig, member, 1); - const beat = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); - expect(beat.result).toMatchObject({ ok: true }); - expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); - }); - - it("a worker that stays disconnected past the TTL: once it reconnects, the next heartbeat answers `LEASE_ENDED_BY_HOST`", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await dropUplink(rig, member); - - // why: one jump fires the worker's redial and the lease's expiry together, before the new socket is up. - await sleep(50); - rig.clock.advance(130_000); - await untilConnected(rig, member); - - expectEndedBySimlock(await envelopeOf(await heartbeat(rig, 1))); - }); - - it("a lease known only from the link's memory (after `simlock release --all`) whose worker then disconnects answers `workerUnreachable()`", async () => { - // why: a worker clock of its own, so moving the gateway's clock past the start window does not make the worker redial. - const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); - const member = first(rig); - await grantedOnGateway(rig, 1); - await rig.gateway.dispatch("lease.release-all", {}, adminSession); - await expect.poll(async () => (await fleetLeases(rig)).length).toBe(0); - await dropUplink(rig, member); - // why: past the 60 s start window an unplaced lease is not unreachable, so only the link's memory can place this one. - rig.clock.advance(70_000); - - expectUnreachable(await envelopeOf(await heartbeat(rig, 1))); - }); - - it("an unreachable command's final renewal fails while disconnected; after reconnect only a new heartbeat renews the retained lease", async () => { - // why: a worker clock of its own, so moving the gateway's clock does not make the worker redial. - const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 30_000 }); - await listedByGateway(rig, 1); - const running = await runningCommand(rig, member, 1); - const tap = tapDispatch(rig.gateway); - const before = await deadlineOf(member, 1); - rig.clock.advance(1_000); - await dropUplink(rig, member); - expectUnreachable(await envelopeOf(await running.promise)); - - rig.clock.advance(10_000); - await sleep(50); - - expect(tap.operations).toContain("lease.renew"); - expect(tap.operations).not.toContain("lease.release"); - expect((await fleetLeases(rig)).map((lease) => lease.id)).toEqual([hex(1)]); - await untilConnected(rig, member); - await advance(rig, 10_000, 5_000); - await sleep(50); - expect(before).toBe(31_000); - expect(await deadlineOf(member, 1)).toBe(before); - expect((await envelopeOf(await heartbeat(rig, 1))).result).toMatchObject({ ok: true }); - expect(await deadlineOf(member, 1)).toBe(member.clock.now() + 30_000); - member.fake.releaseCommand(); - }); - - it("an allocate by the same token while its other lease's worker is disconnected does not mark that lease ended", async () => { - const rig = await setUp({ - workers: [ - { label: "w1", maxRunning: 1 }, - { label: "w2", maxRunning: 1 }, - ], - }); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - const holder = holderOf(rig, 1); - await dropUplink(rig, holder); - - await grantedOnGateway(rig, 2); - expect((await fleetLeases(rig)).map((lease) => lease.id)).toContain(hex(1)); - await untilConnected(rig, holder); - - const beat = await envelopeOf(await heartbeat(rig, 1)); - expect(beat.error).toBeUndefined(); - expect(beat.result).toMatchObject({ ok: true }); - }); - }, -); - -describe( - "the agent-device front, through a gateway: after a gateway restart", - { - timeout: 90_000, - }, - () => { - it("after a gateway restart and the worker's reconnect, the next heartbeat for the lease reaches the worker's agent-device daemon and renews the simlock lease, and a following command works", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, { ttlMs: 60_000 }); - await listedByGateway(rig, 1); - await restarted(rig, member, 1); - const received = heartbeatsReceived(member); - - const beat = await envelopeOf(await heartbeat(rig, 1, { ttlMs: 120_000 })); - - expect(beat.result).toMatchObject({ ok: true }); - expect(heartbeatsReceived(member)).toBe(received + 1); - expect(await deadlineOf(member, 1)).toBe(rig.clock.now() + 120_000); - await commandWorks(rig, member, 1); - }); - - it("after a gateway restart, a command, a release, `devices` and an upload with the lease header, each as the token's first request, work on the adopted lease", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - - await restarted(rig, member, 1); - await commandWorks(rig, member, 1); - - await restarted(rig, member, 1); - member.fake.listDevices([{ id: addressOfLease(rig, 1), name: "adopted" }]); - expect(await listedNames(await command(rig, "devices", undefined))).toEqual(["adopted"]); - - await restarted(rig, member, 1); - const preflight = await send(rig, "/upload/preflight", { body: "{}", headers: leased(1) }); - expect(preflight.status).toBe(200); - expect(pathsReceived(member)).toContain("/upload/preflight"); - - await restarted(rig, member, 1); - const released = await envelopeOf(await release(rig, 1)); - expect(released.result).toMatchObject({ ok: true, released: true }); - expect(await fleetLeases(rig)).toEqual([]); - }); - - it("after a gateway restart, an adopted lease on a TV model reads target `tv`, and `test --target tv` sharding uses it", async () => { - const rig = await setUp({ workers: [{ label: "w1", modelClass: "tv" }] }); - const member = first(rig); - await grantedOnGateway(rig, 1, { target: "tv" }); - await listedByGateway(rig, 1); - await restarted(rig, member, 1); - - const answer = await envelopeOf( - await command(rig, "test", 1, {}, { flags: { shardAll: 1, target: "tv" } }), - ); - - expect(answer.error).toBeUndefined(); - const sent = member.fake.rpcRequests("agent_device.command").at(-1)?.params; - const flags = (sent?.["flags"] ?? {}) as Record; - expect(flags["device"]).toBe(`simlock-${addressOfLease(rig, 1)}`); - }); - - it("after a gateway restart, a command naming a `deviceKey` on the adopted lease is accepted and sets it; a later command naming another `deviceKey` is refused `refused: simlock picks the device`", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await restarted(rig, member, 1); - - const accepted = await envelopeOf( - await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-a" } }), - ); - expect(accepted.error).toBeUndefined(); - const received = commandsReceived(member); - const other = await envelopeOf( - await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-b" } }), - ); - - expect(other.error?.message).toBe(picksTheDevice); - expect(commandsReceived(member)).toBe(received); - const same = await envelopeOf( - await command(rig, "snapshot", 1, {}, { meta: { deviceKey: "key-a" } }), - ); - expect(same.error).toBeUndefined(); - }); - - it("within 60 s of a gateway restart, before the worker reconnects, a heartbeat, a command, a release and an upload with the lease header for the lease each answer `workerUnreachable()`; after 60 s, with another capable worker connected, a heartbeat for it answers `refused: lease is not one of yours on this gateway`", async () => { - // why: a worker clock of its own, so the worker does not redial until the test moves it. - const rig = await setUp({ workers: [{ clockSkewMs: 0, label: "w1" }] }); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await rig.restart(); - const received = member.fake.requests.length; - - await expectUnreachableEverywhere(rig, 1); - rig.clock.advance(gatewayWindowMs - 1_000); - await expectUnreachableEverywhere(rig, 1); - - expect(member.fake.requests.length).toBe(received); - const other = await rig.addWorker({ label: "w2" }); - await other.daemon; - await untilRead(rig, other); - rig.clock.advance(2_000); - const beat = await envelopeOf(await heartbeat(rig, 1)); - expect(beat.error?.message).toBe(notYoursOnGateway(1)); - }); - - it("after a gateway restart, a lease of another token is not adopted, and a heartbeat naming it is refused", async () => { - const rig = await setUp(); - const member = first(rig); - await grantedOnGateway(rig, 1, {}, { token: secrets.two }); - await expect.poll(async () => (await fleetLeases(rig, principals.two)).length).toBe(1); - await rig.restart(); - await untilConnected(rig, member); - await expect.poll(async () => (await fleetLeases(rig, principals.two)).length).toBe(1); - const received = commandsReceived(member); - - const beat = await envelopeOf(await heartbeat(rig, 1)); - const devices = await envelopeOf(await command(rig, "devices", undefined)); - - expect(beat.error?.message).toBe(notYoursOnGateway(1)); - expect(devices.result).toMatchObject({ data: { devices: [] }, ok: true }); - expect(commandsReceived(member)).toBe(received); - }); - - it("after a gateway restart, a lease of this token whose requester is not its own id is not adopted", async () => { - const rig = await setUp(); - const member = first(rig); - await rig.gateway.dispatch( - "lease.request", - { platform: "ios", requesterId: "someone-else", requires: ["agent-device"] }, - sessionFor(principals.one), - ); - await restarted(rig, member, 1); - const received = commandsReceived(member); - - const devices = await envelopeOf(await command(rig, "devices", undefined)); - - expect(devices.result).toMatchObject({ data: { devices: [] }, ok: true }); - expect(commandsReceived(member)).toBe(received); - }); - - it("a lease this gateway process asked for whose allocate failed after the grant (the uplink dropped mid-allocate) is never adopted, and stays out of `devices`, also when the worker's clock is an hour behind or ahead of the gateway's", async () => { - for (const skew of [0, -hourMs, hourMs]) { - const rig = await setUp({ workers: [{ clockSkewMs: skew, label: "w1" }] }); - const member = first(rig); - member.fake.holdAllocate(); - const running = track(allocate(rig, 1)); - await expect.poll(() => allocatesReceived(member, 1).length).toBe(1); - await dropUplink(rig, member); - member.fake.letAllocateGo(); - await running.promise; - await untilConnected(rig, member); - const received = commandsReceived(member); - - const devices = await envelopeOf(await command(rig, "devices", undefined)); - - expect(devices.result, `skew ${String(skew)}`).toMatchObject({ - data: { devices: [] }, - ok: true, - }); - expect(commandsReceived(member)).toBe(received); - } - }); - - it("after a gateway restart, a lease made before it is adopted when the worker's clock is an hour behind or ahead of the gateway's", async () => { - for (const skew of [-hourMs, hourMs]) { - const rig = await setUp({ workers: [{ clockSkewMs: skew, label: "w1" }] }); - const member = first(rig); - await grantedOnGateway(rig, 1); - await listedByGateway(rig, 1); - await restarted(rig, member, 1); - - const beat = await envelopeOf(await heartbeat(rig, 1)); - - expect(beat.error, `skew ${String(skew)}`).toBeUndefined(); - expect(beat.result).toMatchObject({ ok: true }); - await commandWorks(rig, member, 1); - } - }); - }, -); diff --git a/src/agent-device/test-gateway-front.ts b/src/agent-device/test-gateway-front.ts new file mode 100644 index 000000000..17a836d19 --- /dev/null +++ b/src/agent-device/test-gateway-front.ts @@ -0,0 +1,64 @@ +import { expect } from "vitest"; +import { + Member, + Rig, + setUp, + envelopeOf, + hex, + grantedOnGateway, + fleetLeases, +} from "./test-gateway.js"; + +export const LEASE_HEADER = "x-agent-device-lease-id"; + +export const notYoursOnGateway = (n: number): string => + `refused: lease ${hex(n)} is not one of yours on this gateway`; + +export function leased(n: number): Record { + return { [LEASE_HEADER]: hex(n) }; +} + +export function holderOf(rig: Rig, n: number): Member { + const member = rig.members.find((candidate) => candidate.fake.leases.has(hex(n))); + if (member === undefined) throw new Error(`no worker holds the agent-device lease ${hex(n)}`); + return member; +} + +// why: the gateway lists a lease only once its worker's next view reaches it, which can trail the grant. +export async function listedByGateway(rig: Rig, count: number): Promise { + await expect.poll(async () => (await fleetLeases(rig)).length).toBe(count); +} + +// why: a worker with room for one lease each puts two leases on two workers. +export async function twoLeasesOnTwoWorkers(): Promise { + const rig = await setUp({ + workers: [ + { label: "w1", maxRunning: 1 }, + { label: "w2", maxRunning: 1 }, + ], + }); + await grantedOnGateway(rig, 1); + await grantedOnGateway(rig, 2); + await listedByGateway(rig, 2); + expect(holderOf(rig, 1)).not.toBe(holderOf(rig, 2)); + return rig; +} + +export function addressOfLease(rig: Rig, n: number): string { + const address = holderOf(rig, n).fake.leases.get(hex(n))?.address; + expect(address).toBeDefined(); + return address as string; +} + +export function pathsReceived(member: Member): string[] { + return member.fake.requests.map((request) => request.path); +} + +export async function listedNames(response: Response): Promise { + const answer = await envelopeOf(response); + const data = answer.result?.["data"] as { devices?: { name?: string }[] } | undefined; + expect(Array.isArray(data?.devices), "expected a devices list in result.data").toBe(true); + return (data?.devices ?? []).map((device) => String(device.name)).sort(); +} + +export const picksTheDevice = "refused: simlock picks the device"; From cef47ce822e753d2f0bee46c6740049b88054d7c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:30:52 +0100 Subject: [PATCH 48/83] test(agent-device): split agent-device-supervisor.test.ts by topic --- .../agent-device-supervisor-lifecycle.test.ts | 524 +++++++ .../agent-device-supervisor-phones.test.ts | 215 +++ .../agent-device-supervisor-state.test.ts | 335 ++++ .../host/agent-device-supervisor.test.ts | 1378 +---------------- .../host/test-agent-device-supervisor.ts | 377 +++++ 5 files changed, 1476 insertions(+), 1353 deletions(-) create mode 100644 src/agent-device/host/agent-device-supervisor-lifecycle.test.ts create mode 100644 src/agent-device/host/agent-device-supervisor-phones.test.ts create mode 100644 src/agent-device/host/agent-device-supervisor-state.test.ts create mode 100644 src/agent-device/host/test-agent-device-supervisor.ts diff --git a/src/agent-device/host/agent-device-supervisor-lifecycle.test.ts b/src/agent-device/host/agent-device-supervisor-lifecycle.test.ts new file mode 100644 index 000000000..fee819f8f --- /dev/null +++ b/src/agent-device/host/agent-device-supervisor-lifecycle.test.ts @@ -0,0 +1,524 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../../ports/index.js"; +import { + stateDir, + recordPath, + daemonJsonPath, + nodePath, + packageRoot, + internalEntry, + httpPort, + token, + Options, + arrange, + Harness, + run, + versionRun, + daemonSpawn, + ownLine, + record, + flush, + advanceBy, + advancing, + advanceUntil, + stateOf, + unavailable, + running, + commandsRun, + daemonSpawns, + warnings, + readJson, +} from "./test-agent-device-supervisor.js"; + +describe("AgentDeviceSupervisor reaping a daemon a killed simlock left", () => { + const recorded = { livePids: [900], record: record(900) }; + + it("sends SIGTERM to a live recorded pid whose line ends with the marker and holds the entry, removes the record once it is gone, and starts a new daemon", async () => { + const harness = await run({ + ...recorded, + commandLines: [[900, [ownLine()]]], + }); + + expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); + expect(harness.processes.isAlive(900)).toBe(false); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); + expect(stateOf(harness)).toBe("running"); + }); + + it("escalates to SIGKILL after 10 s when the recorded pid ignores SIGTERM and is still identified", async () => { + const harness = await arrange({ + ...recorded, + behaviours: new Map([[900, "ignores-sigterm"]]), + commandLines: [[900, [ownLine(), ownLine()]]], + }); + const startedAt = harness.clock.now(); + + await advancing(harness, harness.begin()); + + expect(harness.processes.signals).toEqual([ + { pid: 900, signal: "SIGTERM" }, + { pid: 900, signal: "SIGKILL" }, + ]); + expect(harness.clock.now() - startedAt).toBeGreaterThanOrEqual(10_000); + expect(stateOf(harness)).toBe("running"); + }); + + it("sends no SIGKILL to a pid whose second command line no longer passes the identity check", async () => { + const harness = await run({ + ...recorded, + behaviours: new Map([[900, "ignores-sigterm"]]), + commandLines: [[900, [ownLine(), "/usr/bin/vim notes.txt"]]], + }); + + expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); + }); + + it("keeps the record and reports start-failed naming the pid when it is still alive 2 s after SIGKILL", async () => { + const harness = await run({ + ...recorded, + behaviours: new Map([[900, "immortal"]]), + commandLines: [[900, [ownLine(), ownLine()]]], + expectations: [], + }); + + expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("900"); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); + expect(daemonSpawns(harness)).toEqual([]); + }); + + it("spawns with the resolved home as the marker when SIMLOCK_HOME is reached through a symlink, and a later start reaps that child", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/private/tmp/x"); + await filesystem.mkdirp("/tmp"); + filesystem.defineSymlink("/tmp/x", "/private/tmp/x"); + const first = await run({ + expectations: [versionRun(), daemonSpawn(internalEntry, {}, "/private/tmp/x")], + filesystem, + home: "/tmp/x", + }); + const childPid = first.children[0]?.pid as number; + expect(first.children[0]?.args).toEqual([internalEntry, "--simlock-home=/private/tmp/x"]); + + const second = await run({ + commandLines: [[childPid, [ownLine(internalEntry, "/private/tmp/x")]]], + expectations: [versionRun(), daemonSpawn(internalEntry, {}, "/private/tmp/x")], + filesystem, + home: "/tmp/x", + livePids: [childPid], + }); + + expect(second.processes.signals).toEqual([{ pid: childPid, signal: "SIGTERM" }]); + }); + + it("reaps the live pid daemon.json names when there is no record and the line ends with the marker", async () => { + const harness = await run({ + commandLines: [[901, [ownLine()]]], + livePids: [901], + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid: 901, token }), + ); + }, + }); + + expect(harness.processes.signals).toEqual([{ pid: 901, signal: "SIGTERM" }]); + }); + + it("does not read daemon.json and signals nothing when the state directory is a symlink", async () => { + const harness = await run({ + expectations: [versionRun()], + livePids: [901], + stateDirContents: async (filesystem) => { + await filesystem.mkdirp("/elsewhere"); + await filesystem.writeFileAtomic("/elsewhere/daemon.json", JSON.stringify({ pid: 901 })); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + }); + + expect(harness.processes.signals).toEqual([]); + expect(harness.processes.commandLineReads).toEqual([]); + expect(unavailable(harness).reason).toBe("state-dir"); + }); + + it.each([ + ["a dead recorded pid", [], undefined], + ["a command line of another command", [900], "/usr/bin/vim notes.txt"], + ["a line holding the entry but no marker", [900], `${nodePath} ${internalEntry}`], + ["a line with another home's marker", [900], ownLine(internalEntry, "/other/home")], + [ + "a line with this home's marker but another entry than the recorded one", + [900], + ownLine("/elsewhere/daemon.js"), + ], + ] as const)("removes the record and signals nothing for %s", async (_case, livePids, line) => { + // why: the spawn throws, so no new record can stand in for the old one: only a removal empties it. + const harness = await run({ + commandLines: line === undefined ? [] : [[900, [line]]], + livePids, + record: record(900), + spawnThrows: new Error("no spawn"), + }); + + expect(harness.processes.signals).toEqual([]); + expect(await harness.filesystem.exists(recordPath)).toBe(false); + }); + + it("signals nothing when daemon.json names another pid than the recorded one", async () => { + const harness = await run({ + commandLines: [[900, [ownLine()]]], + livePids: [900], + record: record(900), + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); + }, + }); + + expect(harness.processes.signals).toEqual([]); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); + expect(stateOf(harness)).toBe("running"); + }); + + it("removes the record when daemon.json names another pid than the recorded one", async () => { + const harness = await run({ + commandLines: [[900, [ownLine()]]], + livePids: [900], + record: record(900), + spawnThrows: new Error("no spawn"), + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); + }, + }); + + expect(await harness.filesystem.exists(recordPath)).toBe(false); + }); + + it("signals nothing, keeps the record, starts nothing and reports start-failed naming the pid when its command line cannot be read", async () => { + const harness = await run({ + commandLines: [[900, [undefined]]], + livePids: [900], + record: record(900), + }); + + expect(harness.processes.commandLineReads).toEqual([900]); + expect(harness.processes.signals).toEqual([]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("900"); + expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); + expect(commandsRun(harness).filter((call) => call[0] === nodePath)).toEqual([]); + }); +}); + +describe("AgentDeviceSupervisor after the daemon is running", () => { + async function runningFor( + options: Options, + ms: number, + ): Promise<{ harness: Harness; readyAt: number }> { + const harness = await run(options); + expect(stateOf(harness)).toBe("running"); + const readyAt = harness.clock.now(); + await advanceBy(harness, ms); + return { harness, readyAt }; + } + + it("starts a child that exited 60 s after ready again at once with the same environment, as generation 2, and warns with the exit code", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 60_000, + ); + const firstEnv = harness.runner.calls[1]?.options.env; + + harness.children[0]?.exit(42); + await flush(); + expect(stateOf(harness)).toBe("starting"); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + + expect(daemonSpawns(harness)).toHaveLength(2); + expect(harness.runner.calls[3]?.options.env).toEqual(firstEnv); + expect(running(harness).generation).toBe(2); + expect( + warnings(harness).some( + (line) => line.includes("agent-device (simulators) daemon pid") && line.includes("42"), + ), + ).toBe(true); + }); + + it("reports start-failed for a child that exits 5 s after ready, and starts it again only 30 s later", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 5_000, + ); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toMatch( + /agent-device exited .*9.* 5 s after it was ready/, + ); + await advanceBy(harness, 29_000); + expect(daemonSpawns(harness)).toHaveLength(1); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + expect(daemonSpawns(harness)).toHaveLength(2); + }); + + it("times the stable uptime from ready, not from the spawn: a child ready 10 s after the spawn that exits 62 s after it is start-failed", async () => { + const harness = await arrange({ + daemonJson: () => "none", + expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], + }); + void harness.begin(); + await advanceUntil(harness, () => harness.children.length === 1, 5_000); + const spawnedAt = harness.clock.now(); + await advanceBy(harness, 10_000); + await harness.filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid: harness.children[0]?.pid, token }), + ); + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + await advanceBy(harness, spawnedAt + 62_000 - harness.clock.now()); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(daemonSpawns(harness)).toHaveLength(1); + }); + + it("reports start-failed for a child that exits 59 s after ready instead of starting it at once", async () => { + const { harness } = await runningFor( + { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, + 59_000, + ); + + harness.children[0]?.exit(9); + await flush(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(daemonSpawns(harness)).toHaveLength(1); + }); + + it("retries a failed first start 30 s later and is running when that succeeds", async () => { + const harness = await arrange({ + daemonJson: (index) => (index === 0 ? "none" : "ready"), + expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], + }); + await advancing(harness, harness.begin()); + expect(unavailable(harness).reason).toBe("start-failed"); + const failedAt = harness.clock.now(); + + await advanceBy(harness, 29_000); + expect(daemonSpawns(harness)).toHaveLength(1); + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + + expect(daemonSpawns(harness)).toHaveLength(2); + expect(harness.clock.now() - failedAt).toBeGreaterThanOrEqual(30_000); + expect(harness.clock.now() - failedAt).toBeLessThanOrEqual(30_500); + expect(running(harness).generation).toBe(1); + }); + + it("retries a start that failed after a restart 30 s later", async () => { + const harness = await arrange({ + daemonJson: (index) => (index === 1 ? "none" : "ready"), + expectations: [ + versionRun(), + daemonSpawn(), + versionRun(), + daemonSpawn(), + versionRun(), + daemonSpawn(), + ], + }); + await advancing(harness, harness.begin()); + await advanceBy(harness, 60_000); + harness.children[0]?.exit(1); + await advanceUntil(harness, () => unavailableState(harness), 30_000); + expect(unavailable(harness).reason).toBe("start-failed"); + + await advanceUntil(harness, () => stateOf(harness) === "running", 60_000); + + expect(daemonSpawns(harness)).toHaveLength(3); + expect(stateOf(harness)).toBe("running"); + }); + + function unavailableState(harness: Harness): boolean { + return stateOf(harness) === "unavailable"; + } + + it.each([ + ["not-found", { baseEnvironment: { PATH: "/nowhere" }, expectations: [] }], + [ + "not-agent-device", + { + expectations: [versionRun()], + layout: async (filesystem: MemoryFilesystem) => + filesystem.rm(`${packageRoot}/package.json`), + }, + ], + ["wrong-version", { expectations: [versionRun("0.20.3\n")] }], + [ + "state-dir", + { + expectations: [versionRun()], + stateDirContents: async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/elsewhere"); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + }, + ], + ] satisfies [string, Options][])("does not retry a %s", async (reason, options) => { + const harness = await run(options); + expect(unavailable(harness).reason).toBe(reason); + const calls = harness.runner.calls.length; + + await advanceBy(harness, 120_000); + + expect(harness.runner.calls).toHaveLength(calls); + expect(unavailable(harness).reason).toBe(reason); + }); +}); + +describe("AgentDeviceSupervisor.stop", () => { + it("sends SIGTERM to the child, removes the record once it is gone, starts nothing more and leaves availability off", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid; + const calls = harness.runner.calls.length; + + await advancing(harness, harness.agentDevice.stop()); + + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + expect(harness.events.slice(-1)).toEqual([`exit ${pid}`]); + await advanceBy(harness, 120_000); + expect(harness.runner.calls).toHaveLength(calls); + expect(stateOf(harness)).toBe("off"); + }); + + it("sends SIGKILL after 10 s when the child ignores SIGTERM", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "ignores-sigterm"); + const stoppedAt = harness.clock.now(); + + await advancing(harness, harness.agentDevice.stop()); + + expect(harness.processes.signals).toEqual([ + { pid, signal: "SIGTERM" }, + { pid, signal: "SIGKILL" }, + ]); + expect(harness.clock.now() - stoppedAt).toBeGreaterThanOrEqual(10_000); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("keeps the record and warns naming the pid when the child is alive 2 s after SIGKILL, and availability still ends off", async () => { + const harness = await run(); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.agentDevice.stop()); + + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + expect( + warnings(harness).some((line) => + line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after`), + ), + ).toBe(true); + expect(stateOf(harness)).toBe("off"); + }); + + it("warns naming the pid and keeps the record when stop() ends a start whose child survives SIGKILL", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.agentDevice.stop()); + await advancing(harness, harness.started as Promise); + + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + expect( + warnings(harness).some((line) => + line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after`), + ), + ).toBe(true); + expect(stateOf(harness)).toBe("off"); + }); + + it("spawns nothing when stop() is asked while the state directory is being prepared", async () => { + const harness = await arrange(); + const writeFileAtomic = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + let stopping: Promise | undefined; + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, data) => { + if (path === `${stateDir}/policy.json`) stopping = harness.agentDevice.stop(); + await writeFileAtomic(path, data); + }); + + await advancing(harness, harness.begin()); + await advancing(harness, stopping as Promise); + + expect(daemonSpawns(harness)).toHaveLength(0); + expect(stateOf(harness)).toBe("off"); + }); + + it("cancels a pending retry", async () => { + const harness = await run({ daemonJson: () => "none" }); + expect(unavailable(harness).reason).toBe("start-failed"); + const calls = harness.runner.calls.length; + + await advancing(harness, harness.agentDevice.stop()); + await advanceBy(harness, 120_000); + + expect(harness.runner.calls).toHaveLength(calls); + expect(stateOf(harness)).toBe("off"); + }); + + it("ends a start in progress with the child killed and availability off", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + const pid = harness.children[0]?.pid; + + await advancing(harness, harness.agentDevice.stop()); + await advancing(harness, harness.started as Promise); + + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + expect(stateOf(harness)).toBe("off"); + await advanceBy(harness, 120_000); + expect(daemonSpawns(harness)).toHaveLength(1); + expect(stateOf(harness)).toBe("off"); + }); + + it("leaves no unhandled rejection when a health read rejects after stop() was asked during the port check", async () => { + const harness = await arrange(); + harness.health.rejectWith(new Error("ECONNREFUSED")); + const isListening = harness.probe.isListening.bind(harness.probe); + let stopping: Promise | undefined; + vi.spyOn(harness.probe, "isListening").mockImplementation(async (port) => { + stopping = harness.agentDevice.stop(); + return isListening(port); + }); + const unhandled: unknown[] = []; + const record = (reason: unknown): void => { + unhandled.push(reason); + }; + process.on("unhandledRejection", record); + try { + harness.begin(); + await advanceBy(harness, 5_000); + await advancing(harness, stopping as Promise); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + process.off("unhandledRejection", record); + } + + expect(unhandled).toEqual([]); + expect(stateOf(harness)).toBe("off"); + }); +}); diff --git a/src/agent-device/host/agent-device-supervisor-phones.test.ts b/src/agent-device/host/agent-device-supervisor-phones.test.ts new file mode 100644 index 000000000..1068891e1 --- /dev/null +++ b/src/agent-device/host/agent-device-supervisor-phones.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it } from "vitest"; +import { MemoryFilesystem } from "../../ports/index.js"; +import { + home, + stateDir, + recordPath, + nodePath, + internalEntry, + run, + versionRun, + daemonSpawn, + phoneSpawn, + ownLine, + record, + flush, + advanceBy, + advancing, + advanceUntil, + stateOf, + unavailable, + daemonSpawns, + warnings, + readJson, +} from "./test-agent-device-supervisor.js"; + +describe("AgentDeviceSupervisor in the phones role", () => { + const phonesStateDir = `${home}/agent-device-phones`; + const signing = { + bundleId: "com.example.runner", + provisioningProfile: "Simlock Runner Profile", + signingIdentity: "Apple Development: Ada (XYZ)", + teamId: "ABCDE12345", + }; + const signingEnvironment = { + AGENT_DEVICE_IOS_BUNDLE_ID: "com.example.runner", + AGENT_DEVICE_IOS_PROVISIONING_PROFILE: "Simlock Runner Profile", + AGENT_DEVICE_IOS_SIGNING_IDENTITY: "Apple Development: Ada (XYZ)", + AGENT_DEVICE_IOS_TEAM_ID: "ABCDE12345", + }; + + it.each([ + ["iOS and Android on, keys set", ["ios", "android"], signing, signingEnvironment], + ["iOS and Android on, keys unset", ["ios", "android"], undefined, {}], + [ + "iOS off: the empty private set directory", + ["android"], + signing, + { + ...signingEnvironment, + AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET: `${phonesStateDir}/no-ios-devices`, + }, + ], + [ + "Android off: the allowlist sentinel", + ["ios"], + undefined, + { AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST: "simlock-no-android-device" }, + ], + ] as const)( + "spawns the phone daemon with its own state directory, policy file, pid record and marker suffix, no simulator set and no driver lease key, adb on 5037 and the configured AGENT_DEVICE_IOS_ keys: %s", + async (_case, platforms, ios, signingKeys) => { + const harness = await run({ + baseEnvironment: { + ANDROID_ADB_SERVER_ADDRESS: "operator-host", + ANDROID_ADB_SERVER_PORT: "5038", + ANDROID_SERIAL: "emulator-5554", + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + }, + driverEnvironment: { + ANDROID_ADB_SERVER_PORT: "5038", + SIMLOCK_IOS_DEVICE_SET: "/devices/ios", + }, + expectations: [versionRun(), phoneSpawn()], + ...(ios === undefined ? {} : { ios }), + platforms, + role: "phones", + }); + + expect(daemonSpawns(harness)).toEqual([ + [nodePath, internalEntry, `--simlock-home=${home}`, "--simlock-role=phones"], + ]); + expect(harness.runner.calls[1]?.options.env).toEqual({ + AGENT_DEVICE_CLAIMS_DIR: `${phonesStateDir}/device-claims`, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", + AGENT_DEVICE_DAEMON_POLICY: `${phonesStateDir}/policy.json`, + AGENT_DEVICE_DAEMON_SERVER_MODE: "http", + AGENT_DEVICE_LEASE_MAX_TTL_MS: String(24 * 60 * 60_000), + AGENT_DEVICE_STATE_DIR: phonesStateDir, + ANDROID_ADB_SERVER_PORT: "5037", + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + ...signingKeys, + }); + await expect(harness.filesystem.readFile(`${phonesStateDir}/policy.json`)).resolves.toContain( + "device-shutdown", + ); + expect( + JSON.parse(await harness.filesystem.readFile(`${home}/agent-device-phones-daemon.json`)), + ).toMatchObject({ + entry: internalEntry, + }); + expect(await harness.filesystem.exists(`${stateDir}/policy.json`)).toBe(false); + expect(await harness.filesystem.exists(recordPath)).toBe(false); + }, + ); +}); + +describe("AgentDeviceSupervisor reaping by role", () => { + const phonesRecordPath = `${home}/agent-device-phones-daemon.json`; + const leftBehind = (path: string, pid: number) => async (filesystem: MemoryFilesystem) => { + await filesystem.writeFileAtomic(path, JSON.stringify(record(pid))); + }; + + it.each([ + { + name: "a phones record whose pid line ends with the phones suffix is signalled, with the simulator record left alone", + role: "phones", + line: ownLine(internalEntry, home, "phones"), + otherRecord: recordPath, + signalled: true, + survivingRecord: recordPath, + }, + { + name: "a simulators record whose pid line ends with the simulators suffix is signalled, with the phones record left alone", + role: "simulators", + line: ownLine(), + otherRecord: phonesRecordPath, + signalled: true, + survivingRecord: phonesRecordPath, + }, + { + name: "a phones record whose pid line is a simulator daemon's is not signalled", + role: "phones", + line: ownLine(), + otherRecord: undefined, + signalled: false, + survivingRecord: undefined, + }, + { + name: "a simulators record whose pid line is a phone daemon's is not signalled", + role: "simulators", + line: ownLine(internalEntry, home, "phones"), + otherRecord: undefined, + signalled: false, + survivingRecord: undefined, + }, + ] as const)( + "signals only a pid whose line ends with its own role's suffix, and never touches the other role's record: $name", + async ({ line, otherRecord, role, signalled, survivingRecord }) => { + const harness = await run({ + commandLines: [[900, [line]]], + expectations: [versionRun(), role === "phones" ? phoneSpawn() : daemonSpawn()], + layout: async (filesystem) => { + if (otherRecord !== undefined) await leftBehind(otherRecord, 901)(filesystem); + }, + livePids: [900, 901], + record: record(900), + role, + }); + + expect(harness.processes.signals).toEqual(signalled ? [{ pid: 900, signal: "SIGTERM" }] : []); + expect(stateOf(harness)).toBe("running"); + if (survivingRecord !== undefined) { + expect(await readJson(harness, survivingRecord)).toMatchObject({ pid: 901 }); + } + }, + ); +}); + +describe("AgentDeviceSupervisor stopping and restarting the phone daemon", () => { + const phonesRecordPath = `${home}/agent-device-phones-daemon.json`; + const phones = { + expectations: [versionRun(), phoneSpawn(), versionRun(), phoneSpawn()], + role: "phones", + } as const; + + it("stop signals the phone daemon and removes its record", async () => { + const harness = await run(phones); + const pid = harness.children[0]?.pid; + + await advancing(harness, harness.agentDevice.stop()); + + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + await expect(harness.filesystem.exists(phonesRecordPath)).resolves.toBe(false); + expect(stateOf(harness)).toBe("off"); + }); + + it.each([ + { afterMs: 60_000, restartsAtOnce: true }, + { afterMs: 5_000, restartsAtOnce: false }, + ])( + "a phone daemon that exits $afterMs ms after it was ready restarts at once only when that is 60 s or more, else after 30 s with a warning that names its role", + async ({ afterMs, restartsAtOnce }) => { + const harness = await run(phones); + await advanceBy(harness, afterMs); + + harness.children[0]?.exit(9); + await flush(); + + if (restartsAtOnce) { + expect(stateOf(harness)).toBe("starting"); + } else { + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect( + warnings(harness).some((line) => line.includes("agent-device (phones) start failed")), + ).toBe(true); + await advanceBy(harness, 29_000); + expect(daemonSpawns(harness)).toHaveLength(1); + } + await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); + expect(daemonSpawns(harness)).toHaveLength(2); + }, + ); +}); diff --git a/src/agent-device/host/agent-device-supervisor-state.test.ts b/src/agent-device/host/agent-device-supervisor-state.test.ts new file mode 100644 index 000000000..fbcb19020 --- /dev/null +++ b/src/agent-device/host/agent-device-supervisor-state.test.ts @@ -0,0 +1,335 @@ +import { describe, expect, it, vi } from "vitest"; +import { MemoryFilesystem } from "../../ports/index.js"; +import { + home, + stateDir, + recordPath, + policyPath, + daemonJsonPath, + internalEntry, + httpPort, + token, + healthyBody, + restartCommand, + stemOf, + arrange, + run, + versionRun, + daemonSpawn, + phoneSpawn, + advanceBy, + advancing, + advanceUntil, + stateOf, + unavailable, + daemonSpawns, + warnings, + readJson, +} from "./test-agent-device-supervisor.js"; + +describe("AgentDeviceSupervisor state directory", () => { + it("creates a missing state directory with mode 0700", async () => { + const harness = await run(); + + await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ + kind: "directory", + mode: 0o700, + }); + }); + + it("sets an existing state directory to mode 0700", async () => { + const harness = await run({ + stateDirContents: async (filesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o755 }); + }, + }); + + await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ mode: 0o700 }); + expect(stateOf(harness)).toBe("running"); + }); + + it.each([ + ["simulators" as const, [versionRun(), daemonSpawn()]], + ["phones" as const, [versionRun(), phoneSpawn()]], + ])( + "a daemon start empties the %s daemon's sessions folder and leaves the rest of its state directory", + async (role, expectations) => { + const sessions = `${home}/${stemOf(role)}/sessions`; + const harness = await run({ + expectations, + role, + stateDirContents: async (filesystem) => { + await filesystem.mkdirp(`${sessions}/adc-1`); + await filesystem.writeFileAtomic(`${sessions}/adc-1/events.ndjson`, "{}\n"); + await filesystem.mkdirp(`${sessions}/tok_one_my-session`); + }, + }); + + expect(stateOf(harness)).toBe("running"); + expect(await harness.filesystem.readdir(sessions)).toEqual([]); + expect(await harness.filesystem.exists(`${home}/${stemOf(role)}/policy.json`)).toBe(true); + }, + ); + + it.each([ + [ + "a symlink", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp("/elsewhere"); + filesystem.defineSymlink(stateDir, "/elsewhere"); + }, + ], + [ + "a file", + async (filesystem: MemoryFilesystem) => + filesystem.writeFileAtomic(stateDir, "not a directory"), + ], + [ + "a directory owned by another user", + async (filesystem: MemoryFilesystem) => { + await filesystem.mkdir(stateDir, { mode: 0o700 }); + filesystem.defineAttributes(stateDir, { uid: (process.getuid?.() ?? 0) + 1 }); + }, + ], + ])( + "treats %s as the state directory as state-dir, naming the path, and spawns nothing", + async (_case, stateDirContents) => { + const harness = await run({ expectations: [versionRun()], stateDirContents }); + + expect(unavailable(harness).reason).toBe("state-dir"); + expect(unavailable(harness).message).toContain(stateDir); + expect(unavailable(harness).remedy).toContain(stateDir); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(daemonSpawns(harness)).toEqual([]); + }, + ); + + it("treats a chmod that throws on an existing state directory as state-dir naming the path and the cause", async () => { + const filesystem = new MemoryFilesystem(); + vi.spyOn(filesystem, "chmod").mockRejectedValue(new Error("EPERM: operation not permitted")); + const harness = await run({ + expectations: [versionRun()], + filesystem, + stateDirContents: async (memory) => { + await memory.mkdir(stateDir, { mode: 0o755 }); + }, + }); + + expect(unavailable(harness).reason).toBe("state-dir"); + expect(unavailable(harness).message).toContain(stateDir); + expect(unavailable(harness).message).toContain("EPERM"); + expect(daemonSpawns(harness)).toEqual([]); + }); +}); + +describe("AgentDeviceSupervisor pid record", () => { + it("records the child's pid and entry while daemon.json is not yet written", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + + expect(await readJson(harness, recordPath)).toMatchObject({ + entry: internalEntry, + pid: harness.children[0]?.pid, + }); + expect(typeof (await readJson(harness, recordPath))["startedAt"]).toBe("number"); + }); + + it("kills the child and reports start-failed naming the error when the record write throws", async () => { + const harness = await arrange(); + const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { + if (path === recordPath) throw new Error("ENOSPC: no space left"); + return write(path, ...rest); + }); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("ENOSPC"); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + }); +}); + +describe("AgentDeviceSupervisor readiness", () => { + it("is running with the version, base URL, token and generation 1 once daemon.json names the child, the port listens and /health answers protocol 2", async () => { + const harness = await run(); + + expect(harness.availability()).toEqual({ + baseUrl: `http://127.0.0.1:${httpPort}`, + generation: 1, + leaseMaxTtlMs: 24 * 60 * 60_000, + platforms: ["ios", "android"], + state: "running", + token, + version: "0.21.28", + }); + expect(harness.health.reads).toContain(`http://127.0.0.1:${httpPort}`); + }); + + it("is not ready while daemon.json names another pid, and ready once it names the child", async () => { + const harness = await arrange({ daemonJson: () => "other-pid" }); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + const pid = harness.children[0]?.pid; + await harness.filesystem.writeFileAtomic( + daemonJsonPath, + JSON.stringify({ httpPort, pid, token, version: "0.21.28" }), + ); + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("is not ready while the port is not listening, and ready once it is", async () => { + const harness = await arrange({ listening: false }); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + harness.probe.startListening(httpPort); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("is not ready while a health read rejects, and ready once it answers", async () => { + const harness = await arrange(); + harness.health.rejectWith(new Error("ECONNREFUSED")); + harness.begin(); + await advanceBy(harness, 5_000); + expect(stateOf(harness)).toBe("starting"); + + harness.health.respondWith(healthyBody); + await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); + + expect(stateOf(harness)).toBe("running"); + }); + + it("stops the child and reports wrong-version naming both protocols when /health answers protocol 3", async () => { + const harness = await arrange(); + harness.health.respondWith({ rpcProtocolVersion: 3, service: "agent-device-daemon" }); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ + message: "agent-device 0.21.28 speaks RPC protocol 3, need 2", + reason: "wrong-version", + }); + expect(unavailable(harness).remedy).toContain("RPC protocol 2"); + for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + }); + + it.each([ + ["another service", { rpcProtocolVersion: 2, service: "something-else" }], + ["no numeric protocol", { rpcProtocolVersion: "2", service: "agent-device-daemon" }], + ])( + "stops the child and reports not-agent-device without retrying when /health answers with %s", + async (_case, body) => { + const harness = await arrange(); + harness.health.respondWith(body); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ + message: "/health did not answer as agent-device", + reason: "not-agent-device", + }); + expect(harness.processes.signals).toEqual([ + { pid: harness.children[0]?.pid, signal: "SIGTERM" }, + ]); + await advanceBy(harness, 120_000); + expect(daemonSpawns(harness)).toHaveLength(1); + expect(stateOf(harness)).toBe("unavailable"); + }, + ); +}); + +describe("AgentDeviceSupervisor failed starts", () => { + it("kills the child after 15 s without readiness, removes the record only after it exits, and reports start-failed naming the timeout", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + const remove = harness.filesystem.rm.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "rm").mockImplementation(async (path) => { + if (path === recordPath) harness.events.push("record removed"); + return remove(path); + }); + const spawnedAt = harness.clock.now(); + + await advancing(harness, harness.begin()); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("timed out"); + expect(unavailable(harness).remedy).toContain("30 s"); + expect(unavailable(harness).remedy).toContain("simlock daemon logs"); + expect(harness.clock.now() - spawnedAt).toBeGreaterThanOrEqual(15_000); + expect(harness.clock.now() - spawnedAt).toBeLessThan(16_000); + const pid = harness.children[0]?.pid; + expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); + expect(harness.events.slice(-2)).toEqual([`exit ${pid}`, "record removed"]); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("reports start-failed naming the exit code when the child exits before readiness", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 1_000); + + harness.children[0]?.exit(7); + await advancing(harness, harness.started as Promise); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("7"); + }); + + it("reports start-failed and writes no record when the spawn throws", async () => { + const harness = await run({ spawnThrows: new Error("EACCES: permission denied") }); + + expect(warnings(harness).some((line) => line.includes("EACCES"))).toBe(true); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("EACCES"); + await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); + }); + + it("keeps the record and names the pid when the child is still alive 2 s after SIGKILL", async () => { + const harness = await arrange({ daemonJson: () => "none" }); + harness.begin(); + await advanceBy(harness, 500); + const pid = harness.children[0]?.pid as number; + harness.processes.behaviours.set(pid, "immortal"); + + await advancing(harness, harness.started as Promise); + + expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain(String(pid)); + expect( + warnings(harness).some((line) => + line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after SIGKILL`), + ), + ).toBe(true); + expect(await readJson(harness, recordPath)).toMatchObject({ pid }); + }); + + it("resolves start() with start-failed carrying the message when the filesystem throws unexpectedly while writing the policy file", async () => { + const harness = await arrange(); + const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); + vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { + if (path === policyPath) throw new Error("disk on fire"); + return write(path, ...rest); + }); + + await expect(advancing(harness, harness.begin())).resolves.toBeUndefined(); + + expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); + expect(unavailable(harness).message).toContain("disk on fire"); + expect(daemonSpawns(harness)).toEqual([]); + }); +}); diff --git a/src/agent-device/host/agent-device-supervisor.test.ts b/src/agent-device/host/agent-device-supervisor.test.ts index 3a3609e01..dba74280f 100644 --- a/src/agent-device/host/agent-device-supervisor.test.ts +++ b/src/agent-device/host/agent-device-supervisor.test.ts @@ -1,367 +1,30 @@ import { describe, expect, it, vi } from "vitest"; - -import { - FakeClock, - FakeProcessSupervisor, - FakeTcpProbe, - JsonLinesLogger, - MemoryFilesystem, - MemoryLogSink, - ScriptedProcessRunner, - type ProcessResult, - type ScriptedProcessExpectation, -} from "../../ports/index.js"; -import { FakeAgentDeviceHealth } from "../testing.js"; +import { MemoryFilesystem } from "../../ports/index.js"; import { - AgentDeviceSupervisor, - type AgentDeviceAvailability, - type AgentDeviceIosSigning, - type AgentDevicePlatform, - type AgentDeviceRole, -} from "./agent-device-supervisor.js"; - -const home = "/home/simlock/.simlock"; -const stateDir = `${home}/agent-device`; -const recordPath = `${home}/agent-device-daemon.json`; -const policyPath = `${stateDir}/policy.json`; -const daemonJsonPath = `${stateDir}/daemon.json`; -const nodePath = "/opt/node/bin/node"; -const packageRoot = "/opt/ad/node_modules/agent-device"; -const realCli = `${packageRoot}/bin/agent-device.mjs`; -const internalEntry = `${packageRoot}/dist/src/internal/daemon.js`; -const fallbackEntry = `${packageRoot}/dist/src/daemon.js`; -const httpPort = 4310; -const token = "daemon-token"; -const healthyBody = { rpcProtocolVersion: 2, service: "agent-device-daemon" }; -const restartCommand = ["simlock daemon stop", "simlock daemon start"]; - -type Behaviour = "dies" | "ignores-sigterm" | "immortal"; -type DaemonJson = "ready" | "none" | "other-pid"; - -class ProcessTable extends FakeProcessSupervisor { - readonly behaviours = new Map(); - readonly onDeath = new Map void>(); - - override signal(pid: number, signal: NodeJS.Signals): void { - super.signal(pid, signal); - this.apply(pid, signal); - } - - apply(pid: number, signal: NodeJS.Signals): void { - const behaviour = this.behaviours.get(pid) ?? "dies"; - if (behaviour === "immortal") return; - if (behaviour === "ignores-sigterm" && signal !== "SIGKILL") return; - this.markDead(pid); - this.onDeath.get(pid)?.(); - } -} - -interface Child { - readonly pid: number; - readonly args: readonly string[]; - exit(code: number | null): void; -} - -interface Options { - readonly baseEnvironment?: NodeJS.ProcessEnv; - readonly command?: string; - readonly commandLines?: readonly (readonly [number, readonly (string | undefined)[]])[]; - readonly daemonJson?: (startIndex: number) => DaemonJson; - readonly driverEnvironment?: Readonly>; - readonly expectations?: readonly ScriptedProcessExpectation[]; - readonly filesystem?: MemoryFilesystem; - readonly home?: string; - readonly ios?: AgentDeviceIosSigning; - readonly layout?: (filesystem: MemoryFilesystem) => Promise; - readonly leaseMaxTtlMs?: number; - readonly listening?: boolean; - readonly livePids?: readonly number[]; - readonly nodeVersion?: string; - readonly platforms?: readonly AgentDevicePlatform[]; - readonly role?: AgentDeviceRole; - readonly record?: { readonly pid: number; readonly entry: string; readonly startedAt: number }; - readonly spawnThrows?: Error; - readonly stateDirContents?: (filesystem: MemoryFilesystem) => Promise; - readonly behaviours?: ReadonlyMap; -} - -function stemOf(role: AgentDeviceRole | undefined): string { - return role === "phones" ? "agent-device-phones" : "agent-device"; -} - -function supervisorRole(options: Options): { - readonly ios?: AgentDeviceIosSigning; - readonly role: AgentDeviceRole; -} { - return { - ...(options.ios === undefined ? {} : { ios: options.ios }), - role: options.role ?? "simulators", - }; -} - -async function arrange(options: Options = {}) { - const clock = new FakeClock(1_000_000); - const stem = stemOf(options.role); - const filesystem = options.filesystem ?? new MemoryFilesystem(); - const simlockHome = options.home ?? home; - const sink = new MemoryLogSink(); - const logger = new JsonLinesLogger({ clock, level: "debug", sink }); - const probe = new FakeTcpProbe(); - const health = new FakeAgentDeviceHealth(); - const processes = new ProcessTable(options.livePids ?? []); - for (const [pid, behaviour] of options.behaviours ?? []) processes.behaviours.set(pid, behaviour); - for (const [pid, lines] of options.commandLines ?? []) processes.answerCommandLine(pid, ...lines); - const runner = new ScriptedProcessRunner(options.expectations ?? [versionRun(), daemonSpawn()]); - const children: Child[] = []; - const events: string[] = []; - let starts = 0; - - health.respondWith(healthyBody); - await installAgentDevice(filesystem, simlockHome, options); - if (options.record !== undefined) { - await filesystem.writeFileAtomic( - `${simlockHome}/${stem}-daemon.json`, - JSON.stringify(options.record), - ); - } - await options.stateDirContents?.(filesystem); - - const spawn = runner.spawn.bind(runner); - vi.spyOn(runner, "spawn").mockImplementation((command, args, spawnOptions) => { - const handle = spawn(command, args, spawnOptions); - if (command !== nodePath || !isDaemonEntry(args[0])) return handle; - if (options.spawnThrows !== undefined) throw options.spawnThrows; - const pid = handle.pid; - let exit!: (result: ProcessResult) => void; - const exited = new Promise((resolve) => { - exit = resolve; - }); - const child: Child = { - args: [...args], - exit(code) { - events.push(`exit ${pid}`); - processes.markDead(pid); - exit({ code, stderr: "", stdout: "" }); - }, - pid, - }; - children.push(child); - processes.markAlive(pid); - processes.onDeath.set(pid, () => child.exit(null)); - events.push(`spawn ${pid}`); - const startIndex = starts; - starts += 1; - const mode = options.daemonJson?.(startIndex) ?? "ready"; - if (mode !== "none") { - void filesystem.writeFileAtomic( - `${simlockHome}/${stem}/daemon.json`, - JSON.stringify({ - httpPort, - pid: mode === "ready" ? pid : pid + 1_000, - token, - version: "0.21.28", - }), - ); - } - if (mode === "ready" && options.listening !== false) probe.startListening(httpPort); - return { - kill(signal = "SIGTERM") { - processes.signals.push({ pid, signal }); - processes.apply(pid, signal); - handle.kill(signal); - }, - pid, - stderr: handle.stderr, - stdout: handle.stdout, - unref: () => handle.unref(), - wait: () => Promise.race([handle.wait(), exited]), - }; - }); - - const agentDevice = new AgentDeviceSupervisor({ - baseEnvironment: options.baseEnvironment ?? { - HOME: "/home/operator", - PATH: "/usr/local/bin:/usr/bin", - }, - clock, - config: { command: options.command ?? "agent-device" }, - driverEnvironment: options.driverEnvironment ?? {}, - filesystem, - healthProbe: health, - home: simlockHome, - hostPlatform: "linux", - ...supervisorRole(options), - leaseMaxTtlMs: options.leaseMaxTtlMs ?? 4 * 60 * 60_000, - logger, - nodePath, - nodeVersion: options.nodeVersion ?? "22.14.0", - platforms: options.platforms ?? ["ios", "android"], - processRunner: runner, - processSupervisor: processes, - tcpProbe: probe, - }); - const harness = { - agentDevice, - children, - clock, - events, - filesystem, - health, - probe, - processes, - runner, - sink, - started: undefined as Promise | undefined, - begin(): Promise { - const started = agentDevice.start(); - started.catch(() => undefined); - harness.started = started; - return started; - }, - availability: () => agentDevice.availability(), - }; - return harness; -} - -type Harness = Awaited>; - -async function run(options: Options = {}): Promise { - const harness = await arrange(options); - await advancing(harness, harness.begin()); - return harness; -} - -function isDaemonEntry(arg: string | undefined): boolean { - return arg === internalEntry || arg === fallbackEntry; -} - -async function installAgentDevice( - filesystem: MemoryFilesystem, - simlockHome: string, - options: Options, -): Promise { - if (!(await filesystem.exists(simlockHome))) await filesystem.mkdirp(simlockHome); - if (options.command?.includes("/") !== true) { - await filesystem.mkdirp("/usr/local/bin"); - } - await filesystem.mkdirp(`${packageRoot}/bin`); - await filesystem.mkdirp(`${packageRoot}/dist/src/internal`); - await filesystem.writeFileAtomic(realCli, "#!/usr/bin/env node\n"); - filesystem.defineAttributes(realCli, { mode: 0o755 }); - await filesystem.writeFileAtomic(`${packageRoot}/package.json`, '{"name":"agent-device"}'); - await filesystem.writeFileAtomic(internalEntry, ""); - filesystem.defineSymlink("/usr/local/bin/agent-device", realCli); - await options.layout?.(filesystem); -} - -function versionRun(output = "0.21.28\n", extra: Partial = {}) { - return { - match: { args: [realCli, "--version"], command: nodePath }, - result: { code: 0, stderr: "", stdout: output }, - ...extra, - } satisfies ScriptedProcessExpectation; -} - -function daemonSpawn( - entry = internalEntry, - extra: Partial = {}, - simlockHome = home, - marker: readonly string[] = [], -): ScriptedProcessExpectation { - return { - hangs: true, - match: { args: [entry, `--simlock-home=${simlockHome}`, ...marker], command: nodePath }, - ...extra, - }; -} - -function phoneSpawn(simlockHome = home): ScriptedProcessExpectation { - return daemonSpawn(internalEntry, {}, simlockHome, ["--simlock-role=phones"]); -} - -function ownLine(entry = internalEntry, simlockHome = home, role?: "phones"): string { - const suffix = role === undefined ? "" : " --simlock-role=phones"; - return `${nodePath} ${entry} --simlock-home=${simlockHome}${suffix}`; -} - -function record(pid: number, entry = internalEntry) { - return { entry, pid, startedAt: 1 }; -} - -async function flush(): Promise { - await new Promise((resolve) => setImmediate(resolve)); -} - -async function advanceBy(harness: Harness, ms: number): Promise { - for (let elapsed = 0; elapsed < ms; elapsed += 100) { - await flush(); - harness.clock.advance(100); - } - await flush(); -} - -async function advancing(harness: Harness, work: Promise): Promise { - let settled = false; - const tracked = work.finally(() => { - settled = true; - }); - tracked.catch(() => undefined); - for (let tick = 0; tick < 4_000 && !settled; tick += 1) { - await flush(); - harness.clock.advance(100); - } - return tracked; -} - -async function advanceUntil( - harness: Harness, - done: () => boolean, - limitMs = 120_000, -): Promise { - for (let elapsed = 0; elapsed < limitMs && !done(); elapsed += 100) { - await flush(); - harness.clock.advance(100); - } - await flush(); -} - -function stateOf(harness: Harness): AgentDeviceAvailability["state"] { - return harness.availability().state; -} - -function unavailable(harness: Harness) { - const availability = harness.availability(); - if (availability.state !== "unavailable") { - throw new Error(`expected unavailable, got ${availability.state}`); - } - return availability; -} - -function running(harness: Harness) { - const availability = harness.availability(); - if (availability.state !== "running") { - throw new Error(`expected running, got ${availability.state}`); - } - return availability; -} - -function commandsRun(harness: Harness): string[][] { - return harness.runner.calls.map((call) => [call.command, ...call.args]); -} - -function daemonSpawns(harness: Harness): string[][] { - return commandsRun(harness).filter((call) => call[0] === nodePath && isDaemonEntry(call[1])); -} - -function warnings(harness: Harness): string[] { - return harness.sink.records - .filter((entry) => entry.level === "warn") - .map((entry) => JSON.stringify(entry)); -} - -async function readJson(harness: Harness, path: string): Promise> { - return JSON.parse(await harness.filesystem.readFile(path)) as Record; -} + home, + stateDir, + policyPath, + nodePath, + packageRoot, + realCli, + internalEntry, + fallbackEntry, + httpPort, + token, + restartCommand, + arrange, + run, + isDaemonEntry, + versionRun, + daemonSpawn, + advanceBy, + advancing, + stateOf, + unavailable, + running, + commandsRun, + daemonSpawns, +} from "./test-agent-device-supervisor.js"; describe("AgentDeviceSupervisor off and starting", () => { it("enabled, availability is starting before start() settles", async () => { @@ -737,994 +400,3 @@ describe("AgentDeviceSupervisor spawning", () => { await expect(harness.filesystem.readFile(policyPath)).resolves.toBe(expected); }); }); - -describe("AgentDeviceSupervisor in the phones role", () => { - const phonesStateDir = `${home}/agent-device-phones`; - const signing = { - bundleId: "com.example.runner", - provisioningProfile: "Simlock Runner Profile", - signingIdentity: "Apple Development: Ada (XYZ)", - teamId: "ABCDE12345", - }; - const signingEnvironment = { - AGENT_DEVICE_IOS_BUNDLE_ID: "com.example.runner", - AGENT_DEVICE_IOS_PROVISIONING_PROFILE: "Simlock Runner Profile", - AGENT_DEVICE_IOS_SIGNING_IDENTITY: "Apple Development: Ada (XYZ)", - AGENT_DEVICE_IOS_TEAM_ID: "ABCDE12345", - }; - - it.each([ - ["iOS and Android on, keys set", ["ios", "android"], signing, signingEnvironment], - ["iOS and Android on, keys unset", ["ios", "android"], undefined, {}], - [ - "iOS off: the empty private set directory", - ["android"], - signing, - { - ...signingEnvironment, - AGENT_DEVICE_IOS_SIMULATOR_DEVICE_SET: `${phonesStateDir}/no-ios-devices`, - }, - ], - [ - "Android off: the allowlist sentinel", - ["ios"], - undefined, - { AGENT_DEVICE_ANDROID_DEVICE_ALLOWLIST: "simlock-no-android-device" }, - ], - ] as const)( - "spawns the phone daemon with its own state directory, policy file, pid record and marker suffix, no simulator set and no driver lease key, adb on 5037 and the configured AGENT_DEVICE_IOS_ keys: %s", - async (_case, platforms, ios, signingKeys) => { - const harness = await run({ - baseEnvironment: { - ANDROID_ADB_SERVER_ADDRESS: "operator-host", - ANDROID_ADB_SERVER_PORT: "5038", - ANDROID_SERIAL: "emulator-5554", - HOME: "/home/operator", - PATH: "/usr/local/bin:/usr/bin", - }, - driverEnvironment: { - ANDROID_ADB_SERVER_PORT: "5038", - SIMLOCK_IOS_DEVICE_SET: "/devices/ios", - }, - expectations: [versionRun(), phoneSpawn()], - ...(ios === undefined ? {} : { ios }), - platforms, - role: "phones", - }); - - expect(daemonSpawns(harness)).toEqual([ - [nodePath, internalEntry, `--simlock-home=${home}`, "--simlock-role=phones"], - ]); - expect(harness.runner.calls[1]?.options.env).toEqual({ - AGENT_DEVICE_CLAIMS_DIR: `${phonesStateDir}/device-claims`, - AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: "0", - AGENT_DEVICE_DAEMON_POLICY: `${phonesStateDir}/policy.json`, - AGENT_DEVICE_DAEMON_SERVER_MODE: "http", - AGENT_DEVICE_LEASE_MAX_TTL_MS: String(24 * 60 * 60_000), - AGENT_DEVICE_STATE_DIR: phonesStateDir, - ANDROID_ADB_SERVER_PORT: "5037", - HOME: "/home/operator", - PATH: "/usr/local/bin:/usr/bin", - ...signingKeys, - }); - await expect(harness.filesystem.readFile(`${phonesStateDir}/policy.json`)).resolves.toContain( - "device-shutdown", - ); - expect( - JSON.parse(await harness.filesystem.readFile(`${home}/agent-device-phones-daemon.json`)), - ).toMatchObject({ - entry: internalEntry, - }); - expect(await harness.filesystem.exists(`${stateDir}/policy.json`)).toBe(false); - expect(await harness.filesystem.exists(recordPath)).toBe(false); - }, - ); -}); - -describe("AgentDeviceSupervisor reaping by role", () => { - const phonesRecordPath = `${home}/agent-device-phones-daemon.json`; - const leftBehind = (path: string, pid: number) => async (filesystem: MemoryFilesystem) => { - await filesystem.writeFileAtomic(path, JSON.stringify(record(pid))); - }; - - it.each([ - { - name: "a phones record whose pid line ends with the phones suffix is signalled, with the simulator record left alone", - role: "phones", - line: ownLine(internalEntry, home, "phones"), - otherRecord: recordPath, - signalled: true, - survivingRecord: recordPath, - }, - { - name: "a simulators record whose pid line ends with the simulators suffix is signalled, with the phones record left alone", - role: "simulators", - line: ownLine(), - otherRecord: phonesRecordPath, - signalled: true, - survivingRecord: phonesRecordPath, - }, - { - name: "a phones record whose pid line is a simulator daemon's is not signalled", - role: "phones", - line: ownLine(), - otherRecord: undefined, - signalled: false, - survivingRecord: undefined, - }, - { - name: "a simulators record whose pid line is a phone daemon's is not signalled", - role: "simulators", - line: ownLine(internalEntry, home, "phones"), - otherRecord: undefined, - signalled: false, - survivingRecord: undefined, - }, - ] as const)( - "signals only a pid whose line ends with its own role's suffix, and never touches the other role's record: $name", - async ({ line, otherRecord, role, signalled, survivingRecord }) => { - const harness = await run({ - commandLines: [[900, [line]]], - expectations: [versionRun(), role === "phones" ? phoneSpawn() : daemonSpawn()], - layout: async (filesystem) => { - if (otherRecord !== undefined) await leftBehind(otherRecord, 901)(filesystem); - }, - livePids: [900, 901], - record: record(900), - role, - }); - - expect(harness.processes.signals).toEqual(signalled ? [{ pid: 900, signal: "SIGTERM" }] : []); - expect(stateOf(harness)).toBe("running"); - if (survivingRecord !== undefined) { - expect(await readJson(harness, survivingRecord)).toMatchObject({ pid: 901 }); - } - }, - ); -}); - -describe("AgentDeviceSupervisor stopping and restarting the phone daemon", () => { - const phonesRecordPath = `${home}/agent-device-phones-daemon.json`; - const phones = { - expectations: [versionRun(), phoneSpawn(), versionRun(), phoneSpawn()], - role: "phones", - } as const; - - it("stop signals the phone daemon and removes its record", async () => { - const harness = await run(phones); - const pid = harness.children[0]?.pid; - - await advancing(harness, harness.agentDevice.stop()); - - expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); - await expect(harness.filesystem.exists(phonesRecordPath)).resolves.toBe(false); - expect(stateOf(harness)).toBe("off"); - }); - - it.each([ - { afterMs: 60_000, restartsAtOnce: true }, - { afterMs: 5_000, restartsAtOnce: false }, - ])( - "a phone daemon that exits $afterMs ms after it was ready restarts at once only when that is 60 s or more, else after 30 s with a warning that names its role", - async ({ afterMs, restartsAtOnce }) => { - const harness = await run(phones); - await advanceBy(harness, afterMs); - - harness.children[0]?.exit(9); - await flush(); - - if (restartsAtOnce) { - expect(stateOf(harness)).toBe("starting"); - } else { - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect( - warnings(harness).some((line) => line.includes("agent-device (phones) start failed")), - ).toBe(true); - await advanceBy(harness, 29_000); - expect(daemonSpawns(harness)).toHaveLength(1); - } - await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); - expect(daemonSpawns(harness)).toHaveLength(2); - }, - ); -}); - -describe("AgentDeviceSupervisor state directory", () => { - it("creates a missing state directory with mode 0700", async () => { - const harness = await run(); - - await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ - kind: "directory", - mode: 0o700, - }); - }); - - it("sets an existing state directory to mode 0700", async () => { - const harness = await run({ - stateDirContents: async (filesystem) => { - await filesystem.mkdir(stateDir, { mode: 0o755 }); - }, - }); - - await expect(harness.filesystem.lstat(stateDir)).resolves.toMatchObject({ mode: 0o700 }); - expect(stateOf(harness)).toBe("running"); - }); - - it.each([ - ["simulators" as const, [versionRun(), daemonSpawn()]], - ["phones" as const, [versionRun(), phoneSpawn()]], - ])( - "a daemon start empties the %s daemon's sessions folder and leaves the rest of its state directory", - async (role, expectations) => { - const sessions = `${home}/${stemOf(role)}/sessions`; - const harness = await run({ - expectations, - role, - stateDirContents: async (filesystem) => { - await filesystem.mkdirp(`${sessions}/adc-1`); - await filesystem.writeFileAtomic(`${sessions}/adc-1/events.ndjson`, "{}\n"); - await filesystem.mkdirp(`${sessions}/tok_one_my-session`); - }, - }); - - expect(stateOf(harness)).toBe("running"); - expect(await harness.filesystem.readdir(sessions)).toEqual([]); - expect(await harness.filesystem.exists(`${home}/${stemOf(role)}/policy.json`)).toBe(true); - }, - ); - - it.each([ - [ - "a symlink", - async (filesystem: MemoryFilesystem) => { - await filesystem.mkdirp("/elsewhere"); - filesystem.defineSymlink(stateDir, "/elsewhere"); - }, - ], - [ - "a file", - async (filesystem: MemoryFilesystem) => - filesystem.writeFileAtomic(stateDir, "not a directory"), - ], - [ - "a directory owned by another user", - async (filesystem: MemoryFilesystem) => { - await filesystem.mkdir(stateDir, { mode: 0o700 }); - filesystem.defineAttributes(stateDir, { uid: (process.getuid?.() ?? 0) + 1 }); - }, - ], - ])( - "treats %s as the state directory as state-dir, naming the path, and spawns nothing", - async (_case, stateDirContents) => { - const harness = await run({ expectations: [versionRun()], stateDirContents }); - - expect(unavailable(harness).reason).toBe("state-dir"); - expect(unavailable(harness).message).toContain(stateDir); - expect(unavailable(harness).remedy).toContain(stateDir); - for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); - expect(daemonSpawns(harness)).toEqual([]); - }, - ); - - it("treats a chmod that throws on an existing state directory as state-dir naming the path and the cause", async () => { - const filesystem = new MemoryFilesystem(); - vi.spyOn(filesystem, "chmod").mockRejectedValue(new Error("EPERM: operation not permitted")); - const harness = await run({ - expectations: [versionRun()], - filesystem, - stateDirContents: async (memory) => { - await memory.mkdir(stateDir, { mode: 0o755 }); - }, - }); - - expect(unavailable(harness).reason).toBe("state-dir"); - expect(unavailable(harness).message).toContain(stateDir); - expect(unavailable(harness).message).toContain("EPERM"); - expect(daemonSpawns(harness)).toEqual([]); - }); -}); - -describe("AgentDeviceSupervisor pid record", () => { - it("records the child's pid and entry while daemon.json is not yet written", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - harness.begin(); - await advanceBy(harness, 1_000); - - expect(await readJson(harness, recordPath)).toMatchObject({ - entry: internalEntry, - pid: harness.children[0]?.pid, - }); - expect(typeof (await readJson(harness, recordPath))["startedAt"]).toBe("number"); - }); - - it("kills the child and reports start-failed naming the error when the record write throws", async () => { - const harness = await arrange(); - const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); - vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { - if (path === recordPath) throw new Error("ENOSPC: no space left"); - return write(path, ...rest); - }); - - await advancing(harness, harness.begin()); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("ENOSPC"); - expect(harness.processes.signals).toEqual([ - { pid: harness.children[0]?.pid, signal: "SIGTERM" }, - ]); - }); -}); - -describe("AgentDeviceSupervisor readiness", () => { - it("is running with the version, base URL, token and generation 1 once daemon.json names the child, the port listens and /health answers protocol 2", async () => { - const harness = await run(); - - expect(harness.availability()).toEqual({ - baseUrl: `http://127.0.0.1:${httpPort}`, - generation: 1, - leaseMaxTtlMs: 24 * 60 * 60_000, - platforms: ["ios", "android"], - state: "running", - token, - version: "0.21.28", - }); - expect(harness.health.reads).toContain(`http://127.0.0.1:${httpPort}`); - }); - - it("is not ready while daemon.json names another pid, and ready once it names the child", async () => { - const harness = await arrange({ daemonJson: () => "other-pid" }); - harness.begin(); - await advanceBy(harness, 5_000); - expect(stateOf(harness)).toBe("starting"); - - const pid = harness.children[0]?.pid; - await harness.filesystem.writeFileAtomic( - daemonJsonPath, - JSON.stringify({ httpPort, pid, token, version: "0.21.28" }), - ); - harness.probe.startListening(httpPort); - await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); - - expect(stateOf(harness)).toBe("running"); - }); - - it("is not ready while the port is not listening, and ready once it is", async () => { - const harness = await arrange({ listening: false }); - harness.begin(); - await advanceBy(harness, 5_000); - expect(stateOf(harness)).toBe("starting"); - - harness.probe.startListening(httpPort); - await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); - - expect(stateOf(harness)).toBe("running"); - }); - - it("is not ready while a health read rejects, and ready once it answers", async () => { - const harness = await arrange(); - harness.health.rejectWith(new Error("ECONNREFUSED")); - harness.begin(); - await advanceBy(harness, 5_000); - expect(stateOf(harness)).toBe("starting"); - - harness.health.respondWith(healthyBody); - await advanceUntil(harness, () => stateOf(harness) === "running", 10_000); - - expect(stateOf(harness)).toBe("running"); - }); - - it("stops the child and reports wrong-version naming both protocols when /health answers protocol 3", async () => { - const harness = await arrange(); - harness.health.respondWith({ rpcProtocolVersion: 3, service: "agent-device-daemon" }); - - await advancing(harness, harness.begin()); - - expect(unavailable(harness)).toMatchObject({ - message: "agent-device 0.21.28 speaks RPC protocol 3, need 2", - reason: "wrong-version", - }); - expect(unavailable(harness).remedy).toContain("RPC protocol 2"); - for (const phrase of restartCommand) expect(unavailable(harness).remedy).toContain(phrase); - expect(harness.processes.signals).toEqual([ - { pid: harness.children[0]?.pid, signal: "SIGTERM" }, - ]); - }); - - it.each([ - ["another service", { rpcProtocolVersion: 2, service: "something-else" }], - ["no numeric protocol", { rpcProtocolVersion: "2", service: "agent-device-daemon" }], - ])( - "stops the child and reports not-agent-device without retrying when /health answers with %s", - async (_case, body) => { - const harness = await arrange(); - harness.health.respondWith(body); - - await advancing(harness, harness.begin()); - - expect(unavailable(harness)).toMatchObject({ - message: "/health did not answer as agent-device", - reason: "not-agent-device", - }); - expect(harness.processes.signals).toEqual([ - { pid: harness.children[0]?.pid, signal: "SIGTERM" }, - ]); - await advanceBy(harness, 120_000); - expect(daemonSpawns(harness)).toHaveLength(1); - expect(stateOf(harness)).toBe("unavailable"); - }, - ); -}); - -describe("AgentDeviceSupervisor failed starts", () => { - it("kills the child after 15 s without readiness, removes the record only after it exits, and reports start-failed naming the timeout", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - const remove = harness.filesystem.rm.bind(harness.filesystem); - vi.spyOn(harness.filesystem, "rm").mockImplementation(async (path) => { - if (path === recordPath) harness.events.push("record removed"); - return remove(path); - }); - const spawnedAt = harness.clock.now(); - - await advancing(harness, harness.begin()); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("timed out"); - expect(unavailable(harness).remedy).toContain("30 s"); - expect(unavailable(harness).remedy).toContain("simlock daemon logs"); - expect(harness.clock.now() - spawnedAt).toBeGreaterThanOrEqual(15_000); - expect(harness.clock.now() - spawnedAt).toBeLessThan(16_000); - const pid = harness.children[0]?.pid; - expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); - expect(harness.events.slice(-2)).toEqual([`exit ${pid}`, "record removed"]); - await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); - }); - - it("reports start-failed naming the exit code when the child exits before readiness", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - harness.begin(); - await advanceBy(harness, 1_000); - - harness.children[0]?.exit(7); - await advancing(harness, harness.started as Promise); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("7"); - }); - - it("reports start-failed and writes no record when the spawn throws", async () => { - const harness = await run({ spawnThrows: new Error("EACCES: permission denied") }); - - expect(warnings(harness).some((line) => line.includes("EACCES"))).toBe(true); - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("EACCES"); - await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); - }); - - it("keeps the record and names the pid when the child is still alive 2 s after SIGKILL", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - harness.begin(); - await advanceBy(harness, 500); - const pid = harness.children[0]?.pid as number; - harness.processes.behaviours.set(pid, "immortal"); - - await advancing(harness, harness.started as Promise); - - expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain(String(pid)); - expect( - warnings(harness).some((line) => - line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after SIGKILL`), - ), - ).toBe(true); - expect(await readJson(harness, recordPath)).toMatchObject({ pid }); - }); - - it("resolves start() with start-failed carrying the message when the filesystem throws unexpectedly while writing the policy file", async () => { - const harness = await arrange(); - const write = harness.filesystem.writeFileAtomic.bind(harness.filesystem); - vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, ...rest) => { - if (path === policyPath) throw new Error("disk on fire"); - return write(path, ...rest); - }); - - await expect(advancing(harness, harness.begin())).resolves.toBeUndefined(); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("disk on fire"); - expect(daemonSpawns(harness)).toEqual([]); - }); -}); - -describe("AgentDeviceSupervisor reaping a daemon a killed simlock left", () => { - const recorded = { livePids: [900], record: record(900) }; - - it("sends SIGTERM to a live recorded pid whose line ends with the marker and holds the entry, removes the record once it is gone, and starts a new daemon", async () => { - const harness = await run({ - ...recorded, - commandLines: [[900, [ownLine()]]], - }); - - expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); - expect(harness.processes.isAlive(900)).toBe(false); - expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); - expect(stateOf(harness)).toBe("running"); - }); - - it("escalates to SIGKILL after 10 s when the recorded pid ignores SIGTERM and is still identified", async () => { - const harness = await arrange({ - ...recorded, - behaviours: new Map([[900, "ignores-sigterm"]]), - commandLines: [[900, [ownLine(), ownLine()]]], - }); - const startedAt = harness.clock.now(); - - await advancing(harness, harness.begin()); - - expect(harness.processes.signals).toEqual([ - { pid: 900, signal: "SIGTERM" }, - { pid: 900, signal: "SIGKILL" }, - ]); - expect(harness.clock.now() - startedAt).toBeGreaterThanOrEqual(10_000); - expect(stateOf(harness)).toBe("running"); - }); - - it("sends no SIGKILL to a pid whose second command line no longer passes the identity check", async () => { - const harness = await run({ - ...recorded, - behaviours: new Map([[900, "ignores-sigterm"]]), - commandLines: [[900, [ownLine(), "/usr/bin/vim notes.txt"]]], - }); - - expect(harness.processes.signals).toEqual([{ pid: 900, signal: "SIGTERM" }]); - }); - - it("keeps the record and reports start-failed naming the pid when it is still alive 2 s after SIGKILL", async () => { - const harness = await run({ - ...recorded, - behaviours: new Map([[900, "immortal"]]), - commandLines: [[900, [ownLine(), ownLine()]]], - expectations: [], - }); - - expect(harness.processes.signals.map((entry) => entry.signal)).toEqual(["SIGTERM", "SIGKILL"]); - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("900"); - expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); - expect(daemonSpawns(harness)).toEqual([]); - }); - - it("spawns with the resolved home as the marker when SIMLOCK_HOME is reached through a symlink, and a later start reaps that child", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/private/tmp/x"); - await filesystem.mkdirp("/tmp"); - filesystem.defineSymlink("/tmp/x", "/private/tmp/x"); - const first = await run({ - expectations: [versionRun(), daemonSpawn(internalEntry, {}, "/private/tmp/x")], - filesystem, - home: "/tmp/x", - }); - const childPid = first.children[0]?.pid as number; - expect(first.children[0]?.args).toEqual([internalEntry, "--simlock-home=/private/tmp/x"]); - - const second = await run({ - commandLines: [[childPid, [ownLine(internalEntry, "/private/tmp/x")]]], - expectations: [versionRun(), daemonSpawn(internalEntry, {}, "/private/tmp/x")], - filesystem, - home: "/tmp/x", - livePids: [childPid], - }); - - expect(second.processes.signals).toEqual([{ pid: childPid, signal: "SIGTERM" }]); - }); - - it("reaps the live pid daemon.json names when there is no record and the line ends with the marker", async () => { - const harness = await run({ - commandLines: [[901, [ownLine()]]], - livePids: [901], - stateDirContents: async (filesystem) => { - await filesystem.mkdir(stateDir, { mode: 0o700 }); - await filesystem.writeFileAtomic( - daemonJsonPath, - JSON.stringify({ httpPort, pid: 901, token }), - ); - }, - }); - - expect(harness.processes.signals).toEqual([{ pid: 901, signal: "SIGTERM" }]); - }); - - it("does not read daemon.json and signals nothing when the state directory is a symlink", async () => { - const harness = await run({ - expectations: [versionRun()], - livePids: [901], - stateDirContents: async (filesystem) => { - await filesystem.mkdirp("/elsewhere"); - await filesystem.writeFileAtomic("/elsewhere/daemon.json", JSON.stringify({ pid: 901 })); - filesystem.defineSymlink(stateDir, "/elsewhere"); - }, - }); - - expect(harness.processes.signals).toEqual([]); - expect(harness.processes.commandLineReads).toEqual([]); - expect(unavailable(harness).reason).toBe("state-dir"); - }); - - it.each([ - ["a dead recorded pid", [], undefined], - ["a command line of another command", [900], "/usr/bin/vim notes.txt"], - ["a line holding the entry but no marker", [900], `${nodePath} ${internalEntry}`], - ["a line with another home's marker", [900], ownLine(internalEntry, "/other/home")], - [ - "a line with this home's marker but another entry than the recorded one", - [900], - ownLine("/elsewhere/daemon.js"), - ], - ] as const)("removes the record and signals nothing for %s", async (_case, livePids, line) => { - // why: the spawn throws, so no new record can stand in for the old one: only a removal empties it. - const harness = await run({ - commandLines: line === undefined ? [] : [[900, [line]]], - livePids, - record: record(900), - spawnThrows: new Error("no spawn"), - }); - - expect(harness.processes.signals).toEqual([]); - expect(await harness.filesystem.exists(recordPath)).toBe(false); - }); - - it("signals nothing when daemon.json names another pid than the recorded one", async () => { - const harness = await run({ - commandLines: [[900, [ownLine()]]], - livePids: [900], - record: record(900), - stateDirContents: async (filesystem) => { - await filesystem.mkdir(stateDir, { mode: 0o700 }); - await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); - }, - }); - - expect(harness.processes.signals).toEqual([]); - expect(await readJson(harness, recordPath)).toMatchObject({ pid: harness.children[0]?.pid }); - expect(stateOf(harness)).toBe("running"); - }); - - it("removes the record when daemon.json names another pid than the recorded one", async () => { - const harness = await run({ - commandLines: [[900, [ownLine()]]], - livePids: [900], - record: record(900), - spawnThrows: new Error("no spawn"), - stateDirContents: async (filesystem) => { - await filesystem.mkdir(stateDir, { mode: 0o700 }); - await filesystem.writeFileAtomic(daemonJsonPath, JSON.stringify({ pid: 777 })); - }, - }); - - expect(await harness.filesystem.exists(recordPath)).toBe(false); - }); - - it("signals nothing, keeps the record, starts nothing and reports start-failed naming the pid when its command line cannot be read", async () => { - const harness = await run({ - commandLines: [[900, [undefined]]], - livePids: [900], - record: record(900), - }); - - expect(harness.processes.commandLineReads).toEqual([900]); - expect(harness.processes.signals).toEqual([]); - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toContain("900"); - expect(await readJson(harness, recordPath)).toMatchObject({ pid: 900 }); - expect(commandsRun(harness).filter((call) => call[0] === nodePath)).toEqual([]); - }); -}); - -describe("AgentDeviceSupervisor after the daemon is running", () => { - async function runningFor( - options: Options, - ms: number, - ): Promise<{ harness: Harness; readyAt: number }> { - const harness = await run(options); - expect(stateOf(harness)).toBe("running"); - const readyAt = harness.clock.now(); - await advanceBy(harness, ms); - return { harness, readyAt }; - } - - it("starts a child that exited 60 s after ready again at once with the same environment, as generation 2, and warns with the exit code", async () => { - const { harness } = await runningFor( - { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, - 60_000, - ); - const firstEnv = harness.runner.calls[1]?.options.env; - - harness.children[0]?.exit(42); - await flush(); - expect(stateOf(harness)).toBe("starting"); - await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); - - expect(daemonSpawns(harness)).toHaveLength(2); - expect(harness.runner.calls[3]?.options.env).toEqual(firstEnv); - expect(running(harness).generation).toBe(2); - expect( - warnings(harness).some( - (line) => line.includes("agent-device (simulators) daemon pid") && line.includes("42"), - ), - ).toBe(true); - }); - - it("reports start-failed for a child that exits 5 s after ready, and starts it again only 30 s later", async () => { - const { harness } = await runningFor( - { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, - 5_000, - ); - - harness.children[0]?.exit(9); - await flush(); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(unavailable(harness).message).toMatch( - /agent-device exited .*9.* 5 s after it was ready/, - ); - await advanceBy(harness, 29_000); - expect(daemonSpawns(harness)).toHaveLength(1); - await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); - expect(daemonSpawns(harness)).toHaveLength(2); - }); - - it("times the stable uptime from ready, not from the spawn: a child ready 10 s after the spawn that exits 62 s after it is start-failed", async () => { - const harness = await arrange({ - daemonJson: () => "none", - expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], - }); - void harness.begin(); - await advanceUntil(harness, () => harness.children.length === 1, 5_000); - const spawnedAt = harness.clock.now(); - await advanceBy(harness, 10_000); - await harness.filesystem.writeFileAtomic( - daemonJsonPath, - JSON.stringify({ httpPort, pid: harness.children[0]?.pid, token }), - ); - harness.probe.startListening(httpPort); - await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); - await advanceBy(harness, spawnedAt + 62_000 - harness.clock.now()); - - harness.children[0]?.exit(9); - await flush(); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(daemonSpawns(harness)).toHaveLength(1); - }); - - it("reports start-failed for a child that exits 59 s after ready instead of starting it at once", async () => { - const { harness } = await runningFor( - { expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()] }, - 59_000, - ); - - harness.children[0]?.exit(9); - await flush(); - - expect(unavailable(harness)).toMatchObject({ reason: "start-failed" }); - expect(daemonSpawns(harness)).toHaveLength(1); - }); - - it("retries a failed first start 30 s later and is running when that succeeds", async () => { - const harness = await arrange({ - daemonJson: (index) => (index === 0 ? "none" : "ready"), - expectations: [versionRun(), daemonSpawn(), versionRun(), daemonSpawn()], - }); - await advancing(harness, harness.begin()); - expect(unavailable(harness).reason).toBe("start-failed"); - const failedAt = harness.clock.now(); - - await advanceBy(harness, 29_000); - expect(daemonSpawns(harness)).toHaveLength(1); - await advanceUntil(harness, () => stateOf(harness) === "running", 5_000); - - expect(daemonSpawns(harness)).toHaveLength(2); - expect(harness.clock.now() - failedAt).toBeGreaterThanOrEqual(30_000); - expect(harness.clock.now() - failedAt).toBeLessThanOrEqual(30_500); - expect(running(harness).generation).toBe(1); - }); - - it("retries a start that failed after a restart 30 s later", async () => { - const harness = await arrange({ - daemonJson: (index) => (index === 1 ? "none" : "ready"), - expectations: [ - versionRun(), - daemonSpawn(), - versionRun(), - daemonSpawn(), - versionRun(), - daemonSpawn(), - ], - }); - await advancing(harness, harness.begin()); - await advanceBy(harness, 60_000); - harness.children[0]?.exit(1); - await advanceUntil(harness, () => unavailableState(harness), 30_000); - expect(unavailable(harness).reason).toBe("start-failed"); - - await advanceUntil(harness, () => stateOf(harness) === "running", 60_000); - - expect(daemonSpawns(harness)).toHaveLength(3); - expect(stateOf(harness)).toBe("running"); - }); - - function unavailableState(harness: Harness): boolean { - return stateOf(harness) === "unavailable"; - } - - it.each([ - ["not-found", { baseEnvironment: { PATH: "/nowhere" }, expectations: [] }], - [ - "not-agent-device", - { - expectations: [versionRun()], - layout: async (filesystem: MemoryFilesystem) => - filesystem.rm(`${packageRoot}/package.json`), - }, - ], - ["wrong-version", { expectations: [versionRun("0.20.3\n")] }], - [ - "state-dir", - { - expectations: [versionRun()], - stateDirContents: async (filesystem: MemoryFilesystem) => { - await filesystem.mkdirp("/elsewhere"); - filesystem.defineSymlink(stateDir, "/elsewhere"); - }, - }, - ], - ] satisfies [string, Options][])("does not retry a %s", async (reason, options) => { - const harness = await run(options); - expect(unavailable(harness).reason).toBe(reason); - const calls = harness.runner.calls.length; - - await advanceBy(harness, 120_000); - - expect(harness.runner.calls).toHaveLength(calls); - expect(unavailable(harness).reason).toBe(reason); - }); -}); - -describe("AgentDeviceSupervisor.stop", () => { - it("sends SIGTERM to the child, removes the record once it is gone, starts nothing more and leaves availability off", async () => { - const harness = await run(); - const pid = harness.children[0]?.pid; - const calls = harness.runner.calls.length; - - await advancing(harness, harness.agentDevice.stop()); - - expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); - await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); - expect(harness.events.slice(-1)).toEqual([`exit ${pid}`]); - await advanceBy(harness, 120_000); - expect(harness.runner.calls).toHaveLength(calls); - expect(stateOf(harness)).toBe("off"); - }); - - it("sends SIGKILL after 10 s when the child ignores SIGTERM", async () => { - const harness = await run(); - const pid = harness.children[0]?.pid as number; - harness.processes.behaviours.set(pid, "ignores-sigterm"); - const stoppedAt = harness.clock.now(); - - await advancing(harness, harness.agentDevice.stop()); - - expect(harness.processes.signals).toEqual([ - { pid, signal: "SIGTERM" }, - { pid, signal: "SIGKILL" }, - ]); - expect(harness.clock.now() - stoppedAt).toBeGreaterThanOrEqual(10_000); - await expect(harness.filesystem.exists(recordPath)).resolves.toBe(false); - }); - - it("keeps the record and warns naming the pid when the child is alive 2 s after SIGKILL, and availability still ends off", async () => { - const harness = await run(); - const pid = harness.children[0]?.pid as number; - harness.processes.behaviours.set(pid, "immortal"); - - await advancing(harness, harness.agentDevice.stop()); - - expect(await readJson(harness, recordPath)).toMatchObject({ pid }); - expect( - warnings(harness).some((line) => - line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after`), - ), - ).toBe(true); - expect(stateOf(harness)).toBe("off"); - }); - - it("warns naming the pid and keeps the record when stop() ends a start whose child survives SIGKILL", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - harness.begin(); - await advanceBy(harness, 1_000); - const pid = harness.children[0]?.pid as number; - harness.processes.behaviours.set(pid, "immortal"); - - await advancing(harness, harness.agentDevice.stop()); - await advancing(harness, harness.started as Promise); - - expect(await readJson(harness, recordPath)).toMatchObject({ pid }); - expect( - warnings(harness).some((line) => - line.includes(`agent-device (simulators) daemon pid ${pid} is still alive after`), - ), - ).toBe(true); - expect(stateOf(harness)).toBe("off"); - }); - - it("spawns nothing when stop() is asked while the state directory is being prepared", async () => { - const harness = await arrange(); - const writeFileAtomic = harness.filesystem.writeFileAtomic.bind(harness.filesystem); - let stopping: Promise | undefined; - vi.spyOn(harness.filesystem, "writeFileAtomic").mockImplementation(async (path, data) => { - if (path === `${stateDir}/policy.json`) stopping = harness.agentDevice.stop(); - await writeFileAtomic(path, data); - }); - - await advancing(harness, harness.begin()); - await advancing(harness, stopping as Promise); - - expect(daemonSpawns(harness)).toHaveLength(0); - expect(stateOf(harness)).toBe("off"); - }); - - it("cancels a pending retry", async () => { - const harness = await run({ daemonJson: () => "none" }); - expect(unavailable(harness).reason).toBe("start-failed"); - const calls = harness.runner.calls.length; - - await advancing(harness, harness.agentDevice.stop()); - await advanceBy(harness, 120_000); - - expect(harness.runner.calls).toHaveLength(calls); - expect(stateOf(harness)).toBe("off"); - }); - - it("ends a start in progress with the child killed and availability off", async () => { - const harness = await arrange({ daemonJson: () => "none" }); - harness.begin(); - await advanceBy(harness, 1_000); - const pid = harness.children[0]?.pid; - - await advancing(harness, harness.agentDevice.stop()); - await advancing(harness, harness.started as Promise); - - expect(harness.processes.signals).toEqual([{ pid, signal: "SIGTERM" }]); - expect(stateOf(harness)).toBe("off"); - await advanceBy(harness, 120_000); - expect(daemonSpawns(harness)).toHaveLength(1); - expect(stateOf(harness)).toBe("off"); - }); - - it("leaves no unhandled rejection when a health read rejects after stop() was asked during the port check", async () => { - const harness = await arrange(); - harness.health.rejectWith(new Error("ECONNREFUSED")); - const isListening = harness.probe.isListening.bind(harness.probe); - let stopping: Promise | undefined; - vi.spyOn(harness.probe, "isListening").mockImplementation(async (port) => { - stopping = harness.agentDevice.stop(); - return isListening(port); - }); - const unhandled: unknown[] = []; - const record = (reason: unknown): void => { - unhandled.push(reason); - }; - process.on("unhandledRejection", record); - try { - harness.begin(); - await advanceBy(harness, 5_000); - await advancing(harness, stopping as Promise); - await new Promise((resolve) => setImmediate(resolve)); - } finally { - process.off("unhandledRejection", record); - } - - expect(unhandled).toEqual([]); - expect(stateOf(harness)).toBe("off"); - }); -}); diff --git a/src/agent-device/host/test-agent-device-supervisor.ts b/src/agent-device/host/test-agent-device-supervisor.ts new file mode 100644 index 000000000..f0abbf5a2 --- /dev/null +++ b/src/agent-device/host/test-agent-device-supervisor.ts @@ -0,0 +1,377 @@ +import { vi } from "vitest"; +import { + FakeClock, + FakeProcessSupervisor, + FakeTcpProbe, + JsonLinesLogger, + MemoryFilesystem, + MemoryLogSink, + ScriptedProcessRunner, + type ProcessResult, + type ScriptedProcessExpectation, +} from "../../ports/index.js"; +import { FakeAgentDeviceHealth } from "../testing.js"; +import { + AgentDeviceSupervisor, + type AgentDeviceAvailability, + type AgentDeviceIosSigning, + type AgentDevicePlatform, + type AgentDeviceRole, +} from "./agent-device-supervisor.js"; + +export const home = "/home/simlock/.simlock"; + +export const stateDir = `${home}/agent-device`; + +export const recordPath = `${home}/agent-device-daemon.json`; + +export const policyPath = `${stateDir}/policy.json`; + +export const daemonJsonPath = `${stateDir}/daemon.json`; + +export const nodePath = "/opt/node/bin/node"; + +export const packageRoot = "/opt/ad/node_modules/agent-device"; + +export const realCli = `${packageRoot}/bin/agent-device.mjs`; + +export const internalEntry = `${packageRoot}/dist/src/internal/daemon.js`; + +export const fallbackEntry = `${packageRoot}/dist/src/daemon.js`; + +export const httpPort = 4310; + +export const token = "daemon-token"; + +export const healthyBody = { rpcProtocolVersion: 2, service: "agent-device-daemon" }; + +export const restartCommand = ["simlock daemon stop", "simlock daemon start"]; + +export type Behaviour = "dies" | "ignores-sigterm" | "immortal"; + +export type DaemonJson = "ready" | "none" | "other-pid"; + +export class ProcessTable extends FakeProcessSupervisor { + readonly behaviours = new Map(); + readonly onDeath = new Map void>(); + + override signal(pid: number, signal: NodeJS.Signals): void { + super.signal(pid, signal); + this.apply(pid, signal); + } + + apply(pid: number, signal: NodeJS.Signals): void { + const behaviour = this.behaviours.get(pid) ?? "dies"; + if (behaviour === "immortal") return; + if (behaviour === "ignores-sigterm" && signal !== "SIGKILL") return; + this.markDead(pid); + this.onDeath.get(pid)?.(); + } +} + +export interface Child { + readonly pid: number; + readonly args: readonly string[]; + exit(code: number | null): void; +} + +export interface Options { + readonly baseEnvironment?: NodeJS.ProcessEnv; + readonly command?: string; + readonly commandLines?: readonly (readonly [number, readonly (string | undefined)[]])[]; + readonly daemonJson?: (startIndex: number) => DaemonJson; + readonly driverEnvironment?: Readonly>; + readonly expectations?: readonly ScriptedProcessExpectation[]; + readonly filesystem?: MemoryFilesystem; + readonly home?: string; + readonly ios?: AgentDeviceIosSigning; + readonly layout?: (filesystem: MemoryFilesystem) => Promise; + readonly leaseMaxTtlMs?: number; + readonly listening?: boolean; + readonly livePids?: readonly number[]; + readonly nodeVersion?: string; + readonly platforms?: readonly AgentDevicePlatform[]; + readonly role?: AgentDeviceRole; + readonly record?: { readonly pid: number; readonly entry: string; readonly startedAt: number }; + readonly spawnThrows?: Error; + readonly stateDirContents?: (filesystem: MemoryFilesystem) => Promise; + readonly behaviours?: ReadonlyMap; +} + +export function stemOf(role: AgentDeviceRole | undefined): string { + return role === "phones" ? "agent-device-phones" : "agent-device"; +} + +export function supervisorRole(options: Options): { + readonly ios?: AgentDeviceIosSigning; + readonly role: AgentDeviceRole; +} { + return { + ...(options.ios === undefined ? {} : { ios: options.ios }), + role: options.role ?? "simulators", + }; +} + +export async function arrange(options: Options = {}) { + const clock = new FakeClock(1_000_000); + const stem = stemOf(options.role); + const filesystem = options.filesystem ?? new MemoryFilesystem(); + const simlockHome = options.home ?? home; + const sink = new MemoryLogSink(); + const logger = new JsonLinesLogger({ clock, level: "debug", sink }); + const probe = new FakeTcpProbe(); + const health = new FakeAgentDeviceHealth(); + const processes = new ProcessTable(options.livePids ?? []); + for (const [pid, behaviour] of options.behaviours ?? []) processes.behaviours.set(pid, behaviour); + for (const [pid, lines] of options.commandLines ?? []) processes.answerCommandLine(pid, ...lines); + const runner = new ScriptedProcessRunner(options.expectations ?? [versionRun(), daemonSpawn()]); + const children: Child[] = []; + const events: string[] = []; + let starts = 0; + + health.respondWith(healthyBody); + await installAgentDevice(filesystem, simlockHome, options); + if (options.record !== undefined) { + await filesystem.writeFileAtomic( + `${simlockHome}/${stem}-daemon.json`, + JSON.stringify(options.record), + ); + } + await options.stateDirContents?.(filesystem); + + const spawn = runner.spawn.bind(runner); + vi.spyOn(runner, "spawn").mockImplementation((command, args, spawnOptions) => { + const handle = spawn(command, args, spawnOptions); + if (command !== nodePath || !isDaemonEntry(args[0])) return handle; + if (options.spawnThrows !== undefined) throw options.spawnThrows; + const pid = handle.pid; + let exit!: (result: ProcessResult) => void; + const exited = new Promise((resolve) => { + exit = resolve; + }); + const child: Child = { + args: [...args], + exit(code) { + events.push(`exit ${pid}`); + processes.markDead(pid); + exit({ code, stderr: "", stdout: "" }); + }, + pid, + }; + children.push(child); + processes.markAlive(pid); + processes.onDeath.set(pid, () => child.exit(null)); + events.push(`spawn ${pid}`); + const startIndex = starts; + starts += 1; + const mode = options.daemonJson?.(startIndex) ?? "ready"; + if (mode !== "none") { + void filesystem.writeFileAtomic( + `${simlockHome}/${stem}/daemon.json`, + JSON.stringify({ + httpPort, + pid: mode === "ready" ? pid : pid + 1_000, + token, + version: "0.21.28", + }), + ); + } + if (mode === "ready" && options.listening !== false) probe.startListening(httpPort); + return { + kill(signal = "SIGTERM") { + processes.signals.push({ pid, signal }); + processes.apply(pid, signal); + handle.kill(signal); + }, + pid, + stderr: handle.stderr, + stdout: handle.stdout, + unref: () => handle.unref(), + wait: () => Promise.race([handle.wait(), exited]), + }; + }); + + const agentDevice = new AgentDeviceSupervisor({ + baseEnvironment: options.baseEnvironment ?? { + HOME: "/home/operator", + PATH: "/usr/local/bin:/usr/bin", + }, + clock, + config: { command: options.command ?? "agent-device" }, + driverEnvironment: options.driverEnvironment ?? {}, + filesystem, + healthProbe: health, + home: simlockHome, + hostPlatform: "linux", + ...supervisorRole(options), + leaseMaxTtlMs: options.leaseMaxTtlMs ?? 4 * 60 * 60_000, + logger, + nodePath, + nodeVersion: options.nodeVersion ?? "22.14.0", + platforms: options.platforms ?? ["ios", "android"], + processRunner: runner, + processSupervisor: processes, + tcpProbe: probe, + }); + const harness = { + agentDevice, + children, + clock, + events, + filesystem, + health, + probe, + processes, + runner, + sink, + started: undefined as Promise | undefined, + begin(): Promise { + const started = agentDevice.start(); + started.catch(() => undefined); + harness.started = started; + return started; + }, + availability: () => agentDevice.availability(), + }; + return harness; +} + +export type Harness = Awaited>; + +export async function run(options: Options = {}): Promise { + const harness = await arrange(options); + await advancing(harness, harness.begin()); + return harness; +} + +export function isDaemonEntry(arg: string | undefined): boolean { + return arg === internalEntry || arg === fallbackEntry; +} + +export async function installAgentDevice( + filesystem: MemoryFilesystem, + simlockHome: string, + options: Options, +): Promise { + if (!(await filesystem.exists(simlockHome))) await filesystem.mkdirp(simlockHome); + if (options.command?.includes("/") !== true) { + await filesystem.mkdirp("/usr/local/bin"); + } + await filesystem.mkdirp(`${packageRoot}/bin`); + await filesystem.mkdirp(`${packageRoot}/dist/src/internal`); + await filesystem.writeFileAtomic(realCli, "#!/usr/bin/env node\n"); + filesystem.defineAttributes(realCli, { mode: 0o755 }); + await filesystem.writeFileAtomic(`${packageRoot}/package.json`, '{"name":"agent-device"}'); + await filesystem.writeFileAtomic(internalEntry, ""); + filesystem.defineSymlink("/usr/local/bin/agent-device", realCli); + await options.layout?.(filesystem); +} + +export function versionRun(output = "0.21.28\n", extra: Partial = {}) { + return { + match: { args: [realCli, "--version"], command: nodePath }, + result: { code: 0, stderr: "", stdout: output }, + ...extra, + } satisfies ScriptedProcessExpectation; +} + +export function daemonSpawn( + entry = internalEntry, + extra: Partial = {}, + simlockHome = home, + marker: readonly string[] = [], +): ScriptedProcessExpectation { + return { + hangs: true, + match: { args: [entry, `--simlock-home=${simlockHome}`, ...marker], command: nodePath }, + ...extra, + }; +} + +export function phoneSpawn(simlockHome = home): ScriptedProcessExpectation { + return daemonSpawn(internalEntry, {}, simlockHome, ["--simlock-role=phones"]); +} + +export function ownLine(entry = internalEntry, simlockHome = home, role?: "phones"): string { + const suffix = role === undefined ? "" : " --simlock-role=phones"; + return `${nodePath} ${entry} --simlock-home=${simlockHome}${suffix}`; +} + +export function record(pid: number, entry = internalEntry) { + return { entry, pid, startedAt: 1 }; +} + +export async function flush(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +export async function advanceBy(harness: Harness, ms: number): Promise { + for (let elapsed = 0; elapsed < ms; elapsed += 100) { + await flush(); + harness.clock.advance(100); + } + await flush(); +} + +export async function advancing(harness: Harness, work: Promise): Promise { + let settled = false; + const tracked = work.finally(() => { + settled = true; + }); + tracked.catch(() => undefined); + for (let tick = 0; tick < 4_000 && !settled; tick += 1) { + await flush(); + harness.clock.advance(100); + } + return tracked; +} + +export async function advanceUntil( + harness: Harness, + done: () => boolean, + limitMs = 120_000, +): Promise { + for (let elapsed = 0; elapsed < limitMs && !done(); elapsed += 100) { + await flush(); + harness.clock.advance(100); + } + await flush(); +} + +export function stateOf(harness: Harness): AgentDeviceAvailability["state"] { + return harness.availability().state; +} + +export function unavailable(harness: Harness) { + const availability = harness.availability(); + if (availability.state !== "unavailable") { + throw new Error(`expected unavailable, got ${availability.state}`); + } + return availability; +} + +export function running(harness: Harness) { + const availability = harness.availability(); + if (availability.state !== "running") { + throw new Error(`expected running, got ${availability.state}`); + } + return availability; +} + +export function commandsRun(harness: Harness): string[][] { + return harness.runner.calls.map((call) => [call.command, ...call.args]); +} + +export function daemonSpawns(harness: Harness): string[][] { + return commandsRun(harness).filter((call) => call[0] === nodePath && isDaemonEntry(call[1])); +} + +export function warnings(harness: Harness): string[] { + return harness.sink.records + .filter((entry) => entry.level === "warn") + .map((entry) => JSON.stringify(entry)); +} + +export async function readJson(harness: Harness, path: string): Promise> { + return JSON.parse(await harness.filesystem.readFile(path)) as Record; +} From 69604613b9dd096e77e06fe7c2144d1a962b5313 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:32:30 +0100 Subject: [PATCH 49/83] test(agent-device): split edge-request-rules.test.ts by topic --- .../edge-request-rules-devices.test.ts | 543 ++++++++++++++++++ src/agent-device/edge-request-rules.test.ts | 537 +---------------- src/agent-device/test-edge-request-rules.ts | 16 + 3 files changed, 564 insertions(+), 532 deletions(-) create mode 100644 src/agent-device/edge-request-rules-devices.test.ts create mode 100644 src/agent-device/test-edge-request-rules.ts diff --git a/src/agent-device/edge-request-rules-devices.test.ts b/src/agent-device/edge-request-rules-devices.test.ts new file mode 100644 index 000000000..8b3b4ca7e --- /dev/null +++ b/src/agent-device/edge-request-rules-devices.test.ts @@ -0,0 +1,543 @@ +import { describe, expect, it } from "vitest"; +import { tokenIdentity } from "../auth/testing.js"; +import { WorkerView } from "../modules/index.js"; +import { fakeGatewayContext } from "../modules/testing.js"; +import { agentDeviceModule } from "./definition.js"; +import { + phoneSerial, + setUpWithPhone, + listed, + tapDispatch, + simlockPicks, + idOf, + batch, + release, +} from "./test-edge.js"; +import { + secrets, + adminSession, + type Rig, + setUp, + envelopeOf, + hex, + granted, + leases, + refusal, + runCommand, + forwarded, + refused, +} from "./test-support.js"; +import { bothDrivers, commandsReceived, flagsOf } from "./test-edge-request-rules.js"; + +// why: the decomposed front's gateway adoption is exercised at its HTTP mount with a fake fleet view. +describe("the agent-device front's request rules", () => { + describe("the devices filter", () => { + const three = [{ id: "d-1" }, { id: "d-2" }, { id: "d-3" }]; + const listedIds = async (response: Response): Promise => { + const answer = await envelopeOf(response); + const data = answer.result?.["data"] as { devices: { id: string }[] } | undefined; + expect(Array.isArray(data?.devices), "expected a devices list in result.data").toBe(true); + return data?.devices.map((device) => device.id); + }; + + it("`devices` before open answers an empty list while agent-device lists three devices; after open it answers only the leased device, without agent B's", async () => { + const rig = await setUp(); + await granted(rig, 2, {}, { token: secrets.two }); + rig.fake.listDevices([...three, { id: idOf(rig, 2) }]); + const before = commandsReceived(rig).length; + + expect(await listedIds(await runCommand(rig, "devices", undefined))).toEqual([]); + expect(commandsReceived(rig)).toHaveLength(before); + await granted(rig, 1); + rig.fake.listDevices([...three, { id: idOf(rig, 2) }, { id: idOf(rig, 1) }]); + + expect(await listedIds(await runCommand(rig, "devices", 1))).toEqual([idOf(rig, 1)]); + }); + + it("`devices` for an agent holding two devices answers both, and after simlock released one of the two leases (no heartbeat sent since), only the other", async () => { + const rig = await setUp(); + await granted(rig, 1, { ttlMs: 60_000 }); + await granted(rig, 2, { ttlMs: 600_000 }); + rig.fake.listDevices([...three, { id: idOf(rig, 1) }, { id: idOf(rig, 2) }]); + + expect( + ((await listedIds(await runCommand(rig, "devices", undefined))) as string[]).sort(), + ).toEqual([idOf(rig, 1), idOf(rig, 2)].sort()); + await rig.daemon.dispatch("lease.release", { leaseId: hex(1) }, adminSession); + await expect.poll(() => listed(rig)).toEqual([hex(2)]); + + expect(await listedIds(await runCommand(rig, "devices", undefined))).toEqual([idOf(rig, 2)]); + }); + + it("a `devices` error answer from agent-device passes unchanged; a success with no list at `data.devices`, a body over 1 MiB, a non-JSON body and a list holding a non-object each answer `could not read the device list from the worker holding lease `", async () => { + const rig: Rig = await setUp(); + await granted(rig, 1); + rig.fake.listDevices([{ id: idOf(rig, 1) }]); + expect(await listedIds(await runCommand(rig, "devices", 1))).toEqual([idOf(rig, 1)]); + + const failure = { + error: { + code: -32000, + data: { code: "COMMAND_FAILED", message: "inventory down" }, + message: "inventory down", + }, + id: 1, + jsonrpc: "2.0", + }; + rig.fake.answerDevicesWith(JSON.stringify(failure)); + expect(await envelopeOf(await runCommand(rig, "devices", 1))).toMatchObject({ + error: { message: "inventory down" }, + }); + const unusable = [ + JSON.stringify({ id: 1, jsonrpc: "2.0", result: { data: {}, ok: true } }), + JSON.stringify({ + id: 1, + jsonrpc: "2.0", + result: { + data: { devices: [{ id: idOf(rig, 1) }], pad: "x".repeat(1024 * 1024) }, + ok: true, + }, + }), + "not json", + JSON.stringify({ + id: 1, + jsonrpc: "2.0", + result: { data: { devices: [{ id: idOf(rig, 1) }, "stray"] }, ok: true }, + }), + ]; + for (const body of unusable) { + rig.fake.answerDevicesWith(body); + expect((await envelopeOf(await runCommand(rig, "devices", 1))).error).toMatchObject({ + data: { + message: `could not read the device list from the worker holding lease ${hex(1)}`, + }, + }); + } + }); + + // why: the agent-device CLI sets `meta.requestProgress: "command"` on every command it sends without `--json`. + it("`devices` sent as the agent-device CLI sends it, with `meta.requestProgress`, answers an empty list before open and only the leased device after, without agent B's, and reaches agent-device without `requestProgress`, while another command keeps it", async () => { + const rig = await setUp(); + await granted(rig, 2, {}, { token: secrets.two }); + const asTheCli = { meta: { requestProgress: "command" } }; + rig.fake.listDevices([...three, { id: idOf(rig, 2) }]); + + expect(await listedIds(await runCommand(rig, "devices", undefined, asTheCli))).toEqual([]); + await granted(rig, 1); + rig.fake.listDevices([...three, { id: idOf(rig, 2) }, { id: idOf(rig, 1) }]); + const before = commandsReceived(rig).length; + + expect(await listedIds(await runCommand(rig, "devices", 1, asTheCli))).toEqual([ + idOf(rig, 1), + ]); + const sent = commandsReceived(rig) + .slice(before) + .map( + (r) => (r.params?.["meta"] as Record | undefined)?.["requestProgress"], + ); + expect(sent).toEqual([undefined]); + + const beforeSnapshot = commandsReceived(rig).length; + await runCommand(rig, "snapshot", 1, asTheCli); + const kept = commandsReceived(rig) + .slice(beforeSnapshot) + .map( + (r) => (r.params?.["meta"] as Record | undefined)?.["requestProgress"], + ); + expect(kept).toEqual(["command"]); + }); + }); + + describe("the token's leased devices", () => { + it("a `test` shard fill whose lease check fails answers `COMMAND_FAILED` with `could not check your leases; try again` and reaches nothing", async () => { + const rig = await setUp(); + await granted(rig, 1); + const tap = tapDispatch(rig); + const sharded = () => runCommand(rig, "test", 1, { flags: { shardAll: 2 } }); + await forwarded(rig, sharded); + tap.rejectNext("lease.list"); + const before = rig.fake.requests.length; + + const answer = await envelopeOf(await sharded()); + + expect(answer.error).toMatchObject({ + data: { + code: "COMMAND_FAILED", + message: "could not check your leases; try again", + details: { reason: "SIMLOCK_FAILED" }, + retriable: true, + }, + }); + expect(rig.fake.requests.length).toBe(before); + }); + + it("a `devices` whose lease check fails answers `COMMAND_FAILED` with `could not check your leases; try again`, not an empty list", async () => { + const rig = await setUp(); + await granted(rig, 1); + rig.fake.listDevices([{ id: idOf(rig, 1) }]); + const tap = tapDispatch(rig); + tap.rejectNext("lease.list"); + + const answer = await envelopeOf(await runCommand(rig, "devices", 1)); + + expect(answer.error).toMatchObject({ + data: { code: "COMMAND_FAILED", message: "could not check your leases; try again" }, + }); + }); + + it("a batch whose `batchSteps` is not a list is refused with `refused: malformed batchSteps`", async () => { + const rig = await setUp(); + await forwarded(rig, () => batch(rig, undefined, [{ command: "snapshot" }])); + + await refused( + rig, + () => runCommand(rig, "batch", undefined, { flags: { batchSteps: "snapshot" } }), + "refused: malformed batchSteps", + ); + }); + + it("a `flags`, `input` or `runtime` that is not an object, a batch step that is not an object or whose `command` is not a string, and a `meta.deviceKey` that is not a string are each refused with their `malformed` message", async () => { + const rig = await setUp(); + await granted(rig, 1); + await forwarded(rig, () => runCommand(rig, "snapshot", 1, { flags: {} })); + + for (const key of ["flags", "input", "runtime"]) { + await refused( + rig, + () => runCommand(rig, "snapshot", 1, { [key]: "text" }), + `refused: malformed ${key}`, + ); + } + await refused( + rig, + () => batch(rig, 1, ["snapshot"] as unknown as Record[]), + "refused: malformed batchSteps", + ); + await refused(rig, () => batch(rig, 1, [{ command: 5 }]), "refused: malformed command"); + await refused( + rig, + () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: 5 } }), + "refused: malformed deviceKey", + ); + }); + }); + + describe("rule 10, commands agent-device runs without a lease", () => { + it("`capabilities` with no selector reaches agent-device with the leased device's id; from a token holding no device it is refused; `doctor` the same", async () => { + const rig = await setUp(); + await granted(rig, 1); + for (const name of ["capabilities", "doctor"]) { + const params = await forwarded(rig, () => runCommand(rig, name, 1)); + expect(flagsOf(params)["udid"]).toBe(idOf(rig, 1)); + await refused( + rig, + () => runCommand(rig, name, undefined, {}, { token: secrets.two }), + "refused: name the device with --udid ; run devices to see yours", + ); + } + }); + + it("`capabilities` naming no lease is filled with the token's only device; with two devices it is refused unless `platform` leaves one, and then it gets that device's `serial` on Android", async () => { + const rig = await setUp({ drivers: bothDrivers }); + await granted(rig, 1, {}, { token: secrets.two }); + const only = await forwarded(rig, () => + runCommand(rig, "capabilities", undefined, {}, { token: secrets.two }), + ); + expect(flagsOf(only)["udid"]).toBe(idOf(rig, 1)); + + await granted(rig, 2, { device: "Pixel 8", platform: "android" }, { token: secrets.two }); + await refused( + rig, + () => runCommand(rig, "capabilities", undefined, {}, { token: secrets.two }), + "refused: name the device with --udid ; run devices to see yours", + ); + const android = await forwarded(rig, () => + runCommand( + rig, + "capabilities", + undefined, + { flags: { platform: "android" } }, + { token: secrets.two }, + ), + ); + expect(flagsOf(android)["serial"]).toBe(idOf(rig, 2)); + expect(flagsOf(android)).not.toHaveProperty("udid"); + }); + }); + + type BootScenario = { + platform: string; + model: string; + kind: string; + target: string; + omitTarget?: boolean; + flags: Record; + runtime?: Record; + physical?: boolean; + }; + type AdoptedScenario = { + platform: "ios" | "android"; + model: string; + deviceClass: string; + physical: boolean; + kind: string; + target: string; + address: string; + }; + function bootFlagsFor(scenario: BootScenario, address: string): Record { + if ("udid" in scenario.flags) return { udid: address }; + if ("device" in scenario.flags) return { device: `simlock-${address}` }; + return scenario.flags; + } + function allocateFieldsFor(scenario: BootScenario): Record { + return { + platform: scenario.platform, + ...("omitTarget" in scenario ? {} : { target: scenario.target }), + ...(scenario.physical ? { physical: true } : {}), + }; + } + async function bootsOnItsLease(scenario: BootScenario): Promise { + const rig: Rig = scenario.physical + ? await setUpWithPhone() + : await setUp({ drivers: bothDrivers }); + rig.fake.nameDeviceInNextReply("none"); + await granted(rig, 1, allocateFieldsFor(scenario)); + const address = scenario.physical ? phoneSerial : idOf(rig, 1); + const answer = await envelopeOf( + await runCommand(rig, "boot", 1, { + flags: bootFlagsFor(scenario, address), + ...(scenario.runtime && { runtime: scenario.runtime }), + }), + ); + expect(answer.result).toMatchObject({ + ok: true, + data: { + platform: scenario.platform, + target: scenario.target, + device: "device" in scenario.flags ? `simlock-${address}` : scenario.model, + id: address, + kind: scenario.kind, + booted: true, + }, + }); + expect( + commandsReceived(rig).filter((request) => request.params?.["command"] === "boot"), + ).toEqual([]); + } + async function bootsOnAdoptedLease(scenario: AdoptedScenario): Promise { + const leaseId = hex(1); + const entry = { + gatewayLeaseId: leaseId, + workerLeaseId: leaseId, + workerId: "w1", + ownerId: "tok_one", + requesterId: leaseId, + }; + const view = { + id: "w1", + label: "w1", + connection: "connected", + drained: false, + catalogReadAt: 1_000, + lease: { maxTtlMs: 120_000 }, + catalog: [ + { platform: scenario.platform, modelClasses: { [scenario.model]: scenario.deviceClass } }, + ], + leases: [{ id: leaseId, deviceId: "record", ttlMs: 120_000, ttlDeadline: 121_000 }], + } as unknown as WorkerView; + const { ctx, relayed } = await fakeGatewayContext( + "agent-device", + agentDeviceModule.config.defaults, + { serving: [view] }, + ); + const loaded = await agentDeviceModule.loadGateway!(); + const module = loaded.createGateway({ + ...ctx, + leases: { ...ctx.leases, all: () => [entry], resolve: () => entry }, + grantedDevices: () => [ + { + id: "record", + workerId: "w1", + driverDeviceId: scenario.address, + address: scenario.physical ? "not-the-phone-id" : scenario.address, + physical: scenario.physical, + spec: { + platform: scenario.platform, + model: scenario.model, + physical: scenario.physical, + }, + }, + ], + } as never); + const handle = module.mount!({ + dispatch: (() => + Promise.resolve({ leases: [{ id: leaseId, ttlDeadline: 121_000 }] })) as never, + }); + const adopted = await envelopeOf( + await handle( + new Request("http://simlock.test/v1/agent-device/rpc", { + method: "POST", + body: JSON.stringify({ + id: 1, + jsonrpc: "2.0", + method: "agent_device.command", + params: { command: "boot", meta: { leaseId } }, + }), + }), + tokenIdentity("tok_one"), + ), + ); + expect(adopted.result).toMatchObject({ + ok: true, + data: { + platform: scenario.platform, + target: scenario.target, + device: scenario.model, + id: scenario.address, + kind: scenario.kind, + booted: true, + }, + }); + expect(relayed).toEqual([]); + } + it("A runner sends `boot` on its lease and simlock answers it; agent-device gets no `boot`", async () => { + for (const scenario of [ + { + platform: "ios", + model: "iPhone 16", + kind: "simulator", + target: "mobile", + omitTarget: true, + flags: {}, + }, + { platform: "ios", model: "iPhone 16", kind: "simulator", target: "mobile", flags: {} }, + { + platform: "ios", + model: "iPhone 16", + kind: "simulator", + target: "mobile", + flags: { target: "tv" }, + }, + { + platform: "ios", + model: "iPhone 16", + kind: "simulator", + target: "mobile", + flags: {}, + runtime: { udid: "another-device" }, + }, + { platform: "android", model: "Pixel 8", kind: "emulator", target: "mobile", flags: {} }, + { platform: "ios", model: "Apple TV 4K", kind: "simulator", target: "tv", flags: {} }, + { + platform: "ios", + model: "iPhone 16", + kind: "simulator", + target: "mobile", + flags: { udid: true }, + }, + { + platform: "ios", + model: "iPhone 16", + kind: "simulator", + target: "mobile", + flags: { device: true }, + }, + { + platform: "android", + model: "Pixel 8", + kind: "device", + target: "mobile", + flags: {}, + physical: true, + }, + ] satisfies BootScenario[]) { + await bootsOnItsLease(scenario); + } + for (const scenario of [ + { + platform: "ios", + model: "Apple TV 4K", + deviceClass: "tv", + physical: false, + kind: "simulator", + target: "tv", + address: "adopted-udid", + }, + { + platform: "ios", + model: "iPhone", + deviceClass: "phone", + physical: true, + kind: "device", + target: "mobile", + address: "iphone-id", + }, + { + platform: "android", + model: "Pixel", + deviceClass: "phone", + physical: true, + kind: "device", + target: "mobile", + address: "pixel-id", + }, + ] satisfies AdoptedScenario[]) { + await bootsOnAdoptedLease(scenario); + } + }); + it("A `boot` that names no lease, another token's lease, an ended lease or another device is refused", async () => { + const rig = await setUp({ drivers: bothDrivers }); + await granted(rig, 1); + await granted(rig, 2, {}, { token: secrets.two }); + expect((await envelopeOf(await runCommand(rig, "boot", 1))).result).toMatchObject({ ok: true }); + const tap = tapDispatch(rig); + tap.rejectNext("lease.list"); + expect((await envelopeOf(await runCommand(rig, "boot", 1))).error?.data).toMatchObject({ + message: "could not check your leases; try again", + retriable: true, + }); + for (const [lease, fields, message] of [ + [undefined, {}, "refused: boot names no lease; open a device first"], + [2, {}, simlockPicks], + [1, { flags: { udid: idOf(rig, 2) } }, simlockPicks], + [1, { flags: { platform: "android" } }, simlockPicks], + [1, { input: { platform: "android" } }, simlockPicks], + [1, { runtime: { platform: "android" } }, simlockPicks], + [undefined, { flags: { udid: idOf(rig, 1) } }, simlockPicks], + ] as const) { + expect((await envelopeOf(await runCommand(rig, "boot", lease, fields))).error).toMatchObject( + refusal(message), + ); + } + await rig.daemon.dispatch("lease.release", { leaseId: hex(1) }, adminSession); + expect((await envelopeOf(await runCommand(rig, "boot", 1))).error).toMatchObject( + refusal("lease ended by simlock; run disconnect, then open", "UNAUTHORIZED"), + ); + await granted(rig, 3); + await release(rig, 3); + expect((await envelopeOf(await runCommand(rig, "boot", 3))).error).toMatchObject( + refusal("refused: this lease has ended; open a device"), + ); + for (const fields of [{}, { flags: { udid: true } }, { flags: { device: true } }]) { + const expired = await setUp({ drivers: bothDrivers }); + await granted(expired, 4, { ttlMs: 5_000 }); + const address = idOf(expired, 4); + const selector = + "udid" in (fields.flags ?? {}) + ? { flags: { udid: address } } + : "device" in (fields.flags ?? {}) + ? { flags: { device: `simlock-${address}` } } + : {}; + expect((await envelopeOf(await runCommand(expired, "boot", 4))).result).toMatchObject({ + ok: true, + }); + expired.clock.advance(5_001); + await expect.poll(() => leases(expired)).toEqual([]); + const before = commandsReceived(expired).length; + expect( + (await envelopeOf(await runCommand(expired, "boot", 4, selector))).error, + ).toMatchObject(refusal("lease ended by simlock; run disconnect, then open", "UNAUTHORIZED")); + expect(commandsReceived(expired)).toHaveLength(before); + } + }); +}); diff --git a/src/agent-device/edge-request-rules.test.ts b/src/agent-device/edge-request-rules.test.ts index 41119f235..b2ca5ff4a 100644 --- a/src/agent-device/edge-request-rules.test.ts +++ b/src/agent-device/edge-request-rules.test.ts @@ -1,21 +1,12 @@ import { describe, expect, it, vi } from "vitest"; - -import { FakeDriver } from "../core/testing.js"; -import { tokenIdentity } from "../auth/testing.js"; -import { FakeClock } from "../ports/index.js"; -import type { WorkerView } from "../modules/index.js"; -import { fakeGatewayContext } from "../modules/testing.js"; -import { agentDeviceModule } from "./definition.js"; import { phoneSerial, setUpWithPhone, advance, listed, - tapDispatch, simlockPicks, idOf, batch, - release, } from "./test-edge.js"; import { secrets, @@ -24,7 +15,6 @@ import { adminSession, sessionFor, type Rig, - iosDriver, androidDriver, setUp, send, @@ -34,42 +24,35 @@ import { hex, allocate, granted, - leases, requestedSpecs, refusal, runCommand, forwarded, refused, } from "./test-support.js"; +import { bothDrivers, commandsReceived, flagsOf } from "./test-edge-request-rules.js"; // why: Fallow reads a `rig` binding's type only from an annotation in this file, not from an imported // setUp's return type, so a `rig` whose test calls a FakeAgentDevice member only these tests use is // annotated `Rig`; without it Fallow reports the member as never referenced. const onlyIosAndAndroid = "refused: only iOS and Android"; + const uploadOnly = "refused: send the app from your machine as an upload"; + const noDeviceLeased = "refused: no device leased; run open first"; + const deviceSetRefusal = "refused: simlock chooses the device set; run disconnect, then connect proxy without --ios-simulator-device-set"; + const allowlistRefusal = "refused: simlock chooses the devices; run disconnect, then connect proxy without --android-device-allowlist"; -const bothDrivers = (clock: FakeClock): FakeDriver[] => [iosDriver(clock), androidDriver(clock)]; - // why: the fake agent-device names the device it leases `simlock-`, as the real one does. function nameOf(rig: Rig, n: number): string { return `simlock-${idOf(rig, n)}`; } -function commandsReceived(rig: Rig) { - return rig.fake.rpcRequests("agent_device.command"); -} - -function flagsOf(params: Record): Record { - return (params["flags"] ?? {}) as Record; -} - -// why: the decomposed front's gateway adoption is exercised at its HTTP mount with a fake fleet view. describe("the agent-device front's request rules", () => { describe("rule 1, the route", () => { it.each([ @@ -797,514 +780,4 @@ describe("the agent-device front's request rules", () => { } }); }); - - describe("the devices filter", () => { - const three = [{ id: "d-1" }, { id: "d-2" }, { id: "d-3" }]; - const listedIds = async (response: Response): Promise => { - const answer = await envelopeOf(response); - const data = answer.result?.["data"] as { devices: { id: string }[] } | undefined; - expect(Array.isArray(data?.devices), "expected a devices list in result.data").toBe(true); - return data?.devices.map((device) => device.id); - }; - - it("`devices` before open answers an empty list while agent-device lists three devices; after open it answers only the leased device, without agent B's", async () => { - const rig = await setUp(); - await granted(rig, 2, {}, { token: secrets.two }); - rig.fake.listDevices([...three, { id: idOf(rig, 2) }]); - const before = commandsReceived(rig).length; - - expect(await listedIds(await runCommand(rig, "devices", undefined))).toEqual([]); - expect(commandsReceived(rig)).toHaveLength(before); - await granted(rig, 1); - rig.fake.listDevices([...three, { id: idOf(rig, 2) }, { id: idOf(rig, 1) }]); - - expect(await listedIds(await runCommand(rig, "devices", 1))).toEqual([idOf(rig, 1)]); - }); - - it("`devices` for an agent holding two devices answers both, and after simlock released one of the two leases (no heartbeat sent since), only the other", async () => { - const rig = await setUp(); - await granted(rig, 1, { ttlMs: 60_000 }); - await granted(rig, 2, { ttlMs: 600_000 }); - rig.fake.listDevices([...three, { id: idOf(rig, 1) }, { id: idOf(rig, 2) }]); - - expect( - ((await listedIds(await runCommand(rig, "devices", undefined))) as string[]).sort(), - ).toEqual([idOf(rig, 1), idOf(rig, 2)].sort()); - await rig.daemon.dispatch("lease.release", { leaseId: hex(1) }, adminSession); - await expect.poll(() => listed(rig)).toEqual([hex(2)]); - - expect(await listedIds(await runCommand(rig, "devices", undefined))).toEqual([idOf(rig, 2)]); - }); - - it("a `devices` error answer from agent-device passes unchanged; a success with no list at `data.devices`, a body over 1 MiB, a non-JSON body and a list holding a non-object each answer `could not read the device list from the worker holding lease `", async () => { - const rig: Rig = await setUp(); - await granted(rig, 1); - rig.fake.listDevices([{ id: idOf(rig, 1) }]); - expect(await listedIds(await runCommand(rig, "devices", 1))).toEqual([idOf(rig, 1)]); - - const failure = { - error: { - code: -32000, - data: { code: "COMMAND_FAILED", message: "inventory down" }, - message: "inventory down", - }, - id: 1, - jsonrpc: "2.0", - }; - rig.fake.answerDevicesWith(JSON.stringify(failure)); - expect(await envelopeOf(await runCommand(rig, "devices", 1))).toMatchObject({ - error: { message: "inventory down" }, - }); - const unusable = [ - JSON.stringify({ id: 1, jsonrpc: "2.0", result: { data: {}, ok: true } }), - JSON.stringify({ - id: 1, - jsonrpc: "2.0", - result: { - data: { devices: [{ id: idOf(rig, 1) }], pad: "x".repeat(1024 * 1024) }, - ok: true, - }, - }), - "not json", - JSON.stringify({ - id: 1, - jsonrpc: "2.0", - result: { data: { devices: [{ id: idOf(rig, 1) }, "stray"] }, ok: true }, - }), - ]; - for (const body of unusable) { - rig.fake.answerDevicesWith(body); - expect((await envelopeOf(await runCommand(rig, "devices", 1))).error).toMatchObject({ - data: { - message: `could not read the device list from the worker holding lease ${hex(1)}`, - }, - }); - } - }); - - // why: the agent-device CLI sets `meta.requestProgress: "command"` on every command it sends without `--json`. - it("`devices` sent as the agent-device CLI sends it, with `meta.requestProgress`, answers an empty list before open and only the leased device after, without agent B's, and reaches agent-device without `requestProgress`, while another command keeps it", async () => { - const rig = await setUp(); - await granted(rig, 2, {}, { token: secrets.two }); - const asTheCli = { meta: { requestProgress: "command" } }; - rig.fake.listDevices([...three, { id: idOf(rig, 2) }]); - - expect(await listedIds(await runCommand(rig, "devices", undefined, asTheCli))).toEqual([]); - await granted(rig, 1); - rig.fake.listDevices([...three, { id: idOf(rig, 2) }, { id: idOf(rig, 1) }]); - const before = commandsReceived(rig).length; - - expect(await listedIds(await runCommand(rig, "devices", 1, asTheCli))).toEqual([ - idOf(rig, 1), - ]); - const sent = commandsReceived(rig) - .slice(before) - .map( - (r) => (r.params?.["meta"] as Record | undefined)?.["requestProgress"], - ); - expect(sent).toEqual([undefined]); - - const beforeSnapshot = commandsReceived(rig).length; - await runCommand(rig, "snapshot", 1, asTheCli); - const kept = commandsReceived(rig) - .slice(beforeSnapshot) - .map( - (r) => (r.params?.["meta"] as Record | undefined)?.["requestProgress"], - ); - expect(kept).toEqual(["command"]); - }); - }); - - describe("the token's leased devices", () => { - it("a `test` shard fill whose lease check fails answers `COMMAND_FAILED` with `could not check your leases; try again` and reaches nothing", async () => { - const rig = await setUp(); - await granted(rig, 1); - const tap = tapDispatch(rig); - const sharded = () => runCommand(rig, "test", 1, { flags: { shardAll: 2 } }); - await forwarded(rig, sharded); - tap.rejectNext("lease.list"); - const before = rig.fake.requests.length; - - const answer = await envelopeOf(await sharded()); - - expect(answer.error).toMatchObject({ - data: { - code: "COMMAND_FAILED", - message: "could not check your leases; try again", - details: { reason: "SIMLOCK_FAILED" }, - retriable: true, - }, - }); - expect(rig.fake.requests.length).toBe(before); - }); - - it("a `devices` whose lease check fails answers `COMMAND_FAILED` with `could not check your leases; try again`, not an empty list", async () => { - const rig = await setUp(); - await granted(rig, 1); - rig.fake.listDevices([{ id: idOf(rig, 1) }]); - const tap = tapDispatch(rig); - tap.rejectNext("lease.list"); - - const answer = await envelopeOf(await runCommand(rig, "devices", 1)); - - expect(answer.error).toMatchObject({ - data: { code: "COMMAND_FAILED", message: "could not check your leases; try again" }, - }); - }); - - it("a batch whose `batchSteps` is not a list is refused with `refused: malformed batchSteps`", async () => { - const rig = await setUp(); - await forwarded(rig, () => batch(rig, undefined, [{ command: "snapshot" }])); - - await refused( - rig, - () => runCommand(rig, "batch", undefined, { flags: { batchSteps: "snapshot" } }), - "refused: malformed batchSteps", - ); - }); - - it("a `flags`, `input` or `runtime` that is not an object, a batch step that is not an object or whose `command` is not a string, and a `meta.deviceKey` that is not a string are each refused with their `malformed` message", async () => { - const rig = await setUp(); - await granted(rig, 1); - await forwarded(rig, () => runCommand(rig, "snapshot", 1, { flags: {} })); - - for (const key of ["flags", "input", "runtime"]) { - await refused( - rig, - () => runCommand(rig, "snapshot", 1, { [key]: "text" }), - `refused: malformed ${key}`, - ); - } - await refused( - rig, - () => batch(rig, 1, ["snapshot"] as unknown as Record[]), - "refused: malformed batchSteps", - ); - await refused(rig, () => batch(rig, 1, [{ command: 5 }]), "refused: malformed command"); - await refused( - rig, - () => runCommand(rig, "snapshot", 1, { meta: { deviceKey: 5 } }), - "refused: malformed deviceKey", - ); - }); - }); - - describe("rule 10, commands agent-device runs without a lease", () => { - it("`capabilities` with no selector reaches agent-device with the leased device's id; from a token holding no device it is refused; `doctor` the same", async () => { - const rig = await setUp(); - await granted(rig, 1); - for (const name of ["capabilities", "doctor"]) { - const params = await forwarded(rig, () => runCommand(rig, name, 1)); - expect(flagsOf(params)["udid"]).toBe(idOf(rig, 1)); - await refused( - rig, - () => runCommand(rig, name, undefined, {}, { token: secrets.two }), - "refused: name the device with --udid ; run devices to see yours", - ); - } - }); - - it("`capabilities` naming no lease is filled with the token's only device; with two devices it is refused unless `platform` leaves one, and then it gets that device's `serial` on Android", async () => { - const rig = await setUp({ drivers: bothDrivers }); - await granted(rig, 1, {}, { token: secrets.two }); - const only = await forwarded(rig, () => - runCommand(rig, "capabilities", undefined, {}, { token: secrets.two }), - ); - expect(flagsOf(only)["udid"]).toBe(idOf(rig, 1)); - - await granted(rig, 2, { device: "Pixel 8", platform: "android" }, { token: secrets.two }); - await refused( - rig, - () => runCommand(rig, "capabilities", undefined, {}, { token: secrets.two }), - "refused: name the device with --udid ; run devices to see yours", - ); - const android = await forwarded(rig, () => - runCommand( - rig, - "capabilities", - undefined, - { flags: { platform: "android" } }, - { token: secrets.two }, - ), - ); - expect(flagsOf(android)["serial"]).toBe(idOf(rig, 2)); - expect(flagsOf(android)).not.toHaveProperty("udid"); - }); - }); - - type BootScenario = { - platform: string; - model: string; - kind: string; - target: string; - omitTarget?: boolean; - flags: Record; - runtime?: Record; - physical?: boolean; - }; - type AdoptedScenario = { - platform: "ios" | "android"; - model: string; - deviceClass: string; - physical: boolean; - kind: string; - target: string; - address: string; - }; - function bootFlagsFor(scenario: BootScenario, address: string): Record { - if ("udid" in scenario.flags) return { udid: address }; - if ("device" in scenario.flags) return { device: `simlock-${address}` }; - return scenario.flags; - } - function allocateFieldsFor(scenario: BootScenario): Record { - return { - platform: scenario.platform, - ...("omitTarget" in scenario ? {} : { target: scenario.target }), - ...(scenario.physical ? { physical: true } : {}), - }; - } - async function bootsOnItsLease(scenario: BootScenario): Promise { - const rig: Rig = scenario.physical - ? await setUpWithPhone() - : await setUp({ drivers: bothDrivers }); - rig.fake.nameDeviceInNextReply("none"); - await granted(rig, 1, allocateFieldsFor(scenario)); - const address = scenario.physical ? phoneSerial : idOf(rig, 1); - const answer = await envelopeOf( - await runCommand(rig, "boot", 1, { - flags: bootFlagsFor(scenario, address), - ...(scenario.runtime && { runtime: scenario.runtime }), - }), - ); - expect(answer.result).toMatchObject({ - ok: true, - data: { - platform: scenario.platform, - target: scenario.target, - device: "device" in scenario.flags ? `simlock-${address}` : scenario.model, - id: address, - kind: scenario.kind, - booted: true, - }, - }); - expect( - commandsReceived(rig).filter((request) => request.params?.["command"] === "boot"), - ).toEqual([]); - } - async function bootsOnAdoptedLease(scenario: AdoptedScenario): Promise { - const leaseId = hex(1); - const entry = { - gatewayLeaseId: leaseId, - workerLeaseId: leaseId, - workerId: "w1", - ownerId: "tok_one", - requesterId: leaseId, - }; - const view = { - id: "w1", - label: "w1", - connection: "connected", - drained: false, - catalogReadAt: 1_000, - lease: { maxTtlMs: 120_000 }, - catalog: [ - { platform: scenario.platform, modelClasses: { [scenario.model]: scenario.deviceClass } }, - ], - leases: [{ id: leaseId, deviceId: "record", ttlMs: 120_000, ttlDeadline: 121_000 }], - } as unknown as WorkerView; - const { ctx, relayed } = await fakeGatewayContext( - "agent-device", - agentDeviceModule.config.defaults, - { serving: [view] }, - ); - const loaded = await agentDeviceModule.loadGateway!(); - const module = loaded.createGateway({ - ...ctx, - leases: { ...ctx.leases, all: () => [entry], resolve: () => entry }, - grantedDevices: () => [ - { - id: "record", - workerId: "w1", - driverDeviceId: scenario.address, - address: scenario.physical ? "not-the-phone-id" : scenario.address, - physical: scenario.physical, - spec: { - platform: scenario.platform, - model: scenario.model, - physical: scenario.physical, - }, - }, - ], - } as never); - const handle = module.mount!({ - dispatch: (() => - Promise.resolve({ leases: [{ id: leaseId, ttlDeadline: 121_000 }] })) as never, - }); - const adopted = await envelopeOf( - await handle( - new Request("http://simlock.test/v1/agent-device/rpc", { - method: "POST", - body: JSON.stringify({ - id: 1, - jsonrpc: "2.0", - method: "agent_device.command", - params: { command: "boot", meta: { leaseId } }, - }), - }), - tokenIdentity("tok_one"), - ), - ); - expect(adopted.result).toMatchObject({ - ok: true, - data: { - platform: scenario.platform, - target: scenario.target, - device: scenario.model, - id: scenario.address, - kind: scenario.kind, - booted: true, - }, - }); - expect(relayed).toEqual([]); - } - it("A runner sends `boot` on its lease and simlock answers it; agent-device gets no `boot`", async () => { - for (const scenario of [ - { - platform: "ios", - model: "iPhone 16", - kind: "simulator", - target: "mobile", - omitTarget: true, - flags: {}, - }, - { platform: "ios", model: "iPhone 16", kind: "simulator", target: "mobile", flags: {} }, - { - platform: "ios", - model: "iPhone 16", - kind: "simulator", - target: "mobile", - flags: { target: "tv" }, - }, - { - platform: "ios", - model: "iPhone 16", - kind: "simulator", - target: "mobile", - flags: {}, - runtime: { udid: "another-device" }, - }, - { platform: "android", model: "Pixel 8", kind: "emulator", target: "mobile", flags: {} }, - { platform: "ios", model: "Apple TV 4K", kind: "simulator", target: "tv", flags: {} }, - { - platform: "ios", - model: "iPhone 16", - kind: "simulator", - target: "mobile", - flags: { udid: true }, - }, - { - platform: "ios", - model: "iPhone 16", - kind: "simulator", - target: "mobile", - flags: { device: true }, - }, - { - platform: "android", - model: "Pixel 8", - kind: "device", - target: "mobile", - flags: {}, - physical: true, - }, - ] satisfies BootScenario[]) { - await bootsOnItsLease(scenario); - } - for (const scenario of [ - { - platform: "ios", - model: "Apple TV 4K", - deviceClass: "tv", - physical: false, - kind: "simulator", - target: "tv", - address: "adopted-udid", - }, - { - platform: "ios", - model: "iPhone", - deviceClass: "phone", - physical: true, - kind: "device", - target: "mobile", - address: "iphone-id", - }, - { - platform: "android", - model: "Pixel", - deviceClass: "phone", - physical: true, - kind: "device", - target: "mobile", - address: "pixel-id", - }, - ] satisfies AdoptedScenario[]) { - await bootsOnAdoptedLease(scenario); - } - }); - it("A `boot` that names no lease, another token's lease, an ended lease or another device is refused", async () => { - const rig = await setUp({ drivers: bothDrivers }); - await granted(rig, 1); - await granted(rig, 2, {}, { token: secrets.two }); - expect((await envelopeOf(await runCommand(rig, "boot", 1))).result).toMatchObject({ ok: true }); - const tap = tapDispatch(rig); - tap.rejectNext("lease.list"); - expect((await envelopeOf(await runCommand(rig, "boot", 1))).error?.data).toMatchObject({ - message: "could not check your leases; try again", - retriable: true, - }); - for (const [lease, fields, message] of [ - [undefined, {}, "refused: boot names no lease; open a device first"], - [2, {}, simlockPicks], - [1, { flags: { udid: idOf(rig, 2) } }, simlockPicks], - [1, { flags: { platform: "android" } }, simlockPicks], - [1, { input: { platform: "android" } }, simlockPicks], - [1, { runtime: { platform: "android" } }, simlockPicks], - [undefined, { flags: { udid: idOf(rig, 1) } }, simlockPicks], - ] as const) { - expect((await envelopeOf(await runCommand(rig, "boot", lease, fields))).error).toMatchObject( - refusal(message), - ); - } - await rig.daemon.dispatch("lease.release", { leaseId: hex(1) }, adminSession); - expect((await envelopeOf(await runCommand(rig, "boot", 1))).error).toMatchObject( - refusal("lease ended by simlock; run disconnect, then open", "UNAUTHORIZED"), - ); - await granted(rig, 3); - await release(rig, 3); - expect((await envelopeOf(await runCommand(rig, "boot", 3))).error).toMatchObject( - refusal("refused: this lease has ended; open a device"), - ); - for (const fields of [{}, { flags: { udid: true } }, { flags: { device: true } }]) { - const expired = await setUp({ drivers: bothDrivers }); - await granted(expired, 4, { ttlMs: 5_000 }); - const address = idOf(expired, 4); - const selector = - "udid" in (fields.flags ?? {}) - ? { flags: { udid: address } } - : "device" in (fields.flags ?? {}) - ? { flags: { device: `simlock-${address}` } } - : {}; - expect((await envelopeOf(await runCommand(expired, "boot", 4))).result).toMatchObject({ - ok: true, - }); - expired.clock.advance(5_001); - await expect.poll(() => leases(expired)).toEqual([]); - const before = commandsReceived(expired).length; - expect( - (await envelopeOf(await runCommand(expired, "boot", 4, selector))).error, - ).toMatchObject(refusal("lease ended by simlock; run disconnect, then open", "UNAUTHORIZED")); - expect(commandsReceived(expired)).toHaveLength(before); - } - }); }); diff --git a/src/agent-device/test-edge-request-rules.ts b/src/agent-device/test-edge-request-rules.ts new file mode 100644 index 000000000..00f93bd0f --- /dev/null +++ b/src/agent-device/test-edge-request-rules.ts @@ -0,0 +1,16 @@ +import { FakeDriver } from "../core/testing.js"; +import { FakeClock } from "../ports/index.js"; +import { type Rig, iosDriver, androidDriver } from "./test-support.js"; + +export const bothDrivers = (clock: FakeClock): FakeDriver[] => [ + iosDriver(clock), + androidDriver(clock), +]; + +export function commandsReceived(rig: Rig) { + return rig.fake.rpcRequests("agent_device.command"); +} + +export function flagsOf(params: Record): Record { + return (params["flags"] ?? {}) as Record; +} From 13b49fe8a443ee1414f830ceea34ce5ca579b8e6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:25:26 +0100 Subject: [PATCH 50/83] test(core): split policy.test.ts by topic --- .../warm-pool/policy-learned-kinds.test.ts | 305 ++++ .../warm-pool/policy-memory-pressure.test.ts | 143 ++ .../warm-pool/policy-targeted-devices.test.ts | 179 ++ src/core/warm-pool/policy-targets.test.ts | 694 ++++++++ src/core/warm-pool/policy.test.ts | 1490 +---------------- src/core/warm-pool/test-policy.ts | 230 +++ 6 files changed, 1564 insertions(+), 1477 deletions(-) create mode 100644 src/core/warm-pool/policy-learned-kinds.test.ts create mode 100644 src/core/warm-pool/policy-memory-pressure.test.ts create mode 100644 src/core/warm-pool/policy-targeted-devices.test.ts create mode 100644 src/core/warm-pool/policy-targets.test.ts create mode 100644 src/core/warm-pool/test-policy.ts diff --git a/src/core/warm-pool/policy-learned-kinds.test.ts b/src/core/warm-pool/policy-learned-kinds.test.ts new file mode 100644 index 000000000..2b8ee7f4b --- /dev/null +++ b/src/core/warm-pool/policy-learned-kinds.test.ts @@ -0,0 +1,305 @@ +import { describe, expect, it } from "vitest"; +import type { VirtualDevice, VirtualSpec } from "../domain.js"; +import { evaluate as plan } from "./policy.js"; +import { + evaluate, + minute, + now, + shutdownAfterMs, + spec, + device, + learnedOf, + modelDemand, + view, +} from "./test-policy.js"; + +// why: the pure planner needs many capacity, matching and ordering cases through its policy seam. +describe("warm pool policy", () => { + describe("targets", () => { + const kind = spec("iPhone 17"); + const ofKind = (id: string, state: VirtualDevice["state"], extra = {}): VirtualDevice => + device(id, state, { spec: kind, ...extra }); + + describe("learned kinds", () => { + const older = spec("iPad Pro"); + const provision = (wanted: VirtualSpec) => ({ + action: "provision", + reason: "target", + spec: wanted, + target: wanted, + }); + + it("fills configured targets in config order, then learned kinds newest first, booting a shut-down device of the kind before creating", () => { + const learned = [learnedOf(kind, 1, now - minute), learnedOf(older, 1, now - 2 * minute)]; + + const first = plan(view([], { learned, limit: 1, maxConcurrentBoots: 2 })); + expect(first.proposals).toEqual([provision(kind)]); + expect(first.targets.map((report) => [report.target, report.origin, report.short])).toEqual( + [ + ["ios iPhone 17 26.0", "learned", undefined], + ["ios iPad Pro 26.0", "learned", "running-limit"], + ], + ); + + const configured = spec("iPhone 15"); + expect( + evaluate(view([], { learned, limit: 1, targets: [{ count: 1, spec: configured }] })), + ).toEqual([provision(configured)]); + + const shut = ofKind("shut", "shutdown"); + expect(evaluate(view([shut], { learned: [learnedOf(kind, 1, now)] }))).toEqual([ + { action: "boot", deviceId: "shut", reason: "target", target: kind }, + ]); + + const merged = plan( + view([ofKind("a", "leased")], { + learned: [learnedOf(kind, 2, now), learnedOf(older, 1, now - minute)], + limit: 2, + targets: [{ count: 1, spec: kind }], + }), + ); + expect(merged.proposals).toEqual([provision(kind)]); + expect( + merged.targets.map((report) => [report.origin, report.count, report.configuredCount]), + ).toEqual([ + ["configured", 2, 1], + ["learned", 1, 0], + ]); + }); + + it("plans enough copies to keep max(peak - leased, 1) of a learned kind ready, counting copies on their way, for a plain kind and a baseline kind alike", () => { + const baselined = { ...kind, baseline: { name: "demo", version: 1 } }; + const cases = [ + ["peak 1, 1 leased", [ofKind("a", "leased")], 1, kind, [], [provision(kind)]], + ["peak 1, 1 reclaiming", [ofKind("a", "reclaiming")], 1, kind, [], [provision(kind)]], + [ + "peak 1, 1 ready under a claim", + [ofKind("a", "ready")], + 1, + kind, + ["a"], + [provision(kind)], + ], + [ + "peak 3, 3 leased", + [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "leased")], + 3, + kind, + [], + [provision(kind)], + ], + [ + "peak 3, 2 leased + 1 reclaiming", + [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "reclaiming")], + 3, + kind, + [], + [provision(kind)], + ], + [ + "peak 3, 1 leased + 1 ready", + [ofKind("a", "leased"), ofKind("b", "ready")], + 3, + kind, + [], + [provision(kind)], + ], + [ + "peak 3, 3 leased + 1 boot in flight", + [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "leased")], + 3, + kind, + [], + [], + [kind], + ], + [ + "baseline kind, peak 1, 1 leased", + [ofKind("a", "leased", { spec: baselined })], + 1, + baselined, + [], + [provision(baselined)], + ], + [ + "baseline kind, peak 2, 2 copies of version 1 leased while version 2 is newest", + [ + ofKind("a", "leased", { spec: baselined }), + ofKind("b", "leased", { spec: baselined }), + ], + 2, + { ...kind, baseline: { name: "demo", version: 2 } }, + [], + [provision({ ...kind, baseline: { name: "demo", version: 2 } })], + ], + [ + "baseline kind, peak 2, 2 copies of version 1 on another model leased while version 2 is newest", + [ + ofKind("a", "leased", { spec: { ...baselined, model: "iPhone 15" } }), + ofKind("b", "leased", { spec: { ...baselined, model: "iPhone 15" } }), + ], + 2, + { ...kind, baseline: { name: "demo", version: 2 } }, + [], + [provision({ ...kind, baseline: { name: "demo", version: 2 } })], + ], + [ + "baseline kind, peak 2, 2 copies of version 1 on another platform leased while version 2 is newest", + [ + ofKind("a", "leased", { spec: { ...baselined, platform: "android" } }), + ofKind("b", "leased", { spec: { ...baselined, platform: "android" } }), + ], + 2, + { ...kind, baseline: { name: "demo", version: 2 } }, + [], + [provision({ ...kind, baseline: { name: "demo", version: 2 } })], + ], + ] as const; + + for (const [name, devices, peak, wanted, claimed, expected, inFlight = []] of cases) { + expect( + evaluate( + view(devices, { + claimed, + inFlight, + learned: [learnedOf(wanted, peak, now)], + maxConcurrentBoots: 3, + }), + ), + name, + ).toEqual(expected); + } + }); + + it("plans the larger of what the configured count and the learned count each lack for a kind both configured and learned", () => { + const cases = [ + ["configured 1 + peak 2, 1 leased", 1, 2, 1], + ["configured 2 + peak 1, 1 leased", 2, 1, 2], + ["configured 1 + peak 3, 1 leased", 1, 3, 2], + ] as const; + + for (const [name, configuredCount, peak, created] of cases) { + expect( + evaluate( + view([ofKind("a", "leased")], { + learned: [learnedOf(kind, peak, now)], + maxConcurrentBoots: 3, + targets: [{ count: configuredCount, spec: kind }], + }), + ), + name, + ).toEqual(Array.from({ length: created }, () => provision(kind))); + } + }); + + it("keeps a learned kind's never-leased device out of never-leased-idle while the kind wants it, and proposes one beyond its count after idle.shutdownAfterMs", () => { + const old = now - shutdownAfterMs - 1; + const wanted = ofKind("wanted", "ready", { createdAt: 1, readyAt: old }); + const extra = ofKind("extra", "ready", { createdAt: 2, readyAt: old }); + const young = ofKind("young", "ready", { createdAt: 3, readyAt: now - 1_000 }); + const learned = [learnedOf(kind, 1, now)]; + + expect(evaluate(view([wanted], { learned }))).toEqual([]); + expect(evaluate(view([wanted, extra], { learned }))).toEqual([ + { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, + ]); + expect(evaluate(view([wanted, young], { learned }))).toEqual([]); + expect(evaluate(view([wanted, extra], { learned: [learnedOf(kind, 2, now)] }))).toEqual([]); + }); + + it("keeps a learned kind and its devices but plans nothing for it while it is unplanned, and still plans a configured target of its spec", () => { + const learned = [learnedOf(kind, 1, now)]; + const old = now - shutdownAfterMs - 1; + const spare = ofKind("spare", "ready", { createdAt: 1, readyAt: old }); + + expect(evaluate(view([], { learned, unplanned: [kind] }))).toEqual([]); + expect(evaluate(view([spare], { learned, unplanned: [kind] }))).toEqual([]); + expect(evaluate(view([], { learned }))).toEqual([provision(kind)]); + expect( + evaluate(view([], { learned, targets: [{ count: 1, spec: kind }], unplanned: [kind] })), + ).toEqual([provision(kind)]); + }); + + it("displaces the least recently used idle ready device of an older learned kind for a newer one short for a running slot, and never a configured target's device, one that serves a waiting request or is already shut down for the budget, or for a kind short for ram-budget, device-limit or reserve; a configured target displaces it", () => { + const learned = [learnedOf(kind, 1, now - minute), learnedOf(older, 2, now - 2 * minute)]; + const recent = device("recent", "ready", { endedAgo: 5 * minute, spec: older }); + const stale = device("stale", "ready", { endedAgo: 30 * minute, spec: older }); + const displace = { action: "shutdown", deviceId: "stale", reason: "displaced" }; + + const base = plan(view([recent, stale], { learned, limit: 2, maxConcurrentBoots: 2 })); + expect(base.proposals).toEqual([displace, provision(kind)]); + expect(base.targets.map((report) => [report.target, report.ready, report.short])).toEqual([ + ["ios iPhone 17 26.0", 0, undefined], + ["ios iPad Pro 26.0", 1, "running-limit"], + ]); + + const refused = (options: Parameters[1]) => + evaluate(view([recent, stale], { learned, maxConcurrentBoots: 2, ...options })); + expect(refused({ limit: 2 })).toEqual([displace, provision(kind)]); + expect(refused({ limit: 2, targets: [{ count: 1, spec: older }] })).toEqual([]); + expect(refused({ limit: 2, waiting: [modelDemand("iPad Pro")] })).toEqual([]); + expect(evaluate(view([recent], { learned, limit: 1, maxConcurrentBoots: 2 }))).toEqual([ + { ...displace, deviceId: "recent" }, + provision(kind), + ]); + expect(refused({ handoffInFlight: true, limit: 2 })).toEqual([]); + const holder = device("holder", "leased", { spec: spec("iPad Air") }); + expect( + evaluate( + view([holder, stale], { + learned: [learnedOf(kind, 2, now - minute), learnedOf(older, 1, now - 2 * minute)], + limit: 1, + maxConcurrentBoots: 2, + }), + ), + ).toEqual([{ action: "shutdown", deviceId: "stale", reason: "over-budget" }]); + const droid = device("droid", "ready", { + endedAgo: 40 * minute, + spec: spec("Pixel 8", { platform: "android" }), + }); + expect( + evaluate( + view([droid], { + learned: [learnedOf(kind, 1, now), learnedOf(droid.spec, 1, now - minute)], + limit: 2, + limits: { ios: 0 }, + maxConcurrentBoots: 2, + }), + ), + ).toEqual([]); + expect(refused({ limit: 3, refuseProvision: "ram-budget" })).toEqual([]); + // why: short for a slot and for RAM the shutdown would not free: no shutdown, or the pool would reboot the victim every pass. + expect(refused({ limit: 2, refuseProvision: "ram-budget" })).toEqual([]); + expect(refused({ limit: 2, refuseProvisionUntilDown: "ram-budget" })).toEqual([ + displace, + provision(kind), + ]); + expect(refused({ limit: 2, spared: [{ count: 2, spec: older }] })).toEqual([]); + expect(refused({ limit: 3, refuseProvision: "device-limit" })).toEqual([]); + expect(refused({ limit: 3, reserve: { ios: 1 } })).toEqual([]); + + const newer = device("newer", "ready", { endedAgo: 30 * minute, spec: kind }); + expect( + evaluate( + view([newer], { + learned, + limit: 1, + maxConcurrentBoots: 2, + }), + ), + ).toEqual([]); + const phone = spec("iPhone 15"); + expect( + evaluate( + view([recent], { + learned: [learnedOf(older, 1, now)], + limit: 1, + maxConcurrentBoots: 2, + targets: [{ count: 1, spec: phone }], + }), + ), + ).toEqual([{ ...displace, deviceId: "recent" }, provision(phone)]); + }); + }); + }); +}); diff --git a/src/core/warm-pool/policy-memory-pressure.test.ts b/src/core/warm-pool/policy-memory-pressure.test.ts new file mode 100644 index 000000000..5f3f3063c --- /dev/null +++ b/src/core/warm-pool/policy-memory-pressure.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import type { VirtualDevice } from "../domain.js"; +import { evaluate as plan } from "./policy.js"; +import { + evaluate, + minute, + spec, + device, + learnedOf, + classDemand, + modelDemand, + view, +} from "./test-policy.js"; + +// why: the pure planner needs many capacity, matching and ordering cases through its policy seam. +describe("warm pool policy", () => { + describe("memory pressure", () => { + const kind = spec("iPhone 17"); + const paused = { mayShed: false, step: "warming-paused" as const }; + const waiter = (inFlight = false) => classDemand("full", "ios", inFlight, "iPhone 17"); + + it("at warming-paused starts nothing and reports every short target as memory-pressure, in place of any other reason", () => { + const busy = device("busy", "leased", { spec: spec("iPad Pro") }); + const shut = device("shut", "shutdown", { endedAgo: 20 * minute, spec: kind }); + const cases = [ + { name: "configured target", options: { targets: [{ count: 1, spec: kind }] } }, + { name: "learned kind", options: { learned: [learnedOf(kind)] } }, + { + name: "configured target also short for the reserve", + options: { + limits: { ios: 2 }, + reserve: { ios: 1 }, + targets: [{ count: 1, spec: kind }], + }, + other: "reserve", + proposes: false, + }, + { + name: "configured target also short for a boot that failed", + options: { blocked: [kind], targets: [{ count: 1, spec: kind }] }, + other: "boot-failed", + proposes: false, + }, + { + name: "creation for a queued request", + options: { maxConcurrentBoots: 3, waiting: [waiter(true), waiter(), waiter()] }, + }, + { + name: "boot of a shut-down device for a queued request", + devices: [busy, shut], + options: { waiting: [waiter()] }, + }, + ]; + + const refused = plan( + view([busy], { refuseProvision: "memory-pressure", targets: [{ count: 1, spec: kind }] }), + ); + expect(refused.targets[0]?.short, "a capacity refusal for memory pressure").toBe( + "memory-pressure", + ); + + for (const { devices = [busy], name, options, other, proposes = true } of cases) { + const open = plan(view(devices, options)); + const held = plan(view(devices, { ...options, pressure: paused })); + if (proposes) + expect(open.proposals, `${name} is proposed when memory is not short`).not.toEqual([]); + expect(held.proposals, name).toEqual([]); + expect(open.targets[0]?.short, name).toBe(other); + if (options.targets !== undefined || options.learned !== undefined) + expect(held.targets[0]?.short, name).toBe("memory-pressure"); + } + }); + + it("sheds one idle device a reading: a device no configured target keeps first, then a kept one, least recently used first, never one a waiter would take, one the budget already shuts down or one whose shutdown just failed", () => { + const ofKind = (id: string, endedAgo: number) => + device(id, "ready", { endedAgo, spec: kind }); + const keptOldest = ofKind("kept", 90 * minute); + const extra = ofKind("extra", 40 * minute); + const olderOther = device("older", "ready", { + endedAgo: 60 * minute, + spec: spec("iPad Pro"), + }); + const newerOther = device("newer", "ready", { endedAgo: 5 * minute, spec: spec("iPad Pro") }); + const learnedOnly = device("learned", "ready", { endedAgo: 30 * minute }); + const targets = [{ count: 1, spec: kind }]; + const sheds = ( + devices: readonly VirtualDevice[], + options: Parameters[1] = {}, + pressure: NonNullable[1]>["pressure"]> = {}, + ): string[] => + evaluate( + view(devices, { targets, ...options, pressure: { step: "warming-paused", ...pressure } }), + ).flatMap((proposal) => + proposal.action === "shutdown" && proposal.reason === "memory-pressure" + ? [proposal.deviceId] + : [], + ); + + expect(sheds([keptOldest, newerOther, olderOther]), "unkept, least recently used").toEqual([ + "older", + ]); + expect(sheds([keptOldest, extra]), "a copy past the configured count").toEqual(["extra"]); + expect(sheds([keptOldest]), "a kept device when nothing else is idle").toEqual(["kept"]); + expect( + sheds([learnedOnly, keptOldest], { learned: [learnedOf(learnedOnly.spec)] }), + "a learned kind keeps nothing", + ).toEqual(["learned"]); + expect( + sheds([keptOldest, extra], { spared: targets, targets: [] }), + "a spared target's device counts as kept", + ).toEqual(["extra"]); + expect( + sheds([keptOldest, olderOther, newerOther], { + waiting: [modelDemand("iPad Pro")], + }), + "a device a waiter would take", + ).toEqual(["kept"]); + expect( + sheds([keptOldest, olderOther, newerOther], { claimed: ["older"] }), + "a claimed device", + ).toEqual(["newer"]); + expect( + sheds([keptOldest, olderOther, newerOther], {}, { shedFailed: ["older"] }), + "a device whose shutdown just failed", + ).toEqual(["newer"]); + expect(sheds([keptOldest, olderOther], {}, { mayShed: false }), "no new reading").toEqual([]); + expect(sheds([keptOldest, olderOther], {}, { step: "none" }), "step none").toEqual([]); + expect(sheds([device("busy", "leased")]), "nothing idle").toEqual([]); + + const overBudget = evaluate( + view([keptOldest, olderOther, newerOther], { + limit: 2, + pressure: { step: "warming-paused" }, + targets: [], + }), + ).flatMap((proposal) => (proposal.action === "shutdown" ? [proposal.deviceId] : [])); + expect(overBudget, "a device the budget shuts down is not shed again").toEqual([ + "kept", + "older", + ]); + }); + }); +}); diff --git a/src/core/warm-pool/policy-targeted-devices.test.ts b/src/core/warm-pool/policy-targeted-devices.test.ts new file mode 100644 index 000000000..32d704d01 --- /dev/null +++ b/src/core/warm-pool/policy-targeted-devices.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it } from "vitest"; +import type { VirtualDevice, VirtualSpec, WaitingDemand } from "../domain.js"; +import { type ResolvedTarget, targetedDevices, type WarmPolicyView } from "./policy.js"; +import { spec, device, learnedOf, leaseOn, classDemand, modelDemand, view } from "./test-policy.js"; + +// why: the pure planner needs many capacity, matching and ordering cases through its policy seam. +describe("warm pool policy", () => { + describe("targets", () => { + const kind = spec("iPhone 17"); + const target = (count: number, overrides: Partial = {}): ResolvedTarget => ({ + count, + spec: { ...kind, ...overrides }, + }); + const ofKind = (id: string, state: VirtualDevice["state"], extra = {}): VirtualDevice => + device(id, state, { spec: kind, ...extra }); + + describe("targetedDevices", () => { + const input = (devices: readonly VirtualDevice[], targets: readonly ResolvedTarget[]) => ({ + devices, + isClaimed: (id: string) => id === "claimed", + leases: devices.filter((item) => item.state === "leased").map(leaseOn), + learned: [] as WarmPolicyView["learned"], + targets, + waiting: [] as readonly WaitingDemand[], + }); + + it("names, for each target, its ready unleased devices of that kind, oldest first, up to the count", () => { + const newer = ofKind("newer", "ready", { createdAt: 20 }); + const older = ofKind("older", "ready", { createdAt: 10 }); + const oldest = ofKind("oldest", "ready", { createdAt: 5 }); + const iPad = device("ipad", "ready", { spec: spec("iPad Pro") }); + + expect( + [...targetedDevices(input([newer, older, oldest, iPad], [target(2)]))].sort(), + ).toEqual(["older", "oldest"]); + }); + + it("names a learned kind's ready unleased devices up to its count, oldest first, and the larger of a configured and a learned count", () => { + const devices = [ + ofKind("newer", "ready", { createdAt: 20 }), + ofKind("older", "ready", { createdAt: 10 }), + ofKind("oldest", "ready", { createdAt: 5 }), + device("ipad", "ready", { spec: spec("iPad Pro") }), + ]; + const named = (learnedCount: number, targets: readonly ResolvedTarget[] = []) => + [ + ...targetedDevices({ + ...input(devices, targets), + learned: [learnedOf(kind, learnedCount)], + }), + ].sort(); + + expect(named(2)).toEqual(["older", "oldest"]); + expect(named(1)).toEqual(["oldest"]); + expect(named(1, [target(2)])).toEqual(["older", "oldest"]); + expect(named(3, [target(2)])).toEqual(["newer", "older", "oldest"]); + }); + + it("leaves out a leased, claimed, shut-down or spent device and one of another mode", () => { + const devices = [ + ofKind("leased", "leased"), + ofKind("claimed", "ready"), + ofKind("shut", "shutdown"), + ofKind("spent", "ready", { endedAgo: 1, leaseIdentity: "fresh", spent: true }), + device("slim", "ready", { spec: { ...kind, mode: "slim" } }), + ofKind("good", "ready"), + ]; + + expect([...targetedDevices(input(devices, [target(5)]))]).toEqual(["good"]); + }); + + it("protects queued demand only for ready idle grantable devices matching its requirement and mode", () => { + const ready = device("ready", "ready"); + const waiting = [classDemand()]; + const cases = [ + ["matching class", view([ready], { waiting }), ["ready"]], + [ + "matching exact model", + view([ready], { waiting: [modelDemand("iPhone 15")] }), + ["ready"], + ], + ["no devices", view([], { waiting }), []], + [ + "later active waiter", + view([ready], { + waiting: [classDemand("slim"), classDemand("full", "ios", true), ...waiting], + }), + ["ready"], + ], + [ + "union with configured and learned kinds", + view( + [ + ofKind("targeted", "ready"), + device("learned", "ready", { spec: spec("iPad Pro") }), + ready, + ], + { + targets: [target(1)], + learned: [learnedOf(spec("iPad Pro"))], + waiting, + }, + ), + ["targeted", "learned", "ready"], + ], + [ + "another matching model", + view([{ ...ready, spec: spec("iPhone 17") }], { waiting }), + ["ready"], + ], + ["no demand", view([ready]), []], + ["in flight", view([ready], { waiting: [classDemand("full", "ios", true)] }), []], + ["claimed", view([ready], { claimed: [ready.id], waiting }), []], + ["leased record", view([ready], { leased: [ready], waiting }), []], + ["shutdown", view([{ ...ready, state: "shutdown" }], { waiting }), []], + [ + "spent fresh", + view([device("ready", "ready", { endedAgo: 1, leaseIdentity: "fresh", spent: true })], { + waiting, + }), + [], + ], + [ + "spent without fresh identity", + view([device("ready", "ready", { endedAgo: 1, spent: true })], { waiting }), + [], + ], + [ + "unused fresh", + view([device("ready", "ready", { leaseIdentity: "fresh" })], { waiting }), + ["ready"], + ], + ["mode", view([ready], { waiting: [classDemand("slim")] }), []], + ["platform", view([ready], { waiting: [classDemand("full", "android")] }), []], + ["model", view([ready], { waiting: [modelDemand("iPhone 17")] }), []], + ["class", view([{ ...ready, spec: spec("iPad Pro") }], { waiting }), []], + [ + "OS", + view([{ ...ready, spec: spec("iPhone 15", { osVersion: "25.0" }) }], { waiting }), + [], + ], + [ + "image", + view([ready], { + waiting: [ + { + ...classDemand(), + requirement: { ...classDemand().requirement, imageTag: "google_apis" }, + }, + ], + }), + [], + ], + ] as const; + + for (const [name, policyView, expected] of cases) { + expect([...targetedDevices(policyView)], name).toEqual(expected); + } + }); + + it("sums the counts of targets of one spec and keeps a different spec apart", () => { + const devices = [ + ofKind("a", "ready", { createdAt: 1 }), + ofKind("b", "ready", { createdAt: 2 }), + ofKind("c", "ready", { createdAt: 3 }), + device("ipad", "ready", { spec: spec("iPad Pro") }), + ]; + + expect( + [ + ...targetedDevices( + input(devices, [target(1), target(2), { count: 1, spec: spec("iPad Pro") }]), + ), + ].sort(), + ).toEqual(["a", "b", "c", "ipad"]); + }); + }); + }); +}); diff --git a/src/core/warm-pool/policy-targets.test.ts b/src/core/warm-pool/policy-targets.test.ts new file mode 100644 index 000000000..5cf7af22e --- /dev/null +++ b/src/core/warm-pool/policy-targets.test.ts @@ -0,0 +1,694 @@ +import { describe, expect, it } from "vitest"; +import type { VirtualDevice, VirtualSpec } from "../domain.js"; +import { evaluate as plan, type ResolvedTarget, targetedDevices } from "./policy.js"; +import { + evaluate, + minute, + now, + shutdownAfterMs, + spec, + device, + learnedOf, + leaseOn, + classDemand, + modelDemand, + view, +} from "./test-policy.js"; + +// why: the pure planner needs many capacity, matching and ordering cases through its policy seam. +describe("warm pool policy", () => { + describe("targets", () => { + const kind = spec("iPhone 17"); + const target = (count: number, overrides: Partial = {}): ResolvedTarget => ({ + count, + spec: { ...kind, ...overrides }, + }); + const ofKind = (id: string, state: VirtualDevice["state"], extra = {}): VirtualDevice => + device(id, state, { spec: kind, ...extra }); + + it("with a target of two, one ready device of its kind and a shut-down one, proposes one boot of the shut-down one", () => { + const ready = ofKind("ready", "ready"); + const shut = ofKind("shut", "shutdown"); + + const proposals = evaluate(view([ready, shut], { targets: [target(2)] })); + + expect(proposals).toEqual([ + { action: "boot", deviceId: "shut", reason: "target", target: kind }, + ]); + }); + + it("with a target and no device of its kind, proposes one provision with the resolved spec", () => { + const other = device("other", "ready", { endedAgo: minute, spec: spec("iPad Pro") }); + + const proposals = evaluate(view([other], { targets: [target(1)] })); + + expect(proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("proposes nothing for a target whose count of devices of its kind is ready", () => { + const first = ofKind("first", "ready"); + const second = ofKind("second", "ready"); + + expect(evaluate(view([first, second], { targets: [target(2)] }))).toEqual([]); + }); + + it("counts a boot or creation of its kind still in flight toward the target's count", () => { + expect( + evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(1)] })), + ).toEqual([]); + expect( + evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(2)] })), + ).toHaveLength(1); + }); + + it("does not boot a shut-down device that is claimed, leased-identity spent, or reset", () => { + const claimed = ofKind("claimed", "shutdown"); + const spent = ofKind("spent", "shutdown", { + endedAgo: minute, + leaseIdentity: "fresh", + spent: true, + }); + const reset = ofKind("reset", "shutdown"); + + const proposals = evaluate( + view([claimed, spent, reset], { + claimed: ["claimed"], + reset: ["reset"], + targets: [target(1)], + }), + ); + + expect(proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("does not count a leased, claimed or spent device of its kind as ready", () => { + const leased = ofKind("leased", "leased"); + const claimed = ofKind("claimed", "ready"); + const spent = ofKind("spent", "ready", { + endedAgo: minute, + leaseIdentity: "fresh", + spent: true, + }); + + const proposals = evaluate( + view([leased, claimed, spent], { claimed: ["claimed"], limit: 10, targets: [target(1)] }), + ); + + expect(proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("with maxConcurrentBoots 1 and one boot in flight, proposes no second boot or provision", () => { + const shut = ofKind("shut", "shutdown"); + + expect( + evaluate(view([shut], { inFlight: [spec("iPad Pro")], targets: [target(1)] })), + ).toEqual([]); + expect(evaluate(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] }))).toEqual( + [], + ); + expect(evaluate(view([shut], { targets: [target(1)] }))).toHaveLength(1); + }); + + it("with maxConcurrentBoots 2 proposes two of a target of three and counts a waiter's boot against the cap", () => { + const one = ofKind("one", "shutdown"); + const two = ofKind("two", "shutdown"); + const three = ofKind("three", "shutdown"); + + const proposals = evaluate( + view([one, two, three], { maxConcurrentBoots: 2, targets: [target(3)] }), + ); + expect(proposals).toHaveLength(2); + expect(proposals.every((proposal) => proposal.reason === "target")).toBe(true); + + const tablet = device("tablet", "shutdown", { endedAgo: minute, spec: spec("iPad Pro") }); + const withWaiter = evaluate( + view([tablet, one, two], { + maxConcurrentBoots: 2, + targets: [target(2)], + waiting: [modelDemand("iPad Pro")], + }), + ); + expect(withWaiter.map((proposal) => proposal.reason)).toEqual(["waiting-request", "target"]); + }); + + it("proposes nothing while a request is queued on its platform, and proposes while that request is already booting its own device", () => { + const shut = ofKind("shut", "shutdown"); + const queued = modelDemand("iPad Pro"); + + expect(evaluate(view([shut], { targets: [target(1)], waiting: [queued] }))).toEqual([]); + expect( + evaluate( + view([shut], { + maxConcurrentBoots: 2, + targets: [target(1)], + waiting: [{ ...queued, inFlight: true }], + }), + ), + ).toEqual([{ action: "boot", deviceId: "shut", reason: "target", target: kind }]); + expect( + evaluate( + view([shut], { + maxConcurrentBoots: 2, + targets: [target(1)], + waiting: [classDemand("full", "android")], + }), + ), + ).toHaveLength(1); + }); + + it("fills a target after the queue's devices within one cap, and keeps it waiting, not short, only behind a waiter the pass could not serve", () => { + const shut = ofKind("shut", "shutdown"); + const phone = device("phone", "shutdown", { endedAgo: 20 * minute, spec: spec("iPhone 15") }); + const head = modelDemand("iPad Pro"); + const filling = { targets: [target(1)], waiting: [head] }; + + expect(plan(view([shut], { ...filling, maxConcurrentBoots: 2 })).proposals).toEqual([ + { action: "boot", deviceId: "shut", reason: "target", target: kind }, + ]); + const capped = plan(view([shut], { ...filling, maxConcurrentBoots: 1 })); + expect(capped.proposals).toEqual([]); + expect(capped.targets[0]).not.toHaveProperty("short"); + + expect( + evaluate( + view([phone, shut], { + maxConcurrentBoots: 2, + targets: [target(1)], + waiting: [classDemand()], + }), + ), + ).toEqual([ + { action: "boot", deviceId: "phone", reason: "waiting-request" }, + { action: "boot", deviceId: "shut", reason: "target", target: kind }, + ]); + + const unservable = plan( + view([shut], { ...filling, maxConcurrentBoots: 3, refuseProvision: "device-limit" }), + ); + expect(unservable.proposals).toEqual([]); + expect(unservable.targets[0]).not.toHaveProperty("short"); + }); + + it("proposes a shutdown for a never-leased ready device no target counts once it has been ready longer than idle.shutdownAfterMs", () => { + const stale = device("stale", "ready", { readyAt: now - shutdownAfterMs - 1 }); + const boundary = device("boundary", "ready", { readyAt: now - shutdownAfterMs }); + const fresh = device("fresh", "ready", { readyAt: now - 1_000 }); + const leasedBefore = device("leased-before", "ready", { + endedAgo: shutdownAfterMs * 3, + spent: true, + readyAt: now - shutdownAfterMs * 3, + }); + + const proposals = evaluate(view([stale, boundary, fresh, leasedBefore])); + + expect(proposals).toEqual([ + { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, + ]); + }); + + it("reads a record with no readyAt as ready since createdAt", () => { + const old = device("old", "ready", { createdAt: now - shutdownAfterMs - 1 }); + const young = device("young", "ready", { createdAt: now - 1_000 }); + + expect(evaluate(view([old, young]))).toEqual([ + { action: "shutdown", deviceId: "old", reason: "never-leased-idle" }, + ]); + }); + + it("does not propose a shutdown for a never-leased device a target counts, nor one beyond the count's reach that a request serves", () => { + const counted = ofKind("counted", "ready", { readyAt: 1 }); + const extra = ofKind("extra", "ready", { readyAt: 2, createdAt: 2 }); + + // A target of one counts the older device only; the other one is not targeted and goes. + expect(evaluate(view([counted, extra], { targets: [target(1)] }))).toEqual([ + { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, + ]); + // A queued request the extra device serves keeps it: it is about to be granted. + expect( + evaluate( + view([counted, extra], { targets: [target(1)], waiting: [modelDemand("iPhone 17")] }), + ), + ).toEqual([]); + }); + + it("never proposes a never-leased shutdown for a leased, claimed or booting device", () => { + const leased = device("leased", "leased", { readyAt: 1 }); + const claimed = device("claimed", "ready", { readyAt: 1 }); + const booting = device("booting", "provisioning", { readyAt: 1 }); + + expect(evaluate(view([leased, claimed, booting], { claimed: ["claimed"] }))).toEqual([]); + }); + + it("proposes nothing for a target when the running limit is full, and reports running-limit", () => { + const busy = device("busy", "leased", { spec: spec("iPad Pro") }); + + const result = plan(view([busy], { limit: 1, targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets).toEqual([ + { + booting: 0, + configuredCount: 1, + count: 1, + kind: { + mode: "full", + model: "iPhone 17", + osVersion: "26.0", + platform: "ios", + }, + origin: "configured", + ready: 0, + short: "running-limit", + spec: kind, + target: "ios iPhone 17 26.0", + }, + ]); + }); + + it("proposes nothing for a target when the reserve holds the last slot, and reports reserve", () => { + const result = plan( + view([], { limits: { ios: 1 }, reserve: { ios: 1 }, targets: [target(1)] }), + ); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: "reserve", count: 1, ready: 0 }); + }); + + it.each(["device-limit", "ram-budget"] as const)( + "proposes no creation for a target when the capacity strategy refuses it for %s, and reports it", + (reason) => { + const result = plan(view([], { refuseProvision: reason, targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: reason }); + }, + ); + + it("proposes no boot for a target when the capacity strategy refuses the device's RAM, and reports ram-budget", () => { + const shut = ofKind("shut", "shutdown"); + + const result = plan(view([shut], { refuseBoot: "ram-budget", targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: "ram-budget" }); + }); + + it("reports no short for a target that is filled, and its ready count", () => { + const ready = ofKind("ready", "ready"); + + const result = plan(view([ready], { targets: [target(1)] })); + + expect(result.targets).toStrictEqual([ + { + booting: 0, + configuredCount: 1, + count: 1, + kind: { + mode: "full", + model: "iPhone 17", + osVersion: "26.0", + platform: "ios", + }, + origin: "configured", + ready: 1, + spec: kind, + target: "ios iPhone 17 26.0", + }, + ]); + }); + + it("reports a target of two with one ready and one boot running as booting 1, and not short", () => { + const ready = ofKind("ready", "ready"); + + const result = plan(view([ready], { inFlight: [kind], targets: [target(2)] })); + + expect(result.targets[0]).toMatchObject({ booting: 1, count: 2, ready: 1 }); + expect(result.targets[0]).not.toHaveProperty("short"); + }); + + it("counts a creation it proposes in this pass as booting, and leaves another kind's out", () => { + const result = plan(view([], { targets: [target(1), target(1, { model: "iPad Pro" })] })); + + expect(result.proposals).toHaveLength(1); + expect(result.targets.map((report) => report.booting)).toEqual([1, 0]); + }); + + it("reports a target held by the boot cap, or behind a waiter the pass could not serve, with no reason", () => { + const capped = plan(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] })); + expect(capped.targets[0]).toMatchObject({ booting: 0, count: 1, ready: 0 }); + expect(capped.targets[0]).not.toHaveProperty("short"); + + const queued = plan( + view([], { + refuseProvision: "device-limit", + targets: [target(1)], + waiting: [classDemand()], + }), + ); + expect(queued.targets[0]).not.toHaveProperty("short"); + }); + + it("shuts down an idle device no configured count keeps to make room for a target, and never one a configured count keeps", () => { + const idle = device("idle", "ready", { endedAgo: minute, spec: spec("iPad Pro") }); + const make = { action: "provision", reason: "target", spec: kind, target: kind }; + + const result = plan(view([idle], { limit: 1, targets: [target(1)] })); + + expect(result.proposals).toEqual([ + { action: "shutdown", deviceId: "idle", reason: "displaced" }, + make, + ]); + expect(result.targets[0]).toMatchObject({ booting: 1, ready: 0 }); + const pad = { count: 1, spec: spec("iPad Pro") }; + expect(evaluate(view([idle], { limit: 1, targets: [target(1), pad] }))).toEqual([]); + expect(evaluate(view([idle], { limit: 1, targets: [pad, target(1)] }))).toEqual([]); + }); + + it("shuts down a copy past another configured kind's count, for a slot the reserve holds too, and fills every configured count before a learned peak", () => { + const pad = spec("iPad Pro"); + const pads = ["a", "b"].map((id, index) => + device(id, "ready", { endedAgo: (index + 1) * minute, spec: pad }), + ); + const both = [{ count: 1, spec: pad }, target(1)]; + const make = { action: "provision", reason: "target", spec: kind, target: kind }; + + expect(evaluate(view(pads, { limit: 2, targets: both }))).toEqual([ + { action: "shutdown", deviceId: "b", reason: "displaced" }, + make, + ]); + expect( + evaluate(view(pads, { limits: { ios: 3 }, reserve: { ios: 1 }, targets: both })), + ).toEqual([{ action: "shutdown", deviceId: "b", reason: "displaced" }, make]); + expect( + evaluate(view([pads[0]!], { learned: [learnedOf(pad, 2, now)], limit: 2, targets: both })), + ).toEqual([make]); + }); + + it("counts two targets resolving to one spec as one with the summed count", () => { + const ready = ofKind("ready", "ready"); + + const proposals = evaluate( + view([ready], { maxConcurrentBoots: 5, targets: [target(1), target(2)] }), + ); + + expect(proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + expect(plan(view([ready], { targets: [target(1), target(2)] })).targets).toHaveLength(1); + }); + + it("does not count a slim device toward a full target of the same model", () => { + const slim = device("slim", "ready", { endedAgo: minute, spec: { ...kind, mode: "slim" } }); + + const proposals = evaluate(view([slim], { targets: [target(1)] })); + + expect(proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("proposes nothing for a target whose spec may not be attempted yet, and reports boot-failed", () => { + const shut = ofKind("shut", "shutdown"); + + const result = plan(view([shut], { blocked: [kind], targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: "boot-failed", count: 1, ready: 0 }); + }); + + it("proposes nothing for a target and reports none with warmPool disabled", () => { + const result = plan(view([], { enabled: false, targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets).toEqual([]); + }); + + it("does not boot or create for an android target out of an iOS slot", () => { + const androidKind = spec("Pixel 9", { osVersion: "36", platform: "android" }); + const iosBusy = device("busy", "leased"); + + const result = plan( + view([iosBusy], { + limit: 5, + limits: { android: 0 }, + targets: [{ count: 1, spec: androidKind }], + }), + ); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: "running-limit" }); + }); + + it("is not short while held back after a failure when another boot is running, or a request is queued", () => { + const shut = ofKind("shut", "shutdown"); + const held = { blocked: [kind], maxConcurrentBoots: 2, targets: [target(2)] }; + + expect(plan(view([shut], held)).targets[0]).toMatchObject({ short: "boot-failed" }); + expect(plan(view([shut], { ...held, inFlight: [kind] })).targets[0]).not.toHaveProperty( + "short", + ); + expect( + plan( + view([shut], { + ...held, + refuseProvision: "device-limit", + waiting: [modelDemand("iPad Pro")], + }), + ).targets[0], + ).not.toHaveProperty("short"); + }); + + it("spares the never-leased devices of a spared target and plans nothing for it", () => { + const stale = device("stale", "ready", { readyAt: 1, spec: kind }); + + const result = plan(view([stale], { spared: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets).toEqual([]); + expect(evaluate(view([stale]))).toEqual([ + { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, + ]); + }); + + it("reports no short for a filled target whose spec may not be attempted yet", () => { + const ready = ofKind("ready", "ready"); + + const result = plan(view([ready], { blocked: [kind], targets: [target(1)] })); + + expect(result.targets).toStrictEqual([ + { + booting: 0, + configuredCount: 1, + count: 1, + kind: { + mode: "full", + model: "iPhone 17", + osVersion: "26.0", + platform: "ios", + }, + origin: "configured", + ready: 1, + spec: kind, + target: "ios iPhone 17 26.0", + }, + ]); + }); + + it("proposes a boot of each of two shut-down devices once, and a creation for the target's third", () => { + const one = ofKind("one", "shutdown"); + const two = ofKind("two", "shutdown"); + + const proposals = evaluate(view([one, two], { maxConcurrentBoots: 3, targets: [target(3)] })); + + expect(proposals).toEqual([ + { action: "boot", deviceId: "one", reason: "target", target: kind }, + { action: "boot", deviceId: "two", reason: "target", target: kind }, + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("takes a slot for each proposal, so a target of two with room for one proposes one and is not short while it fills", () => { + const result = plan(view([], { limit: 1, maxConcurrentBoots: 2, targets: [target(2)] })); + + expect(result.proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + expect(result.targets[0]).toStrictEqual({ + booting: 1, + configuredCount: 2, + count: 2, + kind: { + mode: "full", + model: "iPhone 17", + osVersion: "26.0", + platform: "ios", + }, + origin: "configured", + ready: 0, + spec: kind, + target: "ios iPhone 17 26.0", + }); + }); + + it("is short for a reason only when a pass could do nothing for it: not with a boot already running", () => { + const busy = device("busy", "leased", { spec: spec("iPad Pro") }); + const full = { limit: 1, maxConcurrentBoots: 2, targets: [target(2)] }; + + expect(plan(view([busy], full)).targets[0]).toMatchObject({ short: "running-limit" }); + expect(plan(view([busy], { ...full, inFlight: [kind] })).targets[0]).not.toHaveProperty( + "short", + ); + }); + + it("reports device-limit before running-limit and reserve, and those before ram-budget", () => { + const both = plan( + view([], { + limit: 1, + refuseProvision: "device-limit", + targets: [target(1)], + reserve: { ios: 1 }, + }), + ); + expect(both.targets[0]).toMatchObject({ short: "device-limit" }); + const full = plan( + view([], { limit: 0, refuseProvision: "ram-budget", targets: [target(1)] }), + ); + expect(full.targets[0]).toMatchObject({ short: "running-limit" }); + const bothLimits = plan( + view([], { limit: 0, refuseProvision: "device-limit", targets: [target(1)] }), + ); + expect(bothLimits.targets[0]).toMatchObject({ short: "device-limit" }); + const reserved = plan( + view([], { + limits: { ios: 1 }, + refuseProvision: "ram-budget", + reserve: { ios: 1 }, + targets: [target(1)], + }), + ); + expect(reserved.targets[0]).toMatchObject({ short: "reserve" }); + }); + + it.each([ + ["the machine", { limit: 1, limits: { ios: 5 } }], + ["the platform", { limit: 5, limits: { ios: 1 } }], + ])( + "stops proposing for a target of two once the slot is taken when %s alone has room for one, and is not short", + (_name, limits) => { + const result = plan(view([], { ...limits, maxConcurrentBoots: 2, targets: [target(2)] })); + + expect(result.proposals).toEqual([ + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + expect(result.targets[0]).not.toHaveProperty("short"); + }, + ); + + it("does not count a boot in flight for another kind toward the target's count", () => { + const result = evaluate( + view([], { inFlight: [spec("iPad Pro")], maxConcurrentBoots: 2, targets: [target(1)] }), + ); + + expect(result).toEqual([{ action: "provision", reason: "target", spec: kind, target: kind }]); + }); + + it("proposes a creation with exactly one running slot free, and none with none free on either the platform or the machine", () => { + const busyAndroid = device("busy", "leased", { + spec: spec("Pixel 9", { osVersion: "36", platform: "android" }), + }); + + expect(evaluate(view([], { limit: 1, targets: [target(1)] }))).toHaveLength(1); + const global = plan( + view([busyAndroid], { limit: 1, limits: { ios: 5 }, targets: [target(1)] }), + ); + expect(global.proposals).toEqual([]); + expect(global.targets[0]).toMatchObject({ short: "running-limit" }); + const platform = plan(view([], { limit: 5, limits: { ios: 0 }, targets: [target(1)] })); + expect(platform.proposals).toEqual([]); + expect(platform.targets[0]).toMatchObject({ short: "running-limit" }); + }); + + it("keeps the reserve out of a target's reach on the machine and on the platform, and uses the last slot beyond it", () => { + const reserve = { android: 1, ios: 1 }; + + const machine = plan(view([], { limit: 2, reserve, targets: [target(1)] })); + expect(machine.proposals).toEqual([]); + expect(machine.targets[0]).toMatchObject({ short: "reserve" }); + expect(evaluate(view([], { limit: 3, reserve, targets: [target(1)] }))).toHaveLength(1); + + const platform = plan( + view([], { limits: { ios: 1 }, reserve: { ios: 1 }, targets: [target(1)] }), + ); + expect(platform.targets[0]).toMatchObject({ short: "reserve" }); + expect( + evaluate(view([], { limits: { ios: 2 }, reserve: { ios: 1 }, targets: [target(1)] })), + ).toHaveLength(1); + }); + + it.each(["global-running-limit", "platform-running-limit"] as const)( + "reports running-limit when the capacity strategy refuses a creation for %s", + (reason) => { + const result = plan(view([], { refuseProvision: reason, targets: [target(1)] })); + + expect(result.proposals).toEqual([]); + expect(result.targets[0]).toMatchObject({ short: "running-limit" }); + }, + ); + + it("frees the slot of a never-leased device it shuts down for a target of another kind in the same pass", () => { + const stale = device("stale", "ready", { readyAt: 1, spec: spec("iPad Pro") }); + + const proposals = evaluate(view([stale], { limit: 1, targets: [target(1)] })); + + expect(proposals).toEqual([ + { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, + { action: "provision", reason: "target", spec: kind, target: kind }, + ]); + }); + + it("proposes a never-leased device once when the budget already proposes it", () => { + const first = device("first", "ready", { readyAt: 1 }); + const second = device("second", "ready", { readyAt: 1 }); + + const proposals = evaluate(view([first, second], { limit: 1 })); + + expect(proposals).toEqual([ + { action: "shutdown", deviceId: "first", reason: "over-budget" }, + { action: "shutdown", deviceId: "second", reason: "never-leased-idle" }, + ]); + }); + + it("spares a never-leased device only for a queued request it serves, not one already booting or one it does not serve", () => { + const stale = device("stale", "ready", { readyAt: 1 }); + const shutdown = { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }; + + expect(evaluate(view([stale], { waiting: [modelDemand("iPhone 15")] }))).toEqual([]); + expect( + evaluate(view([stale], { waiting: [{ ...modelDemand("iPhone 15"), inFlight: true }] })), + ).toEqual([shutdown]); + expect(evaluate(view([stale], { waiting: [modelDemand("iPad Pro")] }))).toEqual([shutdown]); + }); + + it("does not name a device with a lease record as targeted even when its state still reads ready", () => { + const leasedReady = ofKind("leased-ready", "ready"); + + const named = targetedDevices({ + devices: [leasedReady], + isClaimed: () => false, + leases: [leaseOn(leasedReady)], + learned: [], + targets: [target(1)], + waiting: [], + }); + + expect([...named]).toEqual([]); + }); + }); +}); diff --git a/src/core/warm-pool/policy.test.ts b/src/core/warm-pool/policy.test.ts index 53d6c3f63..b0b8cd6b7 100644 --- a/src/core/warm-pool/policy.test.ts +++ b/src/core/warm-pool/policy.test.ts @@ -1,238 +1,18 @@ import { describe, expect, it } from "vitest"; - -import type { CapacityRefusalReason, RunningCapacity } from "../capacity/index.js"; -import type { - DeviceClass, - VirtualDevice, - VirtualSpec, - LeaseRecord, - WaitingDemand, -} from "../domain.js"; -import { sameSpec } from "../domain.js"; +import type { VirtualDevice, WaitingDemand } from "../domain.js"; +import { evaluate as plan, type WarmPolicyView } from "./policy.js"; import { - evaluate as plan, - type ResolvedTarget, - targetedDevices, - type WarmPolicyView, -} from "./policy.js"; - -const evaluate = (policyView: WarmPolicyView) => plan(policyView).proposals; - -const minute = 60_000; -const now = 1_000 * minute; -const shutdownAfterMs = 10 * minute; - -const phones = new Set(["iPhone 15", "iPhone 17"]); -const classOf = (model: string): DeviceClass | undefined => - phones.has(model) ? "phone" : "tablet"; - -function spec(model: string, overrides: Partial = {}): VirtualSpec { - return { model, osVersion: "26.0", physical: false, platform: "ios", ...overrides }; -} - -interface DeviceOptions { - spec?: VirtualSpec; - endedAgo?: number; - createdAt?: number; - readyAt?: number; - leaseIdentity?: "fresh"; - spent?: boolean; -} - -function readyStamp(options: DeviceOptions): Pick | Record { - return options.readyAt === undefined && options.endedAgo === undefined - ? {} - : { readyAt: options.readyAt ?? now }; -} - -// why: a virtual device that ended a lease is spent (ADR 0061 §2) and never grantable; `endedAgo` -// only orders idle devices (least recently used), so it moves createdAt, and only a spent device -// carries the end stamp. Such a device counts as freshly ready, so the never-leased idle rule leaves it be. -function endStamp( - options: DeviceOptions, -): Pick | Record { - return options.spent === true && options.endedAgo !== undefined - ? { lastLeaseEndedAt: now - options.endedAgo } - : {}; -} - -function device( - id: string, - state: VirtualDevice["state"], - options: DeviceOptions = {}, -): VirtualDevice { - return { - physical: false, - createdAt: options.createdAt ?? (options.endedAgo === undefined ? 1 : now - options.endedAgo), - ...readyStamp(options), - ...(options.leaseIdentity === undefined ? {} : { leaseIdentity: options.leaseIdentity }), - driverData: {}, - driverDeviceId: `driver-${id}`, - id, - mode: "full", - spec: options.spec ?? spec("iPhone 15"), - state, - ...endStamp(options), - }; -} - -function learnedOf( - learnedSpec: VirtualSpec, - count = 1, - lastLeasedAt = now, -): WarmPolicyView["learned"][number] { - return { count, lastLeasedAt, spec: learnedSpec }; -} - -function leaseOn(target: VirtualDevice): LeaseRecord { - return { - deviceId: target.id, - grantedAt: 1, - id: `lease-${target.id}`, - lastRenewedAt: 1, - idChosenByRequester: false, - ownerId: "holder", - requesterId: "holder", - ttlDeadline: now + minute, - ttlMs: minute, - }; -} - -function capacityOf( - devices: readonly VirtualDevice[], - limits: { global: number; ios?: number; android?: number }, - reserved = 0, -): RunningCapacity { - const running = (platform?: string): number => - devices.filter( - (item) => - ["ready", "leased", "reclaiming", "quarantined"].includes(item.state) && - (platform === undefined || item.spec.platform === platform), - ).length; - const entry = (maxRunning: number, platform?: string) => ({ - maxRunning, - overLimit: running(platform) + reserved > maxRunning, - reserved, - running: running(platform), - }); - return { - android: entry(limits.android ?? limits.global, "android"), - global: entry(limits.global), - ios: entry(limits.ios ?? limits.global, "ios"), - }; -} - -function classDemand( - mode: "slim" | "full" = "full", - platform: "ios" | "android" = "ios", - inFlight = false, - model = "iPhone 15", -) { - return { - classOf, - inFlight, - mode, - platform, - spec: spec(model, { platform, ...(mode === "slim" ? { mode } : {}) }), - requirement: { - imageTag: undefined, - osVersion: { kind: "installed", versions: ["26.0"] }, - platform, - target: { class: "phone", kind: "class" }, - }, - } satisfies WaitingDemand; -} - -function modelDemand(model: string): WaitingDemand { - return { - classOf, - inFlight: false, - mode: "full", - platform: "ios", - spec: spec(model), - requirement: { - imageTag: undefined, - osVersion: { kind: "exact", version: "26.0" }, - platform: "ios", - target: { kind: "model", model }, - }, - }; -} - -// fallow-ignore-next-line complexity -- one test fixture builder; each option is a plain pass-through to a view field. -function view( - devices: readonly VirtualDevice[], - options: { - limit?: number; - limits?: { ios?: number; android?: number }; - reserve?: { ios?: number; android?: number }; - reserved?: number; - waiting?: readonly WaitingDemand[]; - enabled?: boolean; - claimed?: readonly string[]; - leased?: readonly VirtualDevice[]; - handoffInFlight?: boolean; - reset?: readonly string[]; - targets?: readonly ResolvedTarget[]; - spared?: readonly ResolvedTarget[]; - learned?: WarmPolicyView["learned"]; - inFlight?: readonly VirtualSpec[]; - onItsWay?: WarmPolicyView["onItsWay"]; - maxConcurrentBoots?: number; - blocked?: readonly VirtualSpec[]; - refuseProvision?: CapacityRefusalReason; - refuseBoot?: CapacityRefusalReason; - /** why: a creation the capacity strategy refuses until the plan has shut a device down. */ - refuseProvisionUntilDown?: CapacityRefusalReason; - unplanned?: readonly VirtualSpec[]; - pressure?: { - step?: WarmPolicyView["pressure"]["step"]; - mayShed?: boolean; - shedFailed?: readonly string[]; - }; - } = {}, -): WarmPolicyView { - return { - admit: { - boot: () => options.refuseBoot, - create: (_spec, down) => - options.refuseProvision ?? - ((down?.size ?? 0) === 0 ? options.refuseProvisionUntilDown : undefined), - }, - unplanned: options.unplanned ?? [], - inFlight: options.inFlight ?? [], - onItsWay: options.onItsWay ?? [], - retry: { - mayAttempt: (target) => !(options.blocked ?? []).some((held) => sameSpec(held, target)), - }, - spared: options.spared ?? [], - learned: options.learned ?? [], - targets: options.targets ?? [], - capacity: capacityOf( - devices, - { global: options.limit ?? 10, ...options.limits }, - options.reserved, - ), - config: { - enabled: options.enabled ?? true, - maxConcurrentBoots: options.maxConcurrentBoots ?? 1, - reserveRunning: { android: 0, ios: 0, ...options.reserve }, - shutdownAfterMs, - }, - devices, - handoffInFlight: options.handoffInFlight ?? false, - resetDevices: new Set(options.reset ?? []), - pressure: { - mayShed: options.pressure?.mayShed ?? true, - shedFailed: new Set(options.pressure?.shedFailed ?? []), - step: options.pressure?.step ?? "none", - }, - isClaimed: (id) => options.claimed?.includes(id) ?? false, - leases: (options.leased ?? devices.filter((item) => item.state === "leased")).map(leaseOn), - now, - waiting: options.waiting ?? [], - }; -} + evaluate, + minute, + now, + spec, + device, + learnedOf, + capacityOf, + classDemand, + modelDemand, + view, +} from "./test-policy.js"; // why: the pure planner needs many capacity, matching and ordering cases through its policy seam. describe("warm pool policy", () => { @@ -904,1248 +684,4 @@ describe("warm pool policy", () => { ).toEqual([{ action: "boot", deviceId: "released", reason: "waiting-request" }]); }); }); - describe("targets", () => { - const kind = spec("iPhone 17"); - const target = (count: number, overrides: Partial = {}): ResolvedTarget => ({ - count, - spec: { ...kind, ...overrides }, - }); - const ofKind = (id: string, state: VirtualDevice["state"], extra = {}): VirtualDevice => - device(id, state, { spec: kind, ...extra }); - - it("with a target of two, one ready device of its kind and a shut-down one, proposes one boot of the shut-down one", () => { - const ready = ofKind("ready", "ready"); - const shut = ofKind("shut", "shutdown"); - - const proposals = evaluate(view([ready, shut], { targets: [target(2)] })); - - expect(proposals).toEqual([ - { action: "boot", deviceId: "shut", reason: "target", target: kind }, - ]); - }); - - it("with a target and no device of its kind, proposes one provision with the resolved spec", () => { - const other = device("other", "ready", { endedAgo: minute, spec: spec("iPad Pro") }); - - const proposals = evaluate(view([other], { targets: [target(1)] })); - - expect(proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("proposes nothing for a target whose count of devices of its kind is ready", () => { - const first = ofKind("first", "ready"); - const second = ofKind("second", "ready"); - - expect(evaluate(view([first, second], { targets: [target(2)] }))).toEqual([]); - }); - - it("counts a boot or creation of its kind still in flight toward the target's count", () => { - expect( - evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(1)] })), - ).toEqual([]); - expect( - evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(2)] })), - ).toHaveLength(1); - }); - - it("does not boot a shut-down device that is claimed, leased-identity spent, or reset", () => { - const claimed = ofKind("claimed", "shutdown"); - const spent = ofKind("spent", "shutdown", { - endedAgo: minute, - leaseIdentity: "fresh", - spent: true, - }); - const reset = ofKind("reset", "shutdown"); - - const proposals = evaluate( - view([claimed, spent, reset], { - claimed: ["claimed"], - reset: ["reset"], - targets: [target(1)], - }), - ); - - expect(proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("does not count a leased, claimed or spent device of its kind as ready", () => { - const leased = ofKind("leased", "leased"); - const claimed = ofKind("claimed", "ready"); - const spent = ofKind("spent", "ready", { - endedAgo: minute, - leaseIdentity: "fresh", - spent: true, - }); - - const proposals = evaluate( - view([leased, claimed, spent], { claimed: ["claimed"], limit: 10, targets: [target(1)] }), - ); - - expect(proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("with maxConcurrentBoots 1 and one boot in flight, proposes no second boot or provision", () => { - const shut = ofKind("shut", "shutdown"); - - expect( - evaluate(view([shut], { inFlight: [spec("iPad Pro")], targets: [target(1)] })), - ).toEqual([]); - expect(evaluate(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] }))).toEqual( - [], - ); - expect(evaluate(view([shut], { targets: [target(1)] }))).toHaveLength(1); - }); - - it("with maxConcurrentBoots 2 proposes two of a target of three and counts a waiter's boot against the cap", () => { - const one = ofKind("one", "shutdown"); - const two = ofKind("two", "shutdown"); - const three = ofKind("three", "shutdown"); - - const proposals = evaluate( - view([one, two, three], { maxConcurrentBoots: 2, targets: [target(3)] }), - ); - expect(proposals).toHaveLength(2); - expect(proposals.every((proposal) => proposal.reason === "target")).toBe(true); - - const tablet = device("tablet", "shutdown", { endedAgo: minute, spec: spec("iPad Pro") }); - const withWaiter = evaluate( - view([tablet, one, two], { - maxConcurrentBoots: 2, - targets: [target(2)], - waiting: [modelDemand("iPad Pro")], - }), - ); - expect(withWaiter.map((proposal) => proposal.reason)).toEqual(["waiting-request", "target"]); - }); - - it("proposes nothing while a request is queued on its platform, and proposes while that request is already booting its own device", () => { - const shut = ofKind("shut", "shutdown"); - const queued = modelDemand("iPad Pro"); - - expect(evaluate(view([shut], { targets: [target(1)], waiting: [queued] }))).toEqual([]); - expect( - evaluate( - view([shut], { - maxConcurrentBoots: 2, - targets: [target(1)], - waiting: [{ ...queued, inFlight: true }], - }), - ), - ).toEqual([{ action: "boot", deviceId: "shut", reason: "target", target: kind }]); - expect( - evaluate( - view([shut], { - maxConcurrentBoots: 2, - targets: [target(1)], - waiting: [classDemand("full", "android")], - }), - ), - ).toHaveLength(1); - }); - - it("fills a target after the queue's devices within one cap, and keeps it waiting, not short, only behind a waiter the pass could not serve", () => { - const shut = ofKind("shut", "shutdown"); - const phone = device("phone", "shutdown", { endedAgo: 20 * minute, spec: spec("iPhone 15") }); - const head = modelDemand("iPad Pro"); - const filling = { targets: [target(1)], waiting: [head] }; - - expect(plan(view([shut], { ...filling, maxConcurrentBoots: 2 })).proposals).toEqual([ - { action: "boot", deviceId: "shut", reason: "target", target: kind }, - ]); - const capped = plan(view([shut], { ...filling, maxConcurrentBoots: 1 })); - expect(capped.proposals).toEqual([]); - expect(capped.targets[0]).not.toHaveProperty("short"); - - expect( - evaluate( - view([phone, shut], { - maxConcurrentBoots: 2, - targets: [target(1)], - waiting: [classDemand()], - }), - ), - ).toEqual([ - { action: "boot", deviceId: "phone", reason: "waiting-request" }, - { action: "boot", deviceId: "shut", reason: "target", target: kind }, - ]); - - const unservable = plan( - view([shut], { ...filling, maxConcurrentBoots: 3, refuseProvision: "device-limit" }), - ); - expect(unservable.proposals).toEqual([]); - expect(unservable.targets[0]).not.toHaveProperty("short"); - }); - - it("proposes a shutdown for a never-leased ready device no target counts once it has been ready longer than idle.shutdownAfterMs", () => { - const stale = device("stale", "ready", { readyAt: now - shutdownAfterMs - 1 }); - const boundary = device("boundary", "ready", { readyAt: now - shutdownAfterMs }); - const fresh = device("fresh", "ready", { readyAt: now - 1_000 }); - const leasedBefore = device("leased-before", "ready", { - endedAgo: shutdownAfterMs * 3, - spent: true, - readyAt: now - shutdownAfterMs * 3, - }); - - const proposals = evaluate(view([stale, boundary, fresh, leasedBefore])); - - expect(proposals).toEqual([ - { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, - ]); - }); - - it("reads a record with no readyAt as ready since createdAt", () => { - const old = device("old", "ready", { createdAt: now - shutdownAfterMs - 1 }); - const young = device("young", "ready", { createdAt: now - 1_000 }); - - expect(evaluate(view([old, young]))).toEqual([ - { action: "shutdown", deviceId: "old", reason: "never-leased-idle" }, - ]); - }); - - it("does not propose a shutdown for a never-leased device a target counts, nor one beyond the count's reach that a request serves", () => { - const counted = ofKind("counted", "ready", { readyAt: 1 }); - const extra = ofKind("extra", "ready", { readyAt: 2, createdAt: 2 }); - - // A target of one counts the older device only; the other one is not targeted and goes. - expect(evaluate(view([counted, extra], { targets: [target(1)] }))).toEqual([ - { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, - ]); - // A queued request the extra device serves keeps it: it is about to be granted. - expect( - evaluate( - view([counted, extra], { targets: [target(1)], waiting: [modelDemand("iPhone 17")] }), - ), - ).toEqual([]); - }); - - it("never proposes a never-leased shutdown for a leased, claimed or booting device", () => { - const leased = device("leased", "leased", { readyAt: 1 }); - const claimed = device("claimed", "ready", { readyAt: 1 }); - const booting = device("booting", "provisioning", { readyAt: 1 }); - - expect(evaluate(view([leased, claimed, booting], { claimed: ["claimed"] }))).toEqual([]); - }); - - it("proposes nothing for a target when the running limit is full, and reports running-limit", () => { - const busy = device("busy", "leased", { spec: spec("iPad Pro") }); - - const result = plan(view([busy], { limit: 1, targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets).toEqual([ - { - booting: 0, - configuredCount: 1, - count: 1, - kind: { - mode: "full", - model: "iPhone 17", - osVersion: "26.0", - platform: "ios", - }, - origin: "configured", - ready: 0, - short: "running-limit", - spec: kind, - target: "ios iPhone 17 26.0", - }, - ]); - }); - - it("proposes nothing for a target when the reserve holds the last slot, and reports reserve", () => { - const result = plan( - view([], { limits: { ios: 1 }, reserve: { ios: 1 }, targets: [target(1)] }), - ); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: "reserve", count: 1, ready: 0 }); - }); - - it.each(["device-limit", "ram-budget"] as const)( - "proposes no creation for a target when the capacity strategy refuses it for %s, and reports it", - (reason) => { - const result = plan(view([], { refuseProvision: reason, targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: reason }); - }, - ); - - it("proposes no boot for a target when the capacity strategy refuses the device's RAM, and reports ram-budget", () => { - const shut = ofKind("shut", "shutdown"); - - const result = plan(view([shut], { refuseBoot: "ram-budget", targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: "ram-budget" }); - }); - - it("reports no short for a target that is filled, and its ready count", () => { - const ready = ofKind("ready", "ready"); - - const result = plan(view([ready], { targets: [target(1)] })); - - expect(result.targets).toStrictEqual([ - { - booting: 0, - configuredCount: 1, - count: 1, - kind: { - mode: "full", - model: "iPhone 17", - osVersion: "26.0", - platform: "ios", - }, - origin: "configured", - ready: 1, - spec: kind, - target: "ios iPhone 17 26.0", - }, - ]); - }); - - it("reports a target of two with one ready and one boot running as booting 1, and not short", () => { - const ready = ofKind("ready", "ready"); - - const result = plan(view([ready], { inFlight: [kind], targets: [target(2)] })); - - expect(result.targets[0]).toMatchObject({ booting: 1, count: 2, ready: 1 }); - expect(result.targets[0]).not.toHaveProperty("short"); - }); - - it("counts a creation it proposes in this pass as booting, and leaves another kind's out", () => { - const result = plan(view([], { targets: [target(1), target(1, { model: "iPad Pro" })] })); - - expect(result.proposals).toHaveLength(1); - expect(result.targets.map((report) => report.booting)).toEqual([1, 0]); - }); - - it("reports a target held by the boot cap, or behind a waiter the pass could not serve, with no reason", () => { - const capped = plan(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] })); - expect(capped.targets[0]).toMatchObject({ booting: 0, count: 1, ready: 0 }); - expect(capped.targets[0]).not.toHaveProperty("short"); - - const queued = plan( - view([], { - refuseProvision: "device-limit", - targets: [target(1)], - waiting: [classDemand()], - }), - ); - expect(queued.targets[0]).not.toHaveProperty("short"); - }); - - it("shuts down an idle device no configured count keeps to make room for a target, and never one a configured count keeps", () => { - const idle = device("idle", "ready", { endedAgo: minute, spec: spec("iPad Pro") }); - const make = { action: "provision", reason: "target", spec: kind, target: kind }; - - const result = plan(view([idle], { limit: 1, targets: [target(1)] })); - - expect(result.proposals).toEqual([ - { action: "shutdown", deviceId: "idle", reason: "displaced" }, - make, - ]); - expect(result.targets[0]).toMatchObject({ booting: 1, ready: 0 }); - const pad = { count: 1, spec: spec("iPad Pro") }; - expect(evaluate(view([idle], { limit: 1, targets: [target(1), pad] }))).toEqual([]); - expect(evaluate(view([idle], { limit: 1, targets: [pad, target(1)] }))).toEqual([]); - }); - - it("shuts down a copy past another configured kind's count, for a slot the reserve holds too, and fills every configured count before a learned peak", () => { - const pad = spec("iPad Pro"); - const pads = ["a", "b"].map((id, index) => - device(id, "ready", { endedAgo: (index + 1) * minute, spec: pad }), - ); - const both = [{ count: 1, spec: pad }, target(1)]; - const make = { action: "provision", reason: "target", spec: kind, target: kind }; - - expect(evaluate(view(pads, { limit: 2, targets: both }))).toEqual([ - { action: "shutdown", deviceId: "b", reason: "displaced" }, - make, - ]); - expect( - evaluate(view(pads, { limits: { ios: 3 }, reserve: { ios: 1 }, targets: both })), - ).toEqual([{ action: "shutdown", deviceId: "b", reason: "displaced" }, make]); - expect( - evaluate(view([pads[0]!], { learned: [learnedOf(pad, 2, now)], limit: 2, targets: both })), - ).toEqual([make]); - }); - - it("counts two targets resolving to one spec as one with the summed count", () => { - const ready = ofKind("ready", "ready"); - - const proposals = evaluate( - view([ready], { maxConcurrentBoots: 5, targets: [target(1), target(2)] }), - ); - - expect(proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - expect(plan(view([ready], { targets: [target(1), target(2)] })).targets).toHaveLength(1); - }); - - it("does not count a slim device toward a full target of the same model", () => { - const slim = device("slim", "ready", { endedAgo: minute, spec: { ...kind, mode: "slim" } }); - - const proposals = evaluate(view([slim], { targets: [target(1)] })); - - expect(proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("proposes nothing for a target whose spec may not be attempted yet, and reports boot-failed", () => { - const shut = ofKind("shut", "shutdown"); - - const result = plan(view([shut], { blocked: [kind], targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: "boot-failed", count: 1, ready: 0 }); - }); - - it("proposes nothing for a target and reports none with warmPool disabled", () => { - const result = plan(view([], { enabled: false, targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets).toEqual([]); - }); - - it("does not boot or create for an android target out of an iOS slot", () => { - const androidKind = spec("Pixel 9", { osVersion: "36", platform: "android" }); - const iosBusy = device("busy", "leased"); - - const result = plan( - view([iosBusy], { - limit: 5, - limits: { android: 0 }, - targets: [{ count: 1, spec: androidKind }], - }), - ); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: "running-limit" }); - }); - - it("is not short while held back after a failure when another boot is running, or a request is queued", () => { - const shut = ofKind("shut", "shutdown"); - const held = { blocked: [kind], maxConcurrentBoots: 2, targets: [target(2)] }; - - expect(plan(view([shut], held)).targets[0]).toMatchObject({ short: "boot-failed" }); - expect(plan(view([shut], { ...held, inFlight: [kind] })).targets[0]).not.toHaveProperty( - "short", - ); - expect( - plan( - view([shut], { - ...held, - refuseProvision: "device-limit", - waiting: [modelDemand("iPad Pro")], - }), - ).targets[0], - ).not.toHaveProperty("short"); - }); - - it("spares the never-leased devices of a spared target and plans nothing for it", () => { - const stale = device("stale", "ready", { readyAt: 1, spec: kind }); - - const result = plan(view([stale], { spared: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets).toEqual([]); - expect(evaluate(view([stale]))).toEqual([ - { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, - ]); - }); - - it("reports no short for a filled target whose spec may not be attempted yet", () => { - const ready = ofKind("ready", "ready"); - - const result = plan(view([ready], { blocked: [kind], targets: [target(1)] })); - - expect(result.targets).toStrictEqual([ - { - booting: 0, - configuredCount: 1, - count: 1, - kind: { - mode: "full", - model: "iPhone 17", - osVersion: "26.0", - platform: "ios", - }, - origin: "configured", - ready: 1, - spec: kind, - target: "ios iPhone 17 26.0", - }, - ]); - }); - - it("proposes a boot of each of two shut-down devices once, and a creation for the target's third", () => { - const one = ofKind("one", "shutdown"); - const two = ofKind("two", "shutdown"); - - const proposals = evaluate(view([one, two], { maxConcurrentBoots: 3, targets: [target(3)] })); - - expect(proposals).toEqual([ - { action: "boot", deviceId: "one", reason: "target", target: kind }, - { action: "boot", deviceId: "two", reason: "target", target: kind }, - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("takes a slot for each proposal, so a target of two with room for one proposes one and is not short while it fills", () => { - const result = plan(view([], { limit: 1, maxConcurrentBoots: 2, targets: [target(2)] })); - - expect(result.proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - expect(result.targets[0]).toStrictEqual({ - booting: 1, - configuredCount: 2, - count: 2, - kind: { - mode: "full", - model: "iPhone 17", - osVersion: "26.0", - platform: "ios", - }, - origin: "configured", - ready: 0, - spec: kind, - target: "ios iPhone 17 26.0", - }); - }); - - it("is short for a reason only when a pass could do nothing for it: not with a boot already running", () => { - const busy = device("busy", "leased", { spec: spec("iPad Pro") }); - const full = { limit: 1, maxConcurrentBoots: 2, targets: [target(2)] }; - - expect(plan(view([busy], full)).targets[0]).toMatchObject({ short: "running-limit" }); - expect(plan(view([busy], { ...full, inFlight: [kind] })).targets[0]).not.toHaveProperty( - "short", - ); - }); - - it("reports device-limit before running-limit and reserve, and those before ram-budget", () => { - const both = plan( - view([], { - limit: 1, - refuseProvision: "device-limit", - targets: [target(1)], - reserve: { ios: 1 }, - }), - ); - expect(both.targets[0]).toMatchObject({ short: "device-limit" }); - const full = plan( - view([], { limit: 0, refuseProvision: "ram-budget", targets: [target(1)] }), - ); - expect(full.targets[0]).toMatchObject({ short: "running-limit" }); - const bothLimits = plan( - view([], { limit: 0, refuseProvision: "device-limit", targets: [target(1)] }), - ); - expect(bothLimits.targets[0]).toMatchObject({ short: "device-limit" }); - const reserved = plan( - view([], { - limits: { ios: 1 }, - refuseProvision: "ram-budget", - reserve: { ios: 1 }, - targets: [target(1)], - }), - ); - expect(reserved.targets[0]).toMatchObject({ short: "reserve" }); - }); - - it.each([ - ["the machine", { limit: 1, limits: { ios: 5 } }], - ["the platform", { limit: 5, limits: { ios: 1 } }], - ])( - "stops proposing for a target of two once the slot is taken when %s alone has room for one, and is not short", - (_name, limits) => { - const result = plan(view([], { ...limits, maxConcurrentBoots: 2, targets: [target(2)] })); - - expect(result.proposals).toEqual([ - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - expect(result.targets[0]).not.toHaveProperty("short"); - }, - ); - - it("does not count a boot in flight for another kind toward the target's count", () => { - const result = evaluate( - view([], { inFlight: [spec("iPad Pro")], maxConcurrentBoots: 2, targets: [target(1)] }), - ); - - expect(result).toEqual([{ action: "provision", reason: "target", spec: kind, target: kind }]); - }); - - it("proposes a creation with exactly one running slot free, and none with none free on either the platform or the machine", () => { - const busyAndroid = device("busy", "leased", { - spec: spec("Pixel 9", { osVersion: "36", platform: "android" }), - }); - - expect(evaluate(view([], { limit: 1, targets: [target(1)] }))).toHaveLength(1); - const global = plan( - view([busyAndroid], { limit: 1, limits: { ios: 5 }, targets: [target(1)] }), - ); - expect(global.proposals).toEqual([]); - expect(global.targets[0]).toMatchObject({ short: "running-limit" }); - const platform = plan(view([], { limit: 5, limits: { ios: 0 }, targets: [target(1)] })); - expect(platform.proposals).toEqual([]); - expect(platform.targets[0]).toMatchObject({ short: "running-limit" }); - }); - - it("keeps the reserve out of a target's reach on the machine and on the platform, and uses the last slot beyond it", () => { - const reserve = { android: 1, ios: 1 }; - - const machine = plan(view([], { limit: 2, reserve, targets: [target(1)] })); - expect(machine.proposals).toEqual([]); - expect(machine.targets[0]).toMatchObject({ short: "reserve" }); - expect(evaluate(view([], { limit: 3, reserve, targets: [target(1)] }))).toHaveLength(1); - - const platform = plan( - view([], { limits: { ios: 1 }, reserve: { ios: 1 }, targets: [target(1)] }), - ); - expect(platform.targets[0]).toMatchObject({ short: "reserve" }); - expect( - evaluate(view([], { limits: { ios: 2 }, reserve: { ios: 1 }, targets: [target(1)] })), - ).toHaveLength(1); - }); - - it.each(["global-running-limit", "platform-running-limit"] as const)( - "reports running-limit when the capacity strategy refuses a creation for %s", - (reason) => { - const result = plan(view([], { refuseProvision: reason, targets: [target(1)] })); - - expect(result.proposals).toEqual([]); - expect(result.targets[0]).toMatchObject({ short: "running-limit" }); - }, - ); - - it("frees the slot of a never-leased device it shuts down for a target of another kind in the same pass", () => { - const stale = device("stale", "ready", { readyAt: 1, spec: spec("iPad Pro") }); - - const proposals = evaluate(view([stale], { limit: 1, targets: [target(1)] })); - - expect(proposals).toEqual([ - { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, - { action: "provision", reason: "target", spec: kind, target: kind }, - ]); - }); - - it("proposes a never-leased device once when the budget already proposes it", () => { - const first = device("first", "ready", { readyAt: 1 }); - const second = device("second", "ready", { readyAt: 1 }); - - const proposals = evaluate(view([first, second], { limit: 1 })); - - expect(proposals).toEqual([ - { action: "shutdown", deviceId: "first", reason: "over-budget" }, - { action: "shutdown", deviceId: "second", reason: "never-leased-idle" }, - ]); - }); - - it("spares a never-leased device only for a queued request it serves, not one already booting or one it does not serve", () => { - const stale = device("stale", "ready", { readyAt: 1 }); - const shutdown = { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }; - - expect(evaluate(view([stale], { waiting: [modelDemand("iPhone 15")] }))).toEqual([]); - expect( - evaluate(view([stale], { waiting: [{ ...modelDemand("iPhone 15"), inFlight: true }] })), - ).toEqual([shutdown]); - expect(evaluate(view([stale], { waiting: [modelDemand("iPad Pro")] }))).toEqual([shutdown]); - }); - - it("does not name a device with a lease record as targeted even when its state still reads ready", () => { - const leasedReady = ofKind("leased-ready", "ready"); - - const named = targetedDevices({ - devices: [leasedReady], - isClaimed: () => false, - leases: [leaseOn(leasedReady)], - learned: [], - targets: [target(1)], - waiting: [], - }); - - expect([...named]).toEqual([]); - }); - - describe("targetedDevices", () => { - const input = (devices: readonly VirtualDevice[], targets: readonly ResolvedTarget[]) => ({ - devices, - isClaimed: (id: string) => id === "claimed", - leases: devices.filter((item) => item.state === "leased").map(leaseOn), - learned: [] as WarmPolicyView["learned"], - targets, - waiting: [] as readonly WaitingDemand[], - }); - - it("names, for each target, its ready unleased devices of that kind, oldest first, up to the count", () => { - const newer = ofKind("newer", "ready", { createdAt: 20 }); - const older = ofKind("older", "ready", { createdAt: 10 }); - const oldest = ofKind("oldest", "ready", { createdAt: 5 }); - const iPad = device("ipad", "ready", { spec: spec("iPad Pro") }); - - expect( - [...targetedDevices(input([newer, older, oldest, iPad], [target(2)]))].sort(), - ).toEqual(["older", "oldest"]); - }); - - it("names a learned kind's ready unleased devices up to its count, oldest first, and the larger of a configured and a learned count", () => { - const devices = [ - ofKind("newer", "ready", { createdAt: 20 }), - ofKind("older", "ready", { createdAt: 10 }), - ofKind("oldest", "ready", { createdAt: 5 }), - device("ipad", "ready", { spec: spec("iPad Pro") }), - ]; - const named = (learnedCount: number, targets: readonly ResolvedTarget[] = []) => - [ - ...targetedDevices({ - ...input(devices, targets), - learned: [learnedOf(kind, learnedCount)], - }), - ].sort(); - - expect(named(2)).toEqual(["older", "oldest"]); - expect(named(1)).toEqual(["oldest"]); - expect(named(1, [target(2)])).toEqual(["older", "oldest"]); - expect(named(3, [target(2)])).toEqual(["newer", "older", "oldest"]); - }); - - it("leaves out a leased, claimed, shut-down or spent device and one of another mode", () => { - const devices = [ - ofKind("leased", "leased"), - ofKind("claimed", "ready"), - ofKind("shut", "shutdown"), - ofKind("spent", "ready", { endedAgo: 1, leaseIdentity: "fresh", spent: true }), - device("slim", "ready", { spec: { ...kind, mode: "slim" } }), - ofKind("good", "ready"), - ]; - - expect([...targetedDevices(input(devices, [target(5)]))]).toEqual(["good"]); - }); - - it("protects queued demand only for ready idle grantable devices matching its requirement and mode", () => { - const ready = device("ready", "ready"); - const waiting = [classDemand()]; - const cases = [ - ["matching class", view([ready], { waiting }), ["ready"]], - [ - "matching exact model", - view([ready], { waiting: [modelDemand("iPhone 15")] }), - ["ready"], - ], - ["no devices", view([], { waiting }), []], - [ - "later active waiter", - view([ready], { - waiting: [classDemand("slim"), classDemand("full", "ios", true), ...waiting], - }), - ["ready"], - ], - [ - "union with configured and learned kinds", - view( - [ - ofKind("targeted", "ready"), - device("learned", "ready", { spec: spec("iPad Pro") }), - ready, - ], - { - targets: [target(1)], - learned: [learnedOf(spec("iPad Pro"))], - waiting, - }, - ), - ["targeted", "learned", "ready"], - ], - [ - "another matching model", - view([{ ...ready, spec: spec("iPhone 17") }], { waiting }), - ["ready"], - ], - ["no demand", view([ready]), []], - ["in flight", view([ready], { waiting: [classDemand("full", "ios", true)] }), []], - ["claimed", view([ready], { claimed: [ready.id], waiting }), []], - ["leased record", view([ready], { leased: [ready], waiting }), []], - ["shutdown", view([{ ...ready, state: "shutdown" }], { waiting }), []], - [ - "spent fresh", - view([device("ready", "ready", { endedAgo: 1, leaseIdentity: "fresh", spent: true })], { - waiting, - }), - [], - ], - [ - "spent without fresh identity", - view([device("ready", "ready", { endedAgo: 1, spent: true })], { waiting }), - [], - ], - [ - "unused fresh", - view([device("ready", "ready", { leaseIdentity: "fresh" })], { waiting }), - ["ready"], - ], - ["mode", view([ready], { waiting: [classDemand("slim")] }), []], - ["platform", view([ready], { waiting: [classDemand("full", "android")] }), []], - ["model", view([ready], { waiting: [modelDemand("iPhone 17")] }), []], - ["class", view([{ ...ready, spec: spec("iPad Pro") }], { waiting }), []], - [ - "OS", - view([{ ...ready, spec: spec("iPhone 15", { osVersion: "25.0" }) }], { waiting }), - [], - ], - [ - "image", - view([ready], { - waiting: [ - { - ...classDemand(), - requirement: { ...classDemand().requirement, imageTag: "google_apis" }, - }, - ], - }), - [], - ], - ] as const; - - for (const [name, policyView, expected] of cases) { - expect([...targetedDevices(policyView)], name).toEqual(expected); - } - }); - - it("sums the counts of targets of one spec and keeps a different spec apart", () => { - const devices = [ - ofKind("a", "ready", { createdAt: 1 }), - ofKind("b", "ready", { createdAt: 2 }), - ofKind("c", "ready", { createdAt: 3 }), - device("ipad", "ready", { spec: spec("iPad Pro") }), - ]; - - expect( - [ - ...targetedDevices( - input(devices, [target(1), target(2), { count: 1, spec: spec("iPad Pro") }]), - ), - ].sort(), - ).toEqual(["a", "b", "c", "ipad"]); - }); - }); - - describe("learned kinds", () => { - const older = spec("iPad Pro"); - const provision = (wanted: VirtualSpec) => ({ - action: "provision", - reason: "target", - spec: wanted, - target: wanted, - }); - - it("fills configured targets in config order, then learned kinds newest first, booting a shut-down device of the kind before creating", () => { - const learned = [learnedOf(kind, 1, now - minute), learnedOf(older, 1, now - 2 * minute)]; - - const first = plan(view([], { learned, limit: 1, maxConcurrentBoots: 2 })); - expect(first.proposals).toEqual([provision(kind)]); - expect(first.targets.map((report) => [report.target, report.origin, report.short])).toEqual( - [ - ["ios iPhone 17 26.0", "learned", undefined], - ["ios iPad Pro 26.0", "learned", "running-limit"], - ], - ); - - const configured = spec("iPhone 15"); - expect( - evaluate(view([], { learned, limit: 1, targets: [{ count: 1, spec: configured }] })), - ).toEqual([provision(configured)]); - - const shut = ofKind("shut", "shutdown"); - expect(evaluate(view([shut], { learned: [learnedOf(kind, 1, now)] }))).toEqual([ - { action: "boot", deviceId: "shut", reason: "target", target: kind }, - ]); - - const merged = plan( - view([ofKind("a", "leased")], { - learned: [learnedOf(kind, 2, now), learnedOf(older, 1, now - minute)], - limit: 2, - targets: [{ count: 1, spec: kind }], - }), - ); - expect(merged.proposals).toEqual([provision(kind)]); - expect( - merged.targets.map((report) => [report.origin, report.count, report.configuredCount]), - ).toEqual([ - ["configured", 2, 1], - ["learned", 1, 0], - ]); - }); - - it("plans enough copies to keep max(peak - leased, 1) of a learned kind ready, counting copies on their way, for a plain kind and a baseline kind alike", () => { - const baselined = { ...kind, baseline: { name: "demo", version: 1 } }; - const cases = [ - ["peak 1, 1 leased", [ofKind("a", "leased")], 1, kind, [], [provision(kind)]], - ["peak 1, 1 reclaiming", [ofKind("a", "reclaiming")], 1, kind, [], [provision(kind)]], - [ - "peak 1, 1 ready under a claim", - [ofKind("a", "ready")], - 1, - kind, - ["a"], - [provision(kind)], - ], - [ - "peak 3, 3 leased", - [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "leased")], - 3, - kind, - [], - [provision(kind)], - ], - [ - "peak 3, 2 leased + 1 reclaiming", - [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "reclaiming")], - 3, - kind, - [], - [provision(kind)], - ], - [ - "peak 3, 1 leased + 1 ready", - [ofKind("a", "leased"), ofKind("b", "ready")], - 3, - kind, - [], - [provision(kind)], - ], - [ - "peak 3, 3 leased + 1 boot in flight", - [ofKind("a", "leased"), ofKind("b", "leased"), ofKind("c", "leased")], - 3, - kind, - [], - [], - [kind], - ], - [ - "baseline kind, peak 1, 1 leased", - [ofKind("a", "leased", { spec: baselined })], - 1, - baselined, - [], - [provision(baselined)], - ], - [ - "baseline kind, peak 2, 2 copies of version 1 leased while version 2 is newest", - [ - ofKind("a", "leased", { spec: baselined }), - ofKind("b", "leased", { spec: baselined }), - ], - 2, - { ...kind, baseline: { name: "demo", version: 2 } }, - [], - [provision({ ...kind, baseline: { name: "demo", version: 2 } })], - ], - [ - "baseline kind, peak 2, 2 copies of version 1 on another model leased while version 2 is newest", - [ - ofKind("a", "leased", { spec: { ...baselined, model: "iPhone 15" } }), - ofKind("b", "leased", { spec: { ...baselined, model: "iPhone 15" } }), - ], - 2, - { ...kind, baseline: { name: "demo", version: 2 } }, - [], - [provision({ ...kind, baseline: { name: "demo", version: 2 } })], - ], - [ - "baseline kind, peak 2, 2 copies of version 1 on another platform leased while version 2 is newest", - [ - ofKind("a", "leased", { spec: { ...baselined, platform: "android" } }), - ofKind("b", "leased", { spec: { ...baselined, platform: "android" } }), - ], - 2, - { ...kind, baseline: { name: "demo", version: 2 } }, - [], - [provision({ ...kind, baseline: { name: "demo", version: 2 } })], - ], - ] as const; - - for (const [name, devices, peak, wanted, claimed, expected, inFlight = []] of cases) { - expect( - evaluate( - view(devices, { - claimed, - inFlight, - learned: [learnedOf(wanted, peak, now)], - maxConcurrentBoots: 3, - }), - ), - name, - ).toEqual(expected); - } - }); - - it("plans the larger of what the configured count and the learned count each lack for a kind both configured and learned", () => { - const cases = [ - ["configured 1 + peak 2, 1 leased", 1, 2, 1], - ["configured 2 + peak 1, 1 leased", 2, 1, 2], - ["configured 1 + peak 3, 1 leased", 1, 3, 2], - ] as const; - - for (const [name, configuredCount, peak, created] of cases) { - expect( - evaluate( - view([ofKind("a", "leased")], { - learned: [learnedOf(kind, peak, now)], - maxConcurrentBoots: 3, - targets: [{ count: configuredCount, spec: kind }], - }), - ), - name, - ).toEqual(Array.from({ length: created }, () => provision(kind))); - } - }); - - it("keeps a learned kind's never-leased device out of never-leased-idle while the kind wants it, and proposes one beyond its count after idle.shutdownAfterMs", () => { - const old = now - shutdownAfterMs - 1; - const wanted = ofKind("wanted", "ready", { createdAt: 1, readyAt: old }); - const extra = ofKind("extra", "ready", { createdAt: 2, readyAt: old }); - const young = ofKind("young", "ready", { createdAt: 3, readyAt: now - 1_000 }); - const learned = [learnedOf(kind, 1, now)]; - - expect(evaluate(view([wanted], { learned }))).toEqual([]); - expect(evaluate(view([wanted, extra], { learned }))).toEqual([ - { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, - ]); - expect(evaluate(view([wanted, young], { learned }))).toEqual([]); - expect(evaluate(view([wanted, extra], { learned: [learnedOf(kind, 2, now)] }))).toEqual([]); - }); - - it("keeps a learned kind and its devices but plans nothing for it while it is unplanned, and still plans a configured target of its spec", () => { - const learned = [learnedOf(kind, 1, now)]; - const old = now - shutdownAfterMs - 1; - const spare = ofKind("spare", "ready", { createdAt: 1, readyAt: old }); - - expect(evaluate(view([], { learned, unplanned: [kind] }))).toEqual([]); - expect(evaluate(view([spare], { learned, unplanned: [kind] }))).toEqual([]); - expect(evaluate(view([], { learned }))).toEqual([provision(kind)]); - expect( - evaluate(view([], { learned, targets: [{ count: 1, spec: kind }], unplanned: [kind] })), - ).toEqual([provision(kind)]); - }); - - it("displaces the least recently used idle ready device of an older learned kind for a newer one short for a running slot, and never a configured target's device, one that serves a waiting request or is already shut down for the budget, or for a kind short for ram-budget, device-limit or reserve; a configured target displaces it", () => { - const learned = [learnedOf(kind, 1, now - minute), learnedOf(older, 2, now - 2 * minute)]; - const recent = device("recent", "ready", { endedAgo: 5 * minute, spec: older }); - const stale = device("stale", "ready", { endedAgo: 30 * minute, spec: older }); - const displace = { action: "shutdown", deviceId: "stale", reason: "displaced" }; - - const base = plan(view([recent, stale], { learned, limit: 2, maxConcurrentBoots: 2 })); - expect(base.proposals).toEqual([displace, provision(kind)]); - expect(base.targets.map((report) => [report.target, report.ready, report.short])).toEqual([ - ["ios iPhone 17 26.0", 0, undefined], - ["ios iPad Pro 26.0", 1, "running-limit"], - ]); - - const refused = (options: Parameters[1]) => - evaluate(view([recent, stale], { learned, maxConcurrentBoots: 2, ...options })); - expect(refused({ limit: 2 })).toEqual([displace, provision(kind)]); - expect(refused({ limit: 2, targets: [{ count: 1, spec: older }] })).toEqual([]); - expect(refused({ limit: 2, waiting: [modelDemand("iPad Pro")] })).toEqual([]); - expect(evaluate(view([recent], { learned, limit: 1, maxConcurrentBoots: 2 }))).toEqual([ - { ...displace, deviceId: "recent" }, - provision(kind), - ]); - expect(refused({ handoffInFlight: true, limit: 2 })).toEqual([]); - const holder = device("holder", "leased", { spec: spec("iPad Air") }); - expect( - evaluate( - view([holder, stale], { - learned: [learnedOf(kind, 2, now - minute), learnedOf(older, 1, now - 2 * minute)], - limit: 1, - maxConcurrentBoots: 2, - }), - ), - ).toEqual([{ action: "shutdown", deviceId: "stale", reason: "over-budget" }]); - const droid = device("droid", "ready", { - endedAgo: 40 * minute, - spec: spec("Pixel 8", { platform: "android" }), - }); - expect( - evaluate( - view([droid], { - learned: [learnedOf(kind, 1, now), learnedOf(droid.spec, 1, now - minute)], - limit: 2, - limits: { ios: 0 }, - maxConcurrentBoots: 2, - }), - ), - ).toEqual([]); - expect(refused({ limit: 3, refuseProvision: "ram-budget" })).toEqual([]); - // why: short for a slot and for RAM the shutdown would not free: no shutdown, or the pool would reboot the victim every pass. - expect(refused({ limit: 2, refuseProvision: "ram-budget" })).toEqual([]); - expect(refused({ limit: 2, refuseProvisionUntilDown: "ram-budget" })).toEqual([ - displace, - provision(kind), - ]); - expect(refused({ limit: 2, spared: [{ count: 2, spec: older }] })).toEqual([]); - expect(refused({ limit: 3, refuseProvision: "device-limit" })).toEqual([]); - expect(refused({ limit: 3, reserve: { ios: 1 } })).toEqual([]); - - const newer = device("newer", "ready", { endedAgo: 30 * minute, spec: kind }); - expect( - evaluate( - view([newer], { - learned, - limit: 1, - maxConcurrentBoots: 2, - }), - ), - ).toEqual([]); - const phone = spec("iPhone 15"); - expect( - evaluate( - view([recent], { - learned: [learnedOf(older, 1, now)], - limit: 1, - maxConcurrentBoots: 2, - targets: [{ count: 1, spec: phone }], - }), - ), - ).toEqual([{ ...displace, deviceId: "recent" }, provision(phone)]); - }); - }); - }); - describe("memory pressure", () => { - const kind = spec("iPhone 17"); - const paused = { mayShed: false, step: "warming-paused" as const }; - const waiter = (inFlight = false) => classDemand("full", "ios", inFlight, "iPhone 17"); - - it("at warming-paused starts nothing and reports every short target as memory-pressure, in place of any other reason", () => { - const busy = device("busy", "leased", { spec: spec("iPad Pro") }); - const shut = device("shut", "shutdown", { endedAgo: 20 * minute, spec: kind }); - const cases = [ - { name: "configured target", options: { targets: [{ count: 1, spec: kind }] } }, - { name: "learned kind", options: { learned: [learnedOf(kind)] } }, - { - name: "configured target also short for the reserve", - options: { - limits: { ios: 2 }, - reserve: { ios: 1 }, - targets: [{ count: 1, spec: kind }], - }, - other: "reserve", - proposes: false, - }, - { - name: "configured target also short for a boot that failed", - options: { blocked: [kind], targets: [{ count: 1, spec: kind }] }, - other: "boot-failed", - proposes: false, - }, - { - name: "creation for a queued request", - options: { maxConcurrentBoots: 3, waiting: [waiter(true), waiter(), waiter()] }, - }, - { - name: "boot of a shut-down device for a queued request", - devices: [busy, shut], - options: { waiting: [waiter()] }, - }, - ]; - - const refused = plan( - view([busy], { refuseProvision: "memory-pressure", targets: [{ count: 1, spec: kind }] }), - ); - expect(refused.targets[0]?.short, "a capacity refusal for memory pressure").toBe( - "memory-pressure", - ); - - for (const { devices = [busy], name, options, other, proposes = true } of cases) { - const open = plan(view(devices, options)); - const held = plan(view(devices, { ...options, pressure: paused })); - if (proposes) - expect(open.proposals, `${name} is proposed when memory is not short`).not.toEqual([]); - expect(held.proposals, name).toEqual([]); - expect(open.targets[0]?.short, name).toBe(other); - if (options.targets !== undefined || options.learned !== undefined) - expect(held.targets[0]?.short, name).toBe("memory-pressure"); - } - }); - - it("sheds one idle device a reading: a device no configured target keeps first, then a kept one, least recently used first, never one a waiter would take, one the budget already shuts down or one whose shutdown just failed", () => { - const ofKind = (id: string, endedAgo: number) => - device(id, "ready", { endedAgo, spec: kind }); - const keptOldest = ofKind("kept", 90 * minute); - const extra = ofKind("extra", 40 * minute); - const olderOther = device("older", "ready", { - endedAgo: 60 * minute, - spec: spec("iPad Pro"), - }); - const newerOther = device("newer", "ready", { endedAgo: 5 * minute, spec: spec("iPad Pro") }); - const learnedOnly = device("learned", "ready", { endedAgo: 30 * minute }); - const targets = [{ count: 1, spec: kind }]; - const sheds = ( - devices: readonly VirtualDevice[], - options: Parameters[1] = {}, - pressure: NonNullable[1]>["pressure"]> = {}, - ): string[] => - evaluate( - view(devices, { targets, ...options, pressure: { step: "warming-paused", ...pressure } }), - ).flatMap((proposal) => - proposal.action === "shutdown" && proposal.reason === "memory-pressure" - ? [proposal.deviceId] - : [], - ); - - expect(sheds([keptOldest, newerOther, olderOther]), "unkept, least recently used").toEqual([ - "older", - ]); - expect(sheds([keptOldest, extra]), "a copy past the configured count").toEqual(["extra"]); - expect(sheds([keptOldest]), "a kept device when nothing else is idle").toEqual(["kept"]); - expect( - sheds([learnedOnly, keptOldest], { learned: [learnedOf(learnedOnly.spec)] }), - "a learned kind keeps nothing", - ).toEqual(["learned"]); - expect( - sheds([keptOldest, extra], { spared: targets, targets: [] }), - "a spared target's device counts as kept", - ).toEqual(["extra"]); - expect( - sheds([keptOldest, olderOther, newerOther], { - waiting: [modelDemand("iPad Pro")], - }), - "a device a waiter would take", - ).toEqual(["kept"]); - expect( - sheds([keptOldest, olderOther, newerOther], { claimed: ["older"] }), - "a claimed device", - ).toEqual(["newer"]); - expect( - sheds([keptOldest, olderOther, newerOther], {}, { shedFailed: ["older"] }), - "a device whose shutdown just failed", - ).toEqual(["newer"]); - expect(sheds([keptOldest, olderOther], {}, { mayShed: false }), "no new reading").toEqual([]); - expect(sheds([keptOldest, olderOther], {}, { step: "none" }), "step none").toEqual([]); - expect(sheds([device("busy", "leased")]), "nothing idle").toEqual([]); - - const overBudget = evaluate( - view([keptOldest, olderOther, newerOther], { - limit: 2, - pressure: { step: "warming-paused" }, - targets: [], - }), - ).flatMap((proposal) => (proposal.action === "shutdown" ? [proposal.deviceId] : [])); - expect(overBudget, "a device the budget shuts down is not shed again").toEqual([ - "kept", - "older", - ]); - }); - }); }); diff --git a/src/core/warm-pool/test-policy.ts b/src/core/warm-pool/test-policy.ts new file mode 100644 index 000000000..d0c122f5b --- /dev/null +++ b/src/core/warm-pool/test-policy.ts @@ -0,0 +1,230 @@ +import type { CapacityRefusalReason, RunningCapacity } from "../capacity/index.js"; +import type { + DeviceClass, + VirtualDevice, + VirtualSpec, + LeaseRecord, + WaitingDemand, +} from "../domain.js"; +import { sameSpec } from "../domain.js"; +import { evaluate as plan, type ResolvedTarget, type WarmPolicyView } from "./policy.js"; + +export const evaluate = (policyView: WarmPolicyView) => plan(policyView).proposals; + +export const minute = 60_000; +export const now = 1_000 * minute; +export const shutdownAfterMs = 10 * minute; + +export const phones = new Set(["iPhone 15", "iPhone 17"]); +export const classOf = (model: string): DeviceClass | undefined => + phones.has(model) ? "phone" : "tablet"; + +export function spec(model: string, overrides: Partial = {}): VirtualSpec { + return { model, osVersion: "26.0", physical: false, platform: "ios", ...overrides }; +} + +export interface DeviceOptions { + spec?: VirtualSpec; + endedAgo?: number; + createdAt?: number; + readyAt?: number; + leaseIdentity?: "fresh"; + spent?: boolean; +} + +export function readyStamp( + options: DeviceOptions, +): Pick | Record { + return options.readyAt === undefined && options.endedAgo === undefined + ? {} + : { readyAt: options.readyAt ?? now }; +} + +// why: a virtual device that ended a lease is spent (ADR 0061 §2) and never grantable; `endedAgo` +// only orders idle devices (least recently used), so it moves createdAt, and only a spent device +// carries the end stamp. Such a device counts as freshly ready, so the never-leased idle rule leaves it be. +export function endStamp( + options: DeviceOptions, +): Pick | Record { + return options.spent === true && options.endedAgo !== undefined + ? { lastLeaseEndedAt: now - options.endedAgo } + : {}; +} + +export function device( + id: string, + state: VirtualDevice["state"], + options: DeviceOptions = {}, +): VirtualDevice { + return { + physical: false, + createdAt: options.createdAt ?? (options.endedAgo === undefined ? 1 : now - options.endedAgo), + ...readyStamp(options), + ...(options.leaseIdentity === undefined ? {} : { leaseIdentity: options.leaseIdentity }), + driverData: {}, + driverDeviceId: `driver-${id}`, + id, + mode: "full", + spec: options.spec ?? spec("iPhone 15"), + state, + ...endStamp(options), + }; +} + +export function learnedOf( + learnedSpec: VirtualSpec, + count = 1, + lastLeasedAt = now, +): WarmPolicyView["learned"][number] { + return { count, lastLeasedAt, spec: learnedSpec }; +} + +export function leaseOn(target: VirtualDevice): LeaseRecord { + return { + deviceId: target.id, + grantedAt: 1, + id: `lease-${target.id}`, + lastRenewedAt: 1, + idChosenByRequester: false, + ownerId: "holder", + requesterId: "holder", + ttlDeadline: now + minute, + ttlMs: minute, + }; +} + +export function capacityOf( + devices: readonly VirtualDevice[], + limits: { global: number; ios?: number; android?: number }, + reserved = 0, +): RunningCapacity { + const running = (platform?: string): number => + devices.filter( + (item) => + ["ready", "leased", "reclaiming", "quarantined"].includes(item.state) && + (platform === undefined || item.spec.platform === platform), + ).length; + const entry = (maxRunning: number, platform?: string) => ({ + maxRunning, + overLimit: running(platform) + reserved > maxRunning, + reserved, + running: running(platform), + }); + return { + android: entry(limits.android ?? limits.global, "android"), + global: entry(limits.global), + ios: entry(limits.ios ?? limits.global, "ios"), + }; +} + +export function classDemand( + mode: "slim" | "full" = "full", + platform: "ios" | "android" = "ios", + inFlight = false, + model = "iPhone 15", +) { + return { + classOf, + inFlight, + mode, + platform, + spec: spec(model, { platform, ...(mode === "slim" ? { mode } : {}) }), + requirement: { + imageTag: undefined, + osVersion: { kind: "installed", versions: ["26.0"] }, + platform, + target: { class: "phone", kind: "class" }, + }, + } satisfies WaitingDemand; +} + +export function modelDemand(model: string): WaitingDemand { + return { + classOf, + inFlight: false, + mode: "full", + platform: "ios", + spec: spec(model), + requirement: { + imageTag: undefined, + osVersion: { kind: "exact", version: "26.0" }, + platform: "ios", + target: { kind: "model", model }, + }, + }; +} + +// fallow-ignore-next-line complexity -- one test fixture builder; each option is a plain pass-through to a view field. +export function view( + devices: readonly VirtualDevice[], + options: { + limit?: number; + limits?: { ios?: number; android?: number }; + reserve?: { ios?: number; android?: number }; + reserved?: number; + waiting?: readonly WaitingDemand[]; + enabled?: boolean; + claimed?: readonly string[]; + leased?: readonly VirtualDevice[]; + handoffInFlight?: boolean; + reset?: readonly string[]; + targets?: readonly ResolvedTarget[]; + spared?: readonly ResolvedTarget[]; + learned?: WarmPolicyView["learned"]; + inFlight?: readonly VirtualSpec[]; + onItsWay?: WarmPolicyView["onItsWay"]; + maxConcurrentBoots?: number; + blocked?: readonly VirtualSpec[]; + refuseProvision?: CapacityRefusalReason; + refuseBoot?: CapacityRefusalReason; + /** why: a creation the capacity strategy refuses until the plan has shut a device down. */ + refuseProvisionUntilDown?: CapacityRefusalReason; + unplanned?: readonly VirtualSpec[]; + pressure?: { + step?: WarmPolicyView["pressure"]["step"]; + mayShed?: boolean; + shedFailed?: readonly string[]; + }; + } = {}, +): WarmPolicyView { + return { + admit: { + boot: () => options.refuseBoot, + create: (_spec, down) => + options.refuseProvision ?? + ((down?.size ?? 0) === 0 ? options.refuseProvisionUntilDown : undefined), + }, + unplanned: options.unplanned ?? [], + inFlight: options.inFlight ?? [], + onItsWay: options.onItsWay ?? [], + retry: { + mayAttempt: (target) => !(options.blocked ?? []).some((held) => sameSpec(held, target)), + }, + spared: options.spared ?? [], + learned: options.learned ?? [], + targets: options.targets ?? [], + capacity: capacityOf( + devices, + { global: options.limit ?? 10, ...options.limits }, + options.reserved, + ), + config: { + enabled: options.enabled ?? true, + maxConcurrentBoots: options.maxConcurrentBoots ?? 1, + reserveRunning: { android: 0, ios: 0, ...options.reserve }, + shutdownAfterMs, + }, + devices, + handoffInFlight: options.handoffInFlight ?? false, + resetDevices: new Set(options.reset ?? []), + pressure: { + mayShed: options.pressure?.mayShed ?? true, + shedFailed: new Set(options.pressure?.shedFailed ?? []), + step: options.pressure?.step ?? "none", + }, + isClaimed: (id) => options.claimed?.includes(id) ?? false, + leases: (options.leased ?? devices.filter((item) => item.state === "leased")).map(leaseOn), + now, + waiting: options.waiting ?? [], + }; +} From 7df7598432cb4b384fa16c1a16fecf1130cca94f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:28:22 +0100 Subject: [PATCH 51/83] test(core): split converger.test.ts by topic --- .../converger-memory-pressure.test.ts | 117 ++ .../converger-targets-creation.test.ts | 582 +++++++ .../converger-targets-passes.test.ts | 340 ++++ .../converger-targets-reports.test.ts | 433 +++++ src/core/warm-pool/converger.test.ts | 1499 +---------------- src/core/warm-pool/test-converger.ts | 42 + 6 files changed, 1515 insertions(+), 1498 deletions(-) create mode 100644 src/core/warm-pool/converger-memory-pressure.test.ts create mode 100644 src/core/warm-pool/converger-targets-creation.test.ts create mode 100644 src/core/warm-pool/converger-targets-passes.test.ts create mode 100644 src/core/warm-pool/converger-targets-reports.test.ts create mode 100644 src/core/warm-pool/test-converger.ts diff --git a/src/core/warm-pool/converger-memory-pressure.test.ts b/src/core/warm-pool/converger-memory-pressure.test.ts new file mode 100644 index 000000000..429d315e8 --- /dev/null +++ b/src/core/warm-pool/converger-memory-pressure.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { device, fakeMemoryPressure, harness, iphone15, minute, now } from "./test-fixtures.js"; + +describe("warm pool memory pressure", () => { + const paused = (readingAt: number) => ({ + cause: { availablePercent: 12, level: "warn" as const, trigger: "level" as const }, + readingAt, + since: now, + step: "warming-paused" as const, + }); + + it("each reading while warming is paused shuts down one idle device", async () => { + const pressure = fakeMemoryPressure(paused(now)); + const rig = harness( + [ + device("old", "ready", 90 * minute), + device("mid", "ready", 60 * minute), + device("new", "ready", 5 * minute), + ], + { memoryPressure: pressure.reader }, + ); + rig.pool.start(); + + pressure.reading(paused(now + 15_000)); + await rig.pool.settle(); + expect(rig.shutdownCalls).toEqual(["old"]); + + pressure.reading(paused(now + 30_000)); + await rig.pool.settle(); + expect(rig.shutdownCalls).toEqual(["old", "mid"]); + + await rig.pool.pass(); + expect(rig.shutdownCalls, "a second pass on the same reading").toEqual(["old", "mid"]); + }); + + it("sheds nothing for a reading before start, at step none, with no step reading or nothing idle, and skips a device whose shutdown just failed", async () => { + const idle = () => [device("old", "ready", 90 * minute), device("mid", "ready", 60 * minute)]; + const none = { ...paused(now), step: "none" as const }; + const cases = [ + { name: "before start", reading: paused(now + 15_000), start: false }, + { name: "step none", reading: none }, + { name: "no step reading", reading: undefined }, + { + devices: [device("old", "leased", 90 * minute)], + name: "nothing idle", + reading: paused(now + 15_000), + }, + ]; + for (const { devices = idle(), name, reading, start = true } of cases) { + const pressure = fakeMemoryPressure(reading); + const rig = harness(devices, { memoryPressure: pressure.reader }); + if (start) rig.pool.start(); + pressure.reading(reading === undefined ? undefined : { ...reading, readingAt: now + 15_000 }); + await rig.pool.settle(); + expect(rig.shutdownCalls, name).toEqual([]); + } + + const pressure = fakeMemoryPressure(paused(now)); + const failing = harness(idle(), { + memoryPressure: pressure.reader, + shutdown: async (target) => { + if (target.id === "old") throw new Error("the simulator would not stop"); + return { ...target, state: "shutdown" }; + }, + }); + failing.pool.start(); + pressure.reading(paused(now + 15_000)); + await failing.pool.settle(); + pressure.reading(paused(now + 30_000)); + await failing.pool.settle(); + expect(failing.shutdownCalls, "the next reading takes the next device").toEqual(["old", "mid"]); + }); + + it("when warming resumes the pool fills its targets again on memory.pressure-changed, with no tick", async () => { + const pressure = fakeMemoryPressure(paused(now)); + const rig = harness([], { memoryPressure: pressure.reader, targets: [iphone15] }); + rig.pool.start(); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls, "nothing is created while paused").toEqual([]); + + pressure.reading({ readingAt: now + 15_000, step: "none" }); + rig.eventBus.emit( + "memory.pressure-changed", + { from: "warming-paused", to: "none" }, + "host-load", + ); + await rig.pool.settle(); + + expect(rig.provisionCalls).toHaveLength(1); + }); + + it("at warming-paused reports a short target memory-pressure, a target that does not resolve its own reason, and boots nothing", async () => { + const rig = harness([device("shut", "shutdown", 1_000)], { + memoryPressure: fakeMemoryPressure(paused(now)).reader, + resolve: (request) => + request.model === "iPhone 17" + ? { message: "iOS 27.0 is not installed", refusal: "runtime-missing" } + : { spec: { model: "iPhone 15", osVersion: "26.0", physical: false, platform: "ios" } }, + targets: [iphone15, { ...iphone15, model: "iPhone 17", osVersion: "27.0" }], + }); + const reasons: Record = {}; + rig.eventBus.subscribe("warm-pool.target-missed", ({ payload }) => { + reasons[payload.model] = payload.reason; + }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(reasons).toEqual({ "iPhone 15": "memory-pressure", "iPhone 17": "runtime-missing" }); + expect( + Object.fromEntries(rig.pool.figures().targets.map((target) => [target.model, target.short])), + ).toEqual({ "iPhone 15": "memory-pressure", "iPhone 17": "runtime-missing" }); + expect(rig.bootCalls).toEqual([]); + expect(rig.provisionCalls).toEqual([]); + }); +}); diff --git a/src/core/warm-pool/converger-targets-creation.test.ts b/src/core/warm-pool/converger-targets-creation.test.ts new file mode 100644 index 000000000..8d675a3e0 --- /dev/null +++ b/src/core/warm-pool/converger-targets-creation.test.ts @@ -0,0 +1,582 @@ +import { describe, expect, it } from "vitest"; +import type { VirtualDevice, WaitingDemand } from "../domain.js"; +import { WARM_POOL_TICK_MS } from "./converger.js"; +import { device, harness, minute, neverLeased, now, waiter } from "./test-fixtures.js"; +import { iphone17, kind, ofKind, held, flush } from "./test-converger.js"; + +// why: the warm pool is a decomposed module (ADR 0063 section 2); these tests drive its converger part directly. +describe("warm pool targets", () => { + it("with a target and no device, makes one creation under an ownerless boot claim and a provisioning reservation, and releases the claim once the device is ready", async () => { + const rig = harness([], { targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toHaveLength(1); + expect(rig.provisionCalls[0]?.spec).toEqual(kind); + expect(rig.provisionCalls[0]?.claim).toEqual({ kind: "boot" }); + expect(rig.provisionReservations).toHaveLength(1); + expect(rig.claims.isClaimed("new-1")).toBe(false); + expect(rig.state.devices.map((item) => item.state)).toEqual(["ready"]); + expect(rig.kick).toHaveBeenCalled(); + }); + + it("runs a waiting-request creation under an ownerless boot claim and a provisioning reservation, releases the claim once ready, kicks, and on a throw logs once and holds the spec back a minute", async () => { + const behind = { ...kind, model: "iPhone 16" }; + const waiting = [waiter({ ...kind, model: "iPhone 15" }, true), waiter(behind)]; + const ok = harness([], { maxConcurrentBoots: 2, waiting: () => waiting }); + + await ok.pool.pass(); + await ok.pool.settle(); + + expect(ok.provisionCalls).toHaveLength(1); + expect(ok.provisionCalls[0]?.spec).toEqual(behind); + expect(ok.provisionCalls[0]?.claim).toEqual({ kind: "boot" }); + expect(ok.provisionReservations).toHaveLength(1); + expect(ok.claims.isClaimed("new-1")).toBe(false); + expect(ok.kick).toHaveBeenCalled(); + + const failing = harness([], { + maxConcurrentBoots: 2, + provision: async () => { + throw new Error("the runtime would not boot"); + }, + waiting: () => waiting, + }); + await failing.pool.pass(); + await failing.pool.settle(); + failing.clock.advance(WARM_POOL_TICK_MS); + await failing.pool.settle(); + + expect(failing.provisionCalls).toHaveLength(1); + const warnings = failing.sink.records.filter( + (record) => record.message === "warm pool creation of a device failed", + ); + expect(warnings).toHaveLength(1); + failing.clock.advance(minute); + await failing.pool.settle(); + expect(failing.provisionCalls).toHaveLength(2); + }); + + it("boots a shut-down device of the target's kind before creating one, and counts the target filled", async () => { + const rig = harness([ofKind("shut", "shutdown")], { targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.bootCalls).toEqual(["shut"]); + expect(rig.provisionCalls).toEqual([]); + expect(rig.reservations[0]?.claimedAtBoot).toBe(true); + expect(rig.claims.isClaimed("shut")).toBe(false); + }); + + it("creates nothing when the target's count of devices is ready, and never shuts a device down for it", async () => { + const rig = harness([ofKind("a", "ready"), ofKind("b", "ready")], { + limit: 2, + targets: [{ ...iphone17, count: 2 }], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.shutdownCalls).toEqual([]); + expect(rig.bootCalls).toEqual([]); + }); + + it("shuts a never-leased device no target counts down with the warm-pool initiator once it has been ready past idle.shutdownAfterMs", async () => { + const rig = harness([{ ...ofKind("extra", "ready"), readyAt: now - 11 * minute }]); + + await rig.pool.pass(); + + expect(rig.shutdownCalls).toEqual(["extra"]); + expect(rig.shutdownArgs).toEqual([["warm-pool", "cleanup"]]); + }); + + it.each([ + { + kind: "pool off", + devices: [device("refused", "ready", 5_000, "android"), device("listed", "ready", 1_000)], + options: { enabled: false, refusedPlatforms: ["android"] as const }, + expected: ["listed"], + }, + { + kind: "over budget", + // why: the capacity fake counts every device including the refused one, so a limit of 1 + // leaves the budget over whether or not the pool can act on the refused device. + devices: [device("refused", "ready", 5_000, "android"), device("listed", "ready", 1_000)], + options: { limit: 1, refusedPlatforms: ["android"] as const }, + expected: ["listed"], + }, + { + kind: "never leased and idle", + devices: [neverLeased("refused", "android"), neverLeased("listed", "ios")], + options: { refusedPlatforms: ["android"] as const }, + expected: ["listed"], + }, + ])( + "proposes no shutdown for a refused platform's device, whatever the reason ($kind)", + async ({ devices, options, expected }) => { + const rig = harness(devices, options); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.shutdownCalls).toEqual(expected); + }, + ); + + it("runs the target creations one after the other with maxConcurrentBoots 1, the second starting only after the first is ready", async () => { + const first = held(); + const second = held(); + const gates = [first, second]; + const rig = harness([], { + provision: async (_spec, attempt) => gates[attempt - 1]?.gate, + targets: [{ ...iphone17, count: 2 }], + }); + + await rig.pool.pass(); + await flush(); + expect(rig.provisionCalls).toHaveLength(1); + + // A further pass while the first is still running starts no second one. + await rig.pool.pass(); + await flush(); + expect(rig.provisionCalls).toHaveLength(1); + + first.finish(); + await flush(); + expect(rig.provisionCalls).toHaveLength(2); + second.finish(); + await rig.pool.settle(); + expect(rig.state.devices.map((item) => item.state)).toEqual(["ready", "ready"]); + }); + + it("runs two creations at once with maxConcurrentBoots 2 and not a third", async () => { + const gate = held(); + const rig = harness([], { + maxConcurrentBoots: 2, + provision: async () => gate.gate, + targets: [{ ...iphone17, count: 3 }], + }); + + await rig.pool.pass(); + await flush(); + + expect(rig.provisionCalls).toHaveLength(2); + gate.finish(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(3); + }); + + it("runs waiting-request boots beside the pass, three at once under maxConcurrentBoots 3, and counts them in the next pass's inFlight", async () => { + const gate = held(); + const shutDown = ["a", "b", "c", "d"].map((id) => device(id, "shutdown", 20 * minute)); + const rig = harness(shutDown, { + boot: async (target) => { + await gate.gate; + const ready = { ...target, state: "ready" as const }; + rig.state.devices = rig.state.devices.map((item) => (item.id === target.id ? ready : item)); + return ready; + }, + maxConcurrentBoots: 3, + waiting: () => shutDown.map((item) => waiter(item.spec)), + }); + + const first = rig.pool.pass(); + await flush(); + expect(rig.bootCalls).toEqual(["a", "b", "c"]); + + const second = rig.pool.pass(); + await flush(); + expect(rig.bootCalls).toEqual(["a", "b", "c"]); + + gate.finish(); + await Promise.all([first, second]); + await rig.pool.settle(); + expect(rig.bootCalls).toContain("d"); + }); + + it("builds onItsWay with one entry per on-its-way device and one per creation flight not yet registered, never two for one creation and never one short", async () => { + const x = device("shut-x", "shutdown", 20 * minute).spec; + const head = waiter({ ...kind, model: "iPad Pro" }, true); + const x2 = waiter(x); + const x3 = waiter(x); + const x4 = waiter(x); + const run = async ( + devices: readonly VirtualDevice[], + first: readonly WaitingDemand[], + provisioning?: Promise, + registers = false, + owner?: string, + ) => { + let waiting: readonly WaitingDemand[] = first; + const gate = held(); + const rig = harness(devices, { + boot: async () => gate.gate.then(() => undefined), + maxConcurrentBoots: 10, + provision: async () => gate.gate, + ...(provisioning === undefined ? {} : { provisioning }), + waiting: () => waiting, + }); + const counts = () => [rig.bootCalls.length, rig.provisionCalls.length]; + if (owner !== undefined) rig.claims.tryClaim("shut-x", "boot", owner); + void rig.pool.pass(); + await flush(); + const started = counts(); + if (registers) { + gate.finish(); + await flush(); + } + void rig.pool.pass(); + await flush(); + const again = counts(); + waiting = [...first, x4]; + void rig.pool.pass(); + await flush(); + return { again, later: counts(), started }; + }; + + const unregistered = await run([], [head, x2]); + expect(unregistered).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); + + const registered = await run([], [head, x2], new Promise(() => undefined), true); + expect(registered).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); + + const bootAndCreation = await run([device("shut-x", "shutdown", 20 * minute)], [head, x2, x3]); + expect(bootAndCreation).toEqual({ again: [1, 1], later: [1, 2], started: [1, 1] }); + + const owned = await run( + [device("shut-x", "shutdown", 20 * minute)], + [head, x2], + undefined, + false, + "an-agent", + ); + expect(owned).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); + }); + + it("does not return from a pass for a creation it started, and settle waits for it", async () => { + const gate = held(); + const rig = harness([], { provision: async () => gate.gate, targets: [iphone17] }); + + await rig.pool.pass(); + let settled = false; + const settling = rig.pool.settle().then(() => { + settled = true; + }); + await flush(); + + expect(rig.provisionCalls).toHaveLength(1); + expect(settled).toBe(false); + gate.finish(); + await settling; + expect(settled).toBe(true); + }); + + it("makes one creation attempt per allowed attempt when it fails: none on the passes between, one a minute after the first, the next two minutes after that", async () => { + const rig = harness([], { + provision: async () => { + throw new Error("the runtime would not boot"); + }, + targets: [iphone17], + }); + rig.pool.start(); + + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + + // Passes triggered by events and ticks within the minute change nothing. + rig.eventBus.emit("device.shutdown", { deviceId: "d", initiator: "test" }, "test"); + await rig.pool.settle(); + rig.clock.advance(WARM_POOL_TICK_MS); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + + rig.clock.advance(minute - WARM_POOL_TICK_MS); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(2); + + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(2); + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(3); + rig.pool.dispose(); + }); + + it("arms a timer for the next allowed attempt, so a pass runs when it is due with no event and no tick", async () => { + const rig = harness([], { + provision: async () => { + throw new Error("the runtime would not boot"); + }, + targets: [iphone17], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + + // `start()` was never called: no tick and no subscription, only the retry timer is armed. + rig.clock.advance(minute); + await rig.pool.settle(); + + expect(rig.provisionCalls).toHaveLength(2); + }); + + it("reports the target short with boot-failed between tries, and logs it once", async () => { + const rig = harness([], { + provision: async () => { + throw new Error("the runtime would not boot"); + }, + targets: [iphone17], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + await rig.pool.pass(); + await rig.pool.pass(); + + expect(rig.pool.targets()).toEqual([ + expect.objectContaining({ count: 1, ready: 0, short: "boot-failed" }), + ]); + const shortLines = rig.sink.records.filter( + (record) => record.message === "a warm pool target is short", + ); + expect(shortLines).toHaveLength(1); + expect(shortLines[0]?.fields).toMatchObject({ + short: "boot-failed", + target: "ios iPhone 17 26.0", + }); + const failure = rig.sink.records.filter( + (record) => record.message === "warm pool creation of a device failed", + ); + expect(failure).toHaveLength(1); + expect(failure[0]?.fields).toMatchObject({ model: "iPhone 17", step: "provision" }); + expect(String(failure[0]?.fields?.["error"])).toContain("the runtime would not boot"); + }); + + it("holds a failed boot of a shut-down device of the kind back for a minute, then boots it again", async () => { + const rig = harness([ofKind("shut", "shutdown")], { + boot: async () => { + throw new Error("simulator did not boot"); + }, + targets: [iphone17], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.bootCalls).toEqual(["shut"]); + + rig.clock.advance(minute - 1); + await rig.pool.settle(); + expect(rig.bootCalls).toEqual(["shut"]); + rig.clock.advance(1); + await rig.pool.settle(); + expect(rig.bootCalls).toEqual(["shut", "shut"]); + }); + + it("clears the schedule when a creation succeeds, so a later failure waits one minute again", async () => { + let failing = true; + const rig = harness([], { + provision: async () => { + if (failing) throw new Error("no luck"); + }, + targets: [iphone17], + }); + await rig.pool.pass(); + await rig.pool.settle(); + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(2); + failing = false; + rig.clock.advance(2 * minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(3); + expect(rig.state.devices).toHaveLength(1); + + // The device is leased: the target is short again, and now fails once more. + failing = true; + rig.lease("new-3"); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(4); + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(5); + }); + + it("reports a target naming a runtime that is not installed as runtime-missing and creates nothing", async () => { + const rig = harness([], { + resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), + targets: [{ ...iphone17, osVersion: "27.0" }], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.pool.targets()).toEqual([ + { + booting: 0, + configuredCount: 1, + count: 1, + kind: { + mode: "full", + model: "iPhone 17", + osVersion: "27.0", + platform: "ios", + }, + message: "iOS 27.0 is not installed", + origin: "configured", + ready: 0, + short: "runtime-missing", + target: "ios iPhone 17 27.0", + }, + ]); + }); + + it.each([ + ["no-driver", "there is no android driver"], + ["unknown-model", "Unknown ios model: iPhone 99"], + ["unresolvable", "the simulator service is down"], + ] as const)( + "reports a target the resolver answers %s for, with its message", + async (refusal, message) => { + const rig = harness([], { resolve: () => ({ message, refusal }), targets: [iphone17] }); + + await rig.pool.pass(); + + expect(rig.pool.targets()).toEqual([expect.objectContaining({ message, short: refusal })]); + expect(rig.provisionCalls).toEqual([]); + }, + ); + + it("resolves every target again each pass, so a runtime installed since takes effect without a restart", async () => { + let installed = false; + const rig = harness([], { + resolve: () => + installed + ? { spec: kind } + : { message: "not installed", refusal: "runtime-missing" as const }, + targets: [iphone17], + }); + await rig.pool.pass(); + expect(rig.provisionCalls).toEqual([]); + + installed = true; + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.resolveCalls.length).toBeGreaterThanOrEqual(2); + expect(rig.provisionCalls).toHaveLength(1); + }); + + it("resolves a target as the request it is: platform, exact model, and the OS and mode it names", async () => { + const rig = harness([], { + targets: [ + { + count: 1, + mode: "slim", + model: "iPhone 16", + osVersion: ">=18", + platform: "ios", + }, + { count: 1, model: "Pixel 9", platform: "android" }, + ], + }); + + await rig.pool.pass(); + + expect(rig.resolveCalls).toStrictEqual([ + { mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, + { model: "Pixel 9", platform: "android" }, + ]); + }); + + it("creates two devices for two targets resolving to one spec, one after the other, as one target of the summed count", async () => { + const first = held(); + const second = held(); + const rig = harness([], { + provision: async (_spec, attempt) => [first, second][attempt - 1]?.gate, + resolve: () => ({ spec: kind }), + targets: [iphone17, { ...iphone17, osVersion: ">=26" }], + }); + + await rig.pool.pass(); + await flush(); + expect(rig.provisionCalls).toHaveLength(1); + first.finish(); + await flush(); + expect(rig.provisionCalls).toHaveLength(2); + second.finish(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toHaveLength(2); + expect(rig.pool.targets()).toHaveLength(1); + expect(rig.pool.targets()[0]).toMatchObject({ count: 2, ready: 2 }); + }); + + it("does nothing for a target while an operator reset holds acquisition closed", async () => { + const rig = harness([], { maintenance: () => true, targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.resolveCalls).toEqual([]); + }); + + it("creates nothing for a target the capacity strategy refuses, and reports why", async () => { + const rig = harness([], { refuseProvision: true, targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "device-limit" })]); + }); + + it("creates nothing, and reports no failure, when the budget refuses the reservation after the policy allowed the creation", async () => { + const rig = harness([], { refuseReservation: true, targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.sink.records.filter((record) => record.level === "error")).toEqual([]); + // A declined creation asks for no pass of its own: the same view would propose it again. + expect(rig.resolveCalls).toHaveLength(1); + // Not a failure: the target is not held back, and the next pass tries it again. + await rig.pool.pass(); + expect(rig.resolveCalls).toHaveLength(2); + expect(rig.pool.targets()[0]?.short).toBeUndefined(); + }); + + it("boots nothing, and asks for no pass of its own, when the device's boot claim is refused", async () => { + const rig = harness([ofKind("shut", "shutdown")], { refuseClaim: true, targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.bootCalls).toEqual([]); + expect(rig.resolveCalls).toHaveLength(1); + expect(rig.reservations.every((reservation) => reservation.released === 1)).toBe(true); + }); + + it("asks for no pass of its own when the lifecycle declines a target's boot", async () => { + const rig = harness([ofKind("shut", "shutdown")], { + boot: async () => undefined, + targets: [iphone17], + }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.bootCalls).toEqual(["shut"]); + expect(rig.resolveCalls).toHaveLength(1); + expect(rig.kick).not.toHaveBeenCalled(); + }); +}); diff --git a/src/core/warm-pool/converger-targets-passes.test.ts b/src/core/warm-pool/converger-targets-passes.test.ts new file mode 100644 index 000000000..fc969a12f --- /dev/null +++ b/src/core/warm-pool/converger-targets-passes.test.ts @@ -0,0 +1,340 @@ +import { describe, expect, it } from "vitest"; +import type { TargetResolution } from "../domain.js"; +import { device, harness, minute, now } from "./test-fixtures.js"; +import { iphone17, kind, ofKind, held, flush } from "./test-converger.js"; + +describe("warm pool targets", () => { + it("reports and logs nothing before its first pass", () => { + const rig = harness([ofKind("ready", "ready")], { targets: [iphone17] }); + + expect(rig.pool.targets()).toEqual([]); + expect(rig.sink.records).toEqual([]); + }); + + it("resolves the targets itself when asked which devices they keep before any pass has run", async () => { + const rig = harness([ofKind("ready", "ready")], { targets: [iphone17] }); + + expect([...(await rig.pool.targeted())]).toEqual(["ready"]); + expect(rig.resolveCalls).toHaveLength(1); + // Resolved once: a later ask reads what is there, and a pass resolves again on its own. + await rig.pool.targeted(); + expect(rig.resolveCalls).toHaveLength(1); + }); + + it("stops acting on the rest of a pass's proposals once a drain begins", async () => { + let rig: ReturnType | undefined; + rig = harness( + [ + device("a", "ready", 90 * minute), + device("b", "ready", 80 * minute), + device("c", "ready", 70 * minute), + ], + { + limit: 1, + shutdown: async (target) => { + void rig?.pool.drain(); + return { ...target, state: "shutdown" }; + }, + }, + ); + + await rig.pool.pass(); + + expect(rig.shutdownCalls).toEqual(["a"]); + }); + + it("resolves no target, reports none and names no device with warmPool disabled", async () => { + const rig = harness([ofKind("ready", "ready")], { + enabled: false, + resolve: () => ({ message: "not installed", refusal: "runtime-missing" }), + targets: [iphone17], + }); + + await rig.pool.pass(); + + expect(rig.pool.targets()).toEqual([]); + expect([...(await rig.pool.targeted())]).toEqual([]); + expect(rig.resolveCalls).toEqual([]); + expect( + rig.sink.records.filter((record) => record.message === "a warm pool target is short"), + ).toEqual([]); + }); + + it("keeps a target's last resolved spec when one pass cannot resolve it, and drops it for a settled refusal", async () => { + let answer: TargetResolution = { spec: kind }; + const stale = { ...ofKind("stale", "ready"), readyAt: now - 11 * minute }; + const rig = harness([stale], { resolve: () => answer, targets: [{ ...iphone17, count: 2 }] }); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.shutdownCalls).toEqual([]); + expect(rig.provisionCalls).toHaveLength(1); + rig.lease("new-1"); + + // The target is one short, but the spec it may no longer have plans no creation for it. + answer = { message: "simctl timed out", refusal: "unresolvable" }; + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + expect(rig.shutdownCalls).toEqual([]); + expect([...(await rig.pool.targeted())]).toEqual(["stale"]); + expect(rig.pool.targets()).toEqual([ + expect.objectContaining({ message: "simctl timed out", ready: 1, short: "unresolvable" }), + ]); + const line = rig.sink.records.filter( + (record) => record.message === "a warm pool target is short", + ); + expect(line[0]?.fields).toMatchObject({ ready: 1, short: "unresolvable" }); + + answer = { message: "gone", refusal: "runtime-missing" }; + await rig.pool.pass(); + expect(rig.shutdownCalls).toEqual(["stale"]); + expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "runtime-missing" })]); + + // The spec it once had is gone with a settled refusal: a later failed read has none to keep. + answer = { message: "simctl timed out", refusal: "unresolvable" }; + rig.state.devices = rig.state.devices.map((item) => + item.id === "stale" ? { ...item, state: "ready" as const } : item, + ); + await rig.pool.pass(); + expect(rig.pool.targets()).toEqual([ + expect.objectContaining({ ready: 0, short: "unresolvable" }), + ]); + expect([...(await rig.pool.targeted())]).toEqual([]); + }); + + it("starts nothing once closed, before it drains", async () => { + const rig = harness([], { targets: [iphone17] }); + + rig.pool.close(); + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.provisionCalls).toEqual([]); + expect(rig.resolveCalls).toEqual([]); + }); + + it("does not count a claimed device as ready in the report of a target it cannot resolve", async () => { + let answer: TargetResolution = { spec: kind }; + const rig = harness([ofKind("busy", "ready")], { resolve: () => answer, targets: [iphone17] }); + await rig.pool.pass(); + expect(rig.pool.targets()).toEqual([expect.objectContaining({ ready: 1 })]); + + rig.claims.tryClaim("busy", "eviction"); + answer = { message: "simctl timed out", refusal: "unresolvable" }; + await rig.pool.pass(); + + expect(rig.pool.targets()).toEqual([ + expect.objectContaining({ ready: 0, short: "unresolvable" }), + ]); + }); + + it("starts nothing new once draining, though a creation that was running ends", async () => { + const gate = held(); + const rig = harness([], { + provision: async () => gate.gate, + targets: [{ ...iphone17, count: 3 }], + }); + await rig.pool.pass(); + await flush(); + expect(rig.provisionCalls).toHaveLength(1); + + const draining = rig.pool.drain(); + gate.finish(); + await draining; + const resolved = rig.resolveCalls.length; + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.resolveCalls).toHaveLength(resolved); + + expect(rig.provisionCalls).toHaveLength(1); + expect(rig.state.devices).toHaveLength(1); + expect(rig.clock.pendingTimerCount).toBe(0); + }); + + it("logs a boot or creation that throws outside the driver, holds the target back and asks for no immediate retry", async () => { + const rig = harness([], { reserveThrows: true, targets: [iphone17] }); + + await rig.pool.pass(); + // Bounded by turns, not by settle: a pool that retried at once would never settle. + await flush(); + + expect(rig.reserveAttempts.count).toBe(1); + const errors = rig.sink.records.filter((record) => record.level === "error"); + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ + fields: { step: "flight" }, + message: "a warm pool boot or creation failed", + }); + expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "boot-failed" })]); + expect(rig.clock.pendingTimerCount).toBe(1); + }); + + it("reports ram-budget and boots nothing when the capacity strategy refuses the shut-down device's boot", async () => { + const rig = harness([ofKind("shut", "shutdown")], { refuseBoot: true, targets: [iphone17] }); + + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.bootCalls).toEqual([]); + expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "ram-budget" })]); + }); + + it("counts a creation still running toward its target, so a second pass starts no second one even with room for two boots", async () => { + const gate = held(); + const rig = harness([], { + maxConcurrentBoots: 2, + provision: async () => gate.gate, + targets: [iphone17], + }); + + await rig.pool.pass(); + await flush(); + await rig.pool.pass(); + await flush(); + + expect(rig.provisionCalls).toHaveLength(1); + expect(rig.sink.records.filter((record) => record.level === "error")).toEqual([]); + gate.finish(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + }); + + it("asks for no pass and arms no timer once disposed, for a creation that ends afterwards", async () => { + const gate = held(); + const rig = harness([], { provision: async () => gate.gate, targets: [iphone17] }); + await rig.pool.pass(); + await flush(); + + rig.pool.dispose(); + gate.fail(new Error("the runtime would not boot")); + await flush(); + + expect(rig.resolveCalls).toHaveLength(1); + expect(rig.clock.pendingTimerCount).toBe(0); + }); + + it("cancels the retry timer on dispose, and arms none when a pass that was running ends after it", async () => { + let hold: Promise | undefined; + const rig = harness([], { + provision: async () => { + throw new Error("no luck"); + }, + resolve: async (request) => { + await hold; + return { + spec: { + physical: false, + model: request.model ?? "", + osVersion: "26.0", + platform: request.platform, + }, + }; + }, + targets: [iphone17], + }); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.clock.pendingTimerCount).toBe(1); + + let release: () => void = () => undefined; + hold = new Promise((resolve) => { + release = resolve; + }); + const running = rig.pool.pass(); + rig.pool.dispose(); + expect(rig.clock.pendingTimerCount).toBe(0); + release(); + await running; + + expect(rig.clock.pendingTimerCount).toBe(0); + }); + + it("keeps one retry timer however many passes run while a target is held back", async () => { + const rig = harness([], { + provision: async () => { + throw new Error("no luck"); + }, + targets: [iphone17], + }); + await rig.pool.pass(); + await rig.pool.settle(); + + await rig.pool.pass(); + await rig.pool.pass(); + await rig.pool.settle(); + + expect(rig.clock.pendingTimerCount).toBe(1); + }); + + it("arms the timer for a second held-back target when the first one's retry succeeds, so no tick is needed", async () => { + let rig: ReturnType | undefined; + rig = harness([], { + provision: async (spec, attempt) => { + // Attempt 1 (iPhone 17) fails at once, attempt 2 (iPhone 16) fails ten seconds later, + // attempt 3 (iPhone 17 again) succeeds, attempt 4 (iPhone 16 again) is the one under test. + if (attempt === 2) rig?.clock.advance(10_000); + if (attempt <= 2) throw new Error(`no luck for ${spec.model}`); + }, + targets: [iphone17, { ...iphone17, model: "iPhone 16" }], + }); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual(["iPhone 17", "iPhone 16"]); + + // iPhone 17 may be tried at 60s and succeeds; iPhone 16 failed ten seconds later. + rig.clock.advance(50_000); + await rig.pool.settle(); + expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual([ + "iPhone 17", + "iPhone 16", + "iPhone 17", + ]); + + rig.clock.advance(10_000); + await rig.pool.settle(); + expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual([ + "iPhone 17", + "iPhone 16", + "iPhone 17", + "iPhone 16", + ]); + }); + + it("clears the schedule when a boot of a shut-down device succeeds, so a later failure waits one minute again", async () => { + let bootAttempts = 0; + let creating = false; + let rig: ReturnType | undefined; + rig = harness([ofKind("shut", "shutdown")], { + boot: async (target) => { + bootAttempts += 1; + if (bootAttempts === 1) throw new Error("simulator did not boot"); + const ready = { ...target, state: "ready" as const }; + if (rig !== undefined) rig.state.devices = [ready]; + return ready; + }, + provision: async () => { + if (creating) throw new Error("no luck"); + }, + targets: [iphone17], + }); + await rig.pool.pass(); + await rig.pool.settle(); + expect(bootAttempts).toBe(1); + + // The boot is retried after a minute and succeeds. + rig.clock.advance(minute); + await rig.pool.settle(); + expect(bootAttempts).toBe(2); + + // The device is then leased, the target is short again, and a creation fails: one minute + // later it is tried again, not two. + creating = true; + rig.lease("shut"); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(1); + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.provisionCalls).toHaveLength(2); + }); +}); diff --git a/src/core/warm-pool/converger-targets-reports.test.ts b/src/core/warm-pool/converger-targets-reports.test.ts new file mode 100644 index 000000000..c4ad83205 --- /dev/null +++ b/src/core/warm-pool/converger-targets-reports.test.ts @@ -0,0 +1,433 @@ +import { describe, expect, it } from "vitest"; +import { LearnedKinds } from "./learned.js"; +import { harness, learnedState, minute, now } from "./test-fixtures.js"; +import { iphone17, ofKind, held } from "./test-converger.js"; + +// why: the warm pool is a decomposed module (ADR 0063 section 2); these tests drive its converger part directly. +describe("warm pool targets", () => { + describe("figures", () => { + const wanted = { + mode: "full", + model: "iPhone 17", + origin: "configured", + osVersion: "26.0", + platform: "ios", + } as const; + + it("lists a target with its kind, count, ready and booting, and no reason while it fills", async () => { + let finish: () => void = () => undefined; + const rig = harness([ofKind("ready", "ready")], { + limit: 5, + provision: () => + new Promise((resolve) => { + finish = resolve; + }), + targets: [{ ...iphone17, count: 2 }], + }); + + await rig.pool.pass(); + + expect(rig.pool.figures()).toStrictEqual({ + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + targets: [{ ...wanted, booting: 1, count: 2, ready: 1 }], + }); + finish(); + await rig.pool.settle(); + }); + + it("lists a target the resolver refuses with the kind it was configured with and the refusal as short", async () => { + const rig = harness([], { + resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), + targets: [{ ...iphone17, osVersion: "27.0" }], + }); + + await rig.pool.pass(); + + expect(rig.pool.figures().targets).toStrictEqual([ + { ...wanted, booting: 0, count: 1, osVersion: "27.0", ready: 0, short: "runtime-missing" }, + ]); + }); + + it("lists every configured target as short disabled with the pool off, before and after a pass", async () => { + const rig = harness([ofKind("ready", "ready")], { + enabled: false, + targets: [iphone17, { ...iphone17, mode: "slim", model: "iPhone 16" }], + }); + const expected = { + enabled: false, + reserveRunning: { android: 0, ios: 0 }, + targets: [ + { ...wanted, booting: 0, count: 1, ready: 0, short: "disabled" }, + { + ...wanted, + booting: 0, + count: 1, + mode: "slim", + model: "iPhone 16", + ready: 0, + short: "disabled", + }, + ], + }; + + expect(rig.pool.figures()).toStrictEqual(expected); + await rig.pool.pass(); + expect(rig.pool.figures()).toStrictEqual(expected); + }); + + it("lists every configured target before the first pass with no reason, so a status read just after a start is not empty", () => { + const rig = harness([], { targets: [iphone17] }); + + expect(rig.pool.figures()).toStrictEqual({ + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + targets: [{ ...wanted, booting: 0, count: 1, ready: 0 }], + }); + }); + + it("names the platform's default mode for a target that names none, before a pass, with the pool off, refused and resolved", async () => { + const { osVersion: _osVersion, ...unversioned } = iphone17; + const modes = async (options: { enabled?: boolean; refuse?: boolean; pass: boolean }) => { + const rig = harness([], { + defaultMode: "slim", + ...(options.enabled === undefined ? {} : { enabled: options.enabled }), + ...(options.refuse === true + ? { resolve: () => ({ message: "why", refusal: "no-driver" as const }) } + : {}), + targets: [unversioned], + }); + if (options.pass) await rig.pool.pass(); + return rig.pool.figures().targets.map((target) => target.mode); + }; + + expect(await modes({ pass: false })).toEqual(["slim"]); + expect(await modes({ enabled: false, pass: true })).toEqual(["slim"]); + expect(await modes({ pass: true, refuse: true })).toEqual(["slim"]); + expect(await modes({ pass: true })).toEqual(["slim"]); + }); + + it("counts only the boots of the spec a target keeps while its resolution is refused unresolvable, and none for one that never resolved", async () => { + const creation = held(); + let refuse17 = false; + const rig = harness([], { + limit: 5, + maxConcurrentBoots: 2, + provision: () => creation.gate, + resolve: (request) => { + if (request.model === "iPhone 15") return { message: "why", refusal: "no-driver" }; + if (request.model === "iPhone 17" && refuse17) { + return { message: "why", refusal: "unresolvable" }; + } + return { + spec: { + physical: false, + model: request.model ?? "", + osVersion: "26.0", + platform: request.platform, + }, + }; + }, + targets: [ + iphone17, + { ...iphone17, model: "iPhone 16" }, + { ...iphone17, model: "iPhone 15" }, + ], + }); + const figures = () => + Object.fromEntries(rig.pool.figures().targets.map((target) => [target.model, target])); + await rig.pool.pass(); + expect(figures()["iPhone 17"]?.booting).toBe(1); + + refuse17 = true; + await rig.pool.pass(); + + expect(figures()["iPhone 17"]).toMatchObject({ booting: 1, short: "unresolvable" }); + expect(figures()["iPhone 16"]).toMatchObject({ booting: 1 }); + expect(figures()["iPhone 15"]).toMatchObject({ booting: 0, short: "no-driver" }); + creation.finish(); + await rig.pool.settle(); + }); + + it("leaves out osVersion for a refused target that names none", async () => { + const { osVersion: _osVersion, ...unversioned } = iphone17; + const rig = harness([], { + resolve: () => ({ message: "no driver", refusal: "no-driver" }), + targets: [unversioned], + }); + + await rig.pool.pass(); + + expect(rig.pool.figures().targets[0]).toStrictEqual({ + booting: 0, + count: 1, + mode: "full", + model: "iPhone 17", + origin: "configured", + platform: "ios", + ready: 0, + short: "no-driver", + }); + }); + }); + + describe("warm-pool.target-missed", () => { + const missed = (rig: ReturnType): unknown[] => { + const payloads: unknown[] = []; + rig.eventBus.subscribe("warm-pool.target-missed", (envelope) => { + payloads.push(envelope.payload); + }); + return payloads; + }; + + it("fires once when a target first ends a pass short, with its kind, count, ready and reason", async () => { + const rig = harness([], { + resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), + targets: [{ ...iphone17, osVersion: "27.0" }], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + + expect(payloads).toStrictEqual([ + { + count: 1, + mode: "full", + model: "iPhone 17", + origin: "configured", + osVersion: "27.0", + platform: "ios", + ready: 0, + reason: "runtime-missing", + }, + ]); + }); + + it("does not fire again on the next pass that leaves the target short", async () => { + const rig = harness([], { + resolve: () => ({ message: "why", refusal: "runtime-missing" }), + targets: [iphone17], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + await rig.pool.pass(); + await rig.pool.pass(); + + expect(payloads).toHaveLength(1); + }); + + it("names the pool as the emitter, and leaves osVersion out for a target that names none", async () => { + const { osVersion: _osVersion, ...unversioned } = iphone17; + const rig = harness([], { + resolve: () => ({ message: "no driver", refusal: "no-driver" }), + targets: [unversioned], + }); + const seen: { module: string; payload: unknown }[] = []; + rig.eventBus.subscribe("warm-pool.target-missed", (envelope) => { + seen.push({ module: envelope.module, payload: envelope.payload }); + }); + + await rig.pool.pass(); + + expect(seen).toStrictEqual([ + { + module: "warm-pool", + payload: { + count: 1, + mode: "full", + model: "iPhone 17", + origin: "configured", + platform: "ios", + ready: 0, + reason: "no-driver", + }, + }, + ]); + }); + + it("fires again when the target was met in between, ready reaching its count, and is missed again", async () => { + let refuse = true; + const rig = harness([ofKind("ready", "ready")], { + resolve: (request) => + refuse + ? { message: "why", refusal: "runtime-missing" } + : { + spec: { + physical: false, + model: request.model ?? "", + osVersion: "26.0", + platform: request.platform, + }, + }, + targets: [iphone17], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + refuse = false; + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.pool.figures().targets[0]).toMatchObject({ count: 1, ready: 1 }); + refuse = true; + await rig.pool.pass(); + + expect(payloads).toHaveLength(2); + }); + + it("does not fire again for a target that is only not short on a pass, filling or retrying, until it is met", async () => { + const rig = harness([], { + limit: 5, + provision: async () => { + throw new Error("no luck"); + }, + targets: [iphone17], + }); + const payloads = missed(rig); + + // The first creation fails and the target is short `boot-failed`; the retry a minute later + // is a pass that is not short (a creation runs), and it fails again. + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.pool.figures().targets[0]?.short).toBe("boot-failed"); + rig.clock.advance(minute); + await rig.pool.settle(); + rig.clock.advance(2 * minute); + await rig.pool.settle(); + + expect(rig.provisionCalls).toHaveLength(3); + expect(payloads).toStrictEqual([expect.objectContaining({ reason: "boot-failed" })]); + }); + + it("fires once for a target with no OS that is refused one pass and short the next, though its kind changes", async () => { + const { osVersion: _osVersion, ...unversioned } = iphone17; + let refuse = true; + const rig = harness([], { + limit: 0, + resolve: (request) => + refuse + ? { message: "why", refusal: "unresolvable" } + : { + spec: { + physical: false, + model: request.model ?? "", + osVersion: "26.0", + platform: request.platform, + }, + }, + targets: [unversioned], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + refuse = false; + await rig.pool.pass(); + + expect(rig.pool.figures().targets[0]).toMatchObject({ + osVersion: "26.0", + short: "running-limit", + }); + expect(payloads).toStrictEqual([expect.objectContaining({ reason: "unresolvable" })]); + }); + + it("fires once for two targets that resolve to one spec, and on no later pass", async () => { + const rig = harness([], { + limit: 0, + targets: [iphone17, { ...iphone17, count: 2 }], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + await rig.pool.pass(); + + expect(payloads).toStrictEqual([ + expect.objectContaining({ count: 3, reason: "running-limit" }), + ]); + }); + + it("fires for each of two configured targets of one kind that are both refused, not once for the kind", async () => { + const rig = harness([], { + resolve: () => ({ message: "why", refusal: "runtime-missing" }), + targets: [iphone17, { ...iphone17, count: 2 }], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + await rig.pool.pass(); + + expect(payloads).toStrictEqual([ + expect.objectContaining({ count: 1, model: "iPhone 17", reason: "runtime-missing" }), + expect.objectContaining({ count: 2, model: "iPhone 17", reason: "runtime-missing" }), + ]); + }); + + it("keeps each configured target's edge apart: one met and one short, then the met one missed, fires for it too", async () => { + const rig = harness([ofKind("ready", "ready")], { + limit: 0, + targets: [iphone17, { ...iphone17, model: "iPhone 16" }], + }); + const payloads = missed(rig); + + await rig.pool.pass(); + rig.lease("ready"); + await rig.pool.pass(); + + expect(payloads.map((payload) => (payload as { model: string }).model)).toEqual([ + "iPhone 16", + "iPhone 17", + ]); + }); + + it("fires nothing for a target that is filling, and nothing with the pool off", async () => { + const filling = harness([], { limit: 5, targets: [iphone17] }); + const fillingPayloads = missed(filling); + await filling.pool.pass(); + await filling.pool.settle(); + const off = harness([], { enabled: false, targets: [iphone17] }); + const offPayloads = missed(off); + await off.pool.pass(); + + expect(fillingPayloads).toEqual([]); + expect(offPayloads).toEqual([]); + }); + }); + + describe("targeted", () => { + it("names, once a pass has resolved the targets, the ready unleased devices of the kind up to the count, oldest first", async () => { + const rig = harness( + [ofKind("newer", "ready", 20), ofKind("older", "ready", 10), ofKind("oldest", "ready", 5)], + { limit: 5, targets: [{ ...iphone17, count: 2 }] }, + ); + await rig.pool.pass(); + + expect([...(await rig.pool.targeted())].sort()).toEqual(["older", "oldest"]); + }); + + it("names a learned kind's ready devices up to its count before the first pass, and again for a pool started over the same record", async () => { + const spec = ofKind("x", "ready").spec; + new LearnedKinds(learnedState).learn(spec, "lease-1", now - minute); + new LearnedKinds(learnedState).learn(spec, "lease-2", now - minute); + const devices = [ofKind("newer", "ready", 20), ofKind("older", "ready", 10)]; + + const before = harness(devices, { limit: 5 }); + const restarted = harness(devices, { limit: 5 }); + const off = harness(devices, { enabled: false, limit: 5 }); + + expect([...(await before.pool.targeted())].sort()).toEqual(["newer", "older"]); + expect([...(await restarted.pool.targeted())].sort()).toEqual(["newer", "older"]); + expect([...(await off.pool.targeted())]).toEqual([]); + }); + + it("does not name a device that is leased or claimed", async () => { + const rig = harness([ofKind("leased", "ready", 1), ofKind("claimed", "ready", 2)], { + limit: 5, + targets: [{ ...iphone17, count: 2 }], + }); + await rig.pool.pass(); + rig.lease("leased"); + rig.claims.tryClaim("claimed", "boot"); + + expect([...(await rig.pool.targeted())]).toEqual([]); + }); + }); +}); diff --git a/src/core/warm-pool/converger.test.ts b/src/core/warm-pool/converger.test.ts index fb1eabeae..d79911778 100644 --- a/src/core/warm-pool/converger.test.ts +++ b/src/core/warm-pool/converger.test.ts @@ -1,21 +1,6 @@ import { describe, expect, it, vi } from "vitest"; - -import type { TargetResolution, VirtualDevice, WaitingDemand } from "../domain.js"; -import type { WarmTarget } from "./config.js"; import { WARM_POOL_TICK_MS } from "./converger.js"; -import { LearnedKinds } from "./learned.js"; -import { - device, - fakeMemoryPressure, - harness, - iphone15, - learnedState, - leaseOn, - minute, - neverLeased, - now, - waiter, -} from "./test-fixtures.js"; +import { device, harness, iphone15, leaseOn, minute } from "./test-fixtures.js"; // why: the warm pool is a decomposed module (ADR 0063 section 2); these tests drive its converger part directly. describe("warm pool converger", () => { @@ -536,1485 +521,3 @@ describe("warm pool converger", () => { expect(rig.waitingDemand.mock.calls.length).toBe(afterDispose); }); }); - -describe("warm pool memory pressure", () => { - const paused = (readingAt: number) => ({ - cause: { availablePercent: 12, level: "warn" as const, trigger: "level" as const }, - readingAt, - since: now, - step: "warming-paused" as const, - }); - - it("each reading while warming is paused shuts down one idle device", async () => { - const pressure = fakeMemoryPressure(paused(now)); - const rig = harness( - [ - device("old", "ready", 90 * minute), - device("mid", "ready", 60 * minute), - device("new", "ready", 5 * minute), - ], - { memoryPressure: pressure.reader }, - ); - rig.pool.start(); - - pressure.reading(paused(now + 15_000)); - await rig.pool.settle(); - expect(rig.shutdownCalls).toEqual(["old"]); - - pressure.reading(paused(now + 30_000)); - await rig.pool.settle(); - expect(rig.shutdownCalls).toEqual(["old", "mid"]); - - await rig.pool.pass(); - expect(rig.shutdownCalls, "a second pass on the same reading").toEqual(["old", "mid"]); - }); - - it("sheds nothing for a reading before start, at step none, with no step reading or nothing idle, and skips a device whose shutdown just failed", async () => { - const idle = () => [device("old", "ready", 90 * minute), device("mid", "ready", 60 * minute)]; - const none = { ...paused(now), step: "none" as const }; - const cases = [ - { name: "before start", reading: paused(now + 15_000), start: false }, - { name: "step none", reading: none }, - { name: "no step reading", reading: undefined }, - { - devices: [device("old", "leased", 90 * minute)], - name: "nothing idle", - reading: paused(now + 15_000), - }, - ]; - for (const { devices = idle(), name, reading, start = true } of cases) { - const pressure = fakeMemoryPressure(reading); - const rig = harness(devices, { memoryPressure: pressure.reader }); - if (start) rig.pool.start(); - pressure.reading(reading === undefined ? undefined : { ...reading, readingAt: now + 15_000 }); - await rig.pool.settle(); - expect(rig.shutdownCalls, name).toEqual([]); - } - - const pressure = fakeMemoryPressure(paused(now)); - const failing = harness(idle(), { - memoryPressure: pressure.reader, - shutdown: async (target) => { - if (target.id === "old") throw new Error("the simulator would not stop"); - return { ...target, state: "shutdown" }; - }, - }); - failing.pool.start(); - pressure.reading(paused(now + 15_000)); - await failing.pool.settle(); - pressure.reading(paused(now + 30_000)); - await failing.pool.settle(); - expect(failing.shutdownCalls, "the next reading takes the next device").toEqual(["old", "mid"]); - }); - - it("when warming resumes the pool fills its targets again on memory.pressure-changed, with no tick", async () => { - const pressure = fakeMemoryPressure(paused(now)); - const rig = harness([], { memoryPressure: pressure.reader, targets: [iphone15] }); - rig.pool.start(); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls, "nothing is created while paused").toEqual([]); - - pressure.reading({ readingAt: now + 15_000, step: "none" }); - rig.eventBus.emit( - "memory.pressure-changed", - { from: "warming-paused", to: "none" }, - "host-load", - ); - await rig.pool.settle(); - - expect(rig.provisionCalls).toHaveLength(1); - }); - - it("at warming-paused reports a short target memory-pressure, a target that does not resolve its own reason, and boots nothing", async () => { - const rig = harness([device("shut", "shutdown", 1_000)], { - memoryPressure: fakeMemoryPressure(paused(now)).reader, - resolve: (request) => - request.model === "iPhone 17" - ? { message: "iOS 27.0 is not installed", refusal: "runtime-missing" } - : { spec: { model: "iPhone 15", osVersion: "26.0", physical: false, platform: "ios" } }, - targets: [iphone15, { ...iphone15, model: "iPhone 17", osVersion: "27.0" }], - }); - const reasons: Record = {}; - rig.eventBus.subscribe("warm-pool.target-missed", ({ payload }) => { - reasons[payload.model] = payload.reason; - }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(reasons).toEqual({ "iPhone 15": "memory-pressure", "iPhone 17": "runtime-missing" }); - expect( - Object.fromEntries(rig.pool.figures().targets.map((target) => [target.model, target.short])), - ).toEqual({ "iPhone 15": "memory-pressure", "iPhone 17": "runtime-missing" }); - expect(rig.bootCalls).toEqual([]); - expect(rig.provisionCalls).toEqual([]); - }); -}); - -describe("warm pool targets", () => { - const iphone17: WarmTarget = { - count: 1, - model: "iPhone 17", - osVersion: "26.0", - platform: "ios", - }; - const kind = { model: "iPhone 17", osVersion: "26.0", physical: false, platform: "ios" } as const; - const ofKind = (id: string, state: VirtualDevice["state"], createdAt = 1): VirtualDevice => ({ - physical: false, - createdAt, - driverData: {}, - driverDeviceId: `driver-${id}`, - id, - mode: "full", - spec: kind, - state, - }); - /** A creation the test holds open until it calls `finish`, or fails with `fail`. */ - function held() { - let finish: () => void = () => undefined; - let fail: (error: Error) => void = () => undefined; - const gate = new Promise((resolve, reject) => { - finish = resolve; - fail = reject; - }); - return { fail, finish, gate }; - } - const flush = async (): Promise => { - for (let turn = 0; turn < 30; turn += 1) await Promise.resolve(); - }; - - it("with a target and no device, makes one creation under an ownerless boot claim and a provisioning reservation, and releases the claim once the device is ready", async () => { - const rig = harness([], { targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toHaveLength(1); - expect(rig.provisionCalls[0]?.spec).toEqual(kind); - expect(rig.provisionCalls[0]?.claim).toEqual({ kind: "boot" }); - expect(rig.provisionReservations).toHaveLength(1); - expect(rig.claims.isClaimed("new-1")).toBe(false); - expect(rig.state.devices.map((item) => item.state)).toEqual(["ready"]); - expect(rig.kick).toHaveBeenCalled(); - }); - - it("runs a waiting-request creation under an ownerless boot claim and a provisioning reservation, releases the claim once ready, kicks, and on a throw logs once and holds the spec back a minute", async () => { - const behind = { ...kind, model: "iPhone 16" }; - const waiting = [waiter({ ...kind, model: "iPhone 15" }, true), waiter(behind)]; - const ok = harness([], { maxConcurrentBoots: 2, waiting: () => waiting }); - - await ok.pool.pass(); - await ok.pool.settle(); - - expect(ok.provisionCalls).toHaveLength(1); - expect(ok.provisionCalls[0]?.spec).toEqual(behind); - expect(ok.provisionCalls[0]?.claim).toEqual({ kind: "boot" }); - expect(ok.provisionReservations).toHaveLength(1); - expect(ok.claims.isClaimed("new-1")).toBe(false); - expect(ok.kick).toHaveBeenCalled(); - - const failing = harness([], { - maxConcurrentBoots: 2, - provision: async () => { - throw new Error("the runtime would not boot"); - }, - waiting: () => waiting, - }); - await failing.pool.pass(); - await failing.pool.settle(); - failing.clock.advance(WARM_POOL_TICK_MS); - await failing.pool.settle(); - - expect(failing.provisionCalls).toHaveLength(1); - const warnings = failing.sink.records.filter( - (record) => record.message === "warm pool creation of a device failed", - ); - expect(warnings).toHaveLength(1); - failing.clock.advance(minute); - await failing.pool.settle(); - expect(failing.provisionCalls).toHaveLength(2); - }); - - it("boots a shut-down device of the target's kind before creating one, and counts the target filled", async () => { - const rig = harness([ofKind("shut", "shutdown")], { targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.bootCalls).toEqual(["shut"]); - expect(rig.provisionCalls).toEqual([]); - expect(rig.reservations[0]?.claimedAtBoot).toBe(true); - expect(rig.claims.isClaimed("shut")).toBe(false); - }); - - it("creates nothing when the target's count of devices is ready, and never shuts a device down for it", async () => { - const rig = harness([ofKind("a", "ready"), ofKind("b", "ready")], { - limit: 2, - targets: [{ ...iphone17, count: 2 }], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.shutdownCalls).toEqual([]); - expect(rig.bootCalls).toEqual([]); - }); - - it("shuts a never-leased device no target counts down with the warm-pool initiator once it has been ready past idle.shutdownAfterMs", async () => { - const rig = harness([{ ...ofKind("extra", "ready"), readyAt: now - 11 * minute }]); - - await rig.pool.pass(); - - expect(rig.shutdownCalls).toEqual(["extra"]); - expect(rig.shutdownArgs).toEqual([["warm-pool", "cleanup"]]); - }); - - it.each([ - { - kind: "pool off", - devices: [device("refused", "ready", 5_000, "android"), device("listed", "ready", 1_000)], - options: { enabled: false, refusedPlatforms: ["android"] as const }, - expected: ["listed"], - }, - { - kind: "over budget", - // why: the capacity fake counts every device including the refused one, so a limit of 1 - // leaves the budget over whether or not the pool can act on the refused device. - devices: [device("refused", "ready", 5_000, "android"), device("listed", "ready", 1_000)], - options: { limit: 1, refusedPlatforms: ["android"] as const }, - expected: ["listed"], - }, - { - kind: "never leased and idle", - devices: [neverLeased("refused", "android"), neverLeased("listed", "ios")], - options: { refusedPlatforms: ["android"] as const }, - expected: ["listed"], - }, - ])( - "proposes no shutdown for a refused platform's device, whatever the reason ($kind)", - async ({ devices, options, expected }) => { - const rig = harness(devices, options); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.shutdownCalls).toEqual(expected); - }, - ); - - it("runs the target creations one after the other with maxConcurrentBoots 1, the second starting only after the first is ready", async () => { - const first = held(); - const second = held(); - const gates = [first, second]; - const rig = harness([], { - provision: async (_spec, attempt) => gates[attempt - 1]?.gate, - targets: [{ ...iphone17, count: 2 }], - }); - - await rig.pool.pass(); - await flush(); - expect(rig.provisionCalls).toHaveLength(1); - - // A further pass while the first is still running starts no second one. - await rig.pool.pass(); - await flush(); - expect(rig.provisionCalls).toHaveLength(1); - - first.finish(); - await flush(); - expect(rig.provisionCalls).toHaveLength(2); - second.finish(); - await rig.pool.settle(); - expect(rig.state.devices.map((item) => item.state)).toEqual(["ready", "ready"]); - }); - - it("runs two creations at once with maxConcurrentBoots 2 and not a third", async () => { - const gate = held(); - const rig = harness([], { - maxConcurrentBoots: 2, - provision: async () => gate.gate, - targets: [{ ...iphone17, count: 3 }], - }); - - await rig.pool.pass(); - await flush(); - - expect(rig.provisionCalls).toHaveLength(2); - gate.finish(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(3); - }); - - it("runs waiting-request boots beside the pass, three at once under maxConcurrentBoots 3, and counts them in the next pass's inFlight", async () => { - const gate = held(); - const shutDown = ["a", "b", "c", "d"].map((id) => device(id, "shutdown", 20 * minute)); - const rig = harness(shutDown, { - boot: async (target) => { - await gate.gate; - const ready = { ...target, state: "ready" as const }; - rig.state.devices = rig.state.devices.map((item) => (item.id === target.id ? ready : item)); - return ready; - }, - maxConcurrentBoots: 3, - waiting: () => shutDown.map((item) => waiter(item.spec)), - }); - - const first = rig.pool.pass(); - await flush(); - expect(rig.bootCalls).toEqual(["a", "b", "c"]); - - const second = rig.pool.pass(); - await flush(); - expect(rig.bootCalls).toEqual(["a", "b", "c"]); - - gate.finish(); - await Promise.all([first, second]); - await rig.pool.settle(); - expect(rig.bootCalls).toContain("d"); - }); - - it("builds onItsWay with one entry per on-its-way device and one per creation flight not yet registered, never two for one creation and never one short", async () => { - const x = device("shut-x", "shutdown", 20 * minute).spec; - const head = waiter({ ...kind, model: "iPad Pro" }, true); - const x2 = waiter(x); - const x3 = waiter(x); - const x4 = waiter(x); - const run = async ( - devices: readonly VirtualDevice[], - first: readonly WaitingDemand[], - provisioning?: Promise, - registers = false, - owner?: string, - ) => { - let waiting: readonly WaitingDemand[] = first; - const gate = held(); - const rig = harness(devices, { - boot: async () => gate.gate.then(() => undefined), - maxConcurrentBoots: 10, - provision: async () => gate.gate, - ...(provisioning === undefined ? {} : { provisioning }), - waiting: () => waiting, - }); - const counts = () => [rig.bootCalls.length, rig.provisionCalls.length]; - if (owner !== undefined) rig.claims.tryClaim("shut-x", "boot", owner); - void rig.pool.pass(); - await flush(); - const started = counts(); - if (registers) { - gate.finish(); - await flush(); - } - void rig.pool.pass(); - await flush(); - const again = counts(); - waiting = [...first, x4]; - void rig.pool.pass(); - await flush(); - return { again, later: counts(), started }; - }; - - const unregistered = await run([], [head, x2]); - expect(unregistered).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); - - const registered = await run([], [head, x2], new Promise(() => undefined), true); - expect(registered).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); - - const bootAndCreation = await run([device("shut-x", "shutdown", 20 * minute)], [head, x2, x3]); - expect(bootAndCreation).toEqual({ again: [1, 1], later: [1, 2], started: [1, 1] }); - - const owned = await run( - [device("shut-x", "shutdown", 20 * minute)], - [head, x2], - undefined, - false, - "an-agent", - ); - expect(owned).toEqual({ again: [0, 1], later: [0, 2], started: [0, 1] }); - }); - - it("does not return from a pass for a creation it started, and settle waits for it", async () => { - const gate = held(); - const rig = harness([], { provision: async () => gate.gate, targets: [iphone17] }); - - await rig.pool.pass(); - let settled = false; - const settling = rig.pool.settle().then(() => { - settled = true; - }); - await flush(); - - expect(rig.provisionCalls).toHaveLength(1); - expect(settled).toBe(false); - gate.finish(); - await settling; - expect(settled).toBe(true); - }); - - it("makes one creation attempt per allowed attempt when it fails: none on the passes between, one a minute after the first, the next two minutes after that", async () => { - const rig = harness([], { - provision: async () => { - throw new Error("the runtime would not boot"); - }, - targets: [iphone17], - }); - rig.pool.start(); - - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - - // Passes triggered by events and ticks within the minute change nothing. - rig.eventBus.emit("device.shutdown", { deviceId: "d", initiator: "test" }, "test"); - await rig.pool.settle(); - rig.clock.advance(WARM_POOL_TICK_MS); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - - rig.clock.advance(minute - WARM_POOL_TICK_MS); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(2); - - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(2); - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(3); - rig.pool.dispose(); - }); - - it("arms a timer for the next allowed attempt, so a pass runs when it is due with no event and no tick", async () => { - const rig = harness([], { - provision: async () => { - throw new Error("the runtime would not boot"); - }, - targets: [iphone17], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - - // `start()` was never called: no tick and no subscription, only the retry timer is armed. - rig.clock.advance(minute); - await rig.pool.settle(); - - expect(rig.provisionCalls).toHaveLength(2); - }); - - it("reports the target short with boot-failed between tries, and logs it once", async () => { - const rig = harness([], { - provision: async () => { - throw new Error("the runtime would not boot"); - }, - targets: [iphone17], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - await rig.pool.pass(); - await rig.pool.pass(); - - expect(rig.pool.targets()).toEqual([ - expect.objectContaining({ count: 1, ready: 0, short: "boot-failed" }), - ]); - const shortLines = rig.sink.records.filter( - (record) => record.message === "a warm pool target is short", - ); - expect(shortLines).toHaveLength(1); - expect(shortLines[0]?.fields).toMatchObject({ - short: "boot-failed", - target: "ios iPhone 17 26.0", - }); - const failure = rig.sink.records.filter( - (record) => record.message === "warm pool creation of a device failed", - ); - expect(failure).toHaveLength(1); - expect(failure[0]?.fields).toMatchObject({ model: "iPhone 17", step: "provision" }); - expect(String(failure[0]?.fields?.["error"])).toContain("the runtime would not boot"); - }); - - it("holds a failed boot of a shut-down device of the kind back for a minute, then boots it again", async () => { - const rig = harness([ofKind("shut", "shutdown")], { - boot: async () => { - throw new Error("simulator did not boot"); - }, - targets: [iphone17], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.bootCalls).toEqual(["shut"]); - - rig.clock.advance(minute - 1); - await rig.pool.settle(); - expect(rig.bootCalls).toEqual(["shut"]); - rig.clock.advance(1); - await rig.pool.settle(); - expect(rig.bootCalls).toEqual(["shut", "shut"]); - }); - - it("clears the schedule when a creation succeeds, so a later failure waits one minute again", async () => { - let failing = true; - const rig = harness([], { - provision: async () => { - if (failing) throw new Error("no luck"); - }, - targets: [iphone17], - }); - await rig.pool.pass(); - await rig.pool.settle(); - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(2); - failing = false; - rig.clock.advance(2 * minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(3); - expect(rig.state.devices).toHaveLength(1); - - // The device is leased: the target is short again, and now fails once more. - failing = true; - rig.lease("new-3"); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(4); - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(5); - }); - - it("reports a target naming a runtime that is not installed as runtime-missing and creates nothing", async () => { - const rig = harness([], { - resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), - targets: [{ ...iphone17, osVersion: "27.0" }], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.pool.targets()).toEqual([ - { - booting: 0, - configuredCount: 1, - count: 1, - kind: { - mode: "full", - model: "iPhone 17", - osVersion: "27.0", - platform: "ios", - }, - message: "iOS 27.0 is not installed", - origin: "configured", - ready: 0, - short: "runtime-missing", - target: "ios iPhone 17 27.0", - }, - ]); - }); - - it.each([ - ["no-driver", "there is no android driver"], - ["unknown-model", "Unknown ios model: iPhone 99"], - ["unresolvable", "the simulator service is down"], - ] as const)( - "reports a target the resolver answers %s for, with its message", - async (refusal, message) => { - const rig = harness([], { resolve: () => ({ message, refusal }), targets: [iphone17] }); - - await rig.pool.pass(); - - expect(rig.pool.targets()).toEqual([expect.objectContaining({ message, short: refusal })]); - expect(rig.provisionCalls).toEqual([]); - }, - ); - - it("resolves every target again each pass, so a runtime installed since takes effect without a restart", async () => { - let installed = false; - const rig = harness([], { - resolve: () => - installed - ? { spec: kind } - : { message: "not installed", refusal: "runtime-missing" as const }, - targets: [iphone17], - }); - await rig.pool.pass(); - expect(rig.provisionCalls).toEqual([]); - - installed = true; - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.resolveCalls.length).toBeGreaterThanOrEqual(2); - expect(rig.provisionCalls).toHaveLength(1); - }); - - it("resolves a target as the request it is: platform, exact model, and the OS and mode it names", async () => { - const rig = harness([], { - targets: [ - { - count: 1, - mode: "slim", - model: "iPhone 16", - osVersion: ">=18", - platform: "ios", - }, - { count: 1, model: "Pixel 9", platform: "android" }, - ], - }); - - await rig.pool.pass(); - - expect(rig.resolveCalls).toStrictEqual([ - { mode: "slim", model: "iPhone 16", osVersion: ">=18", platform: "ios" }, - { model: "Pixel 9", platform: "android" }, - ]); - }); - - it("creates two devices for two targets resolving to one spec, one after the other, as one target of the summed count", async () => { - const first = held(); - const second = held(); - const rig = harness([], { - provision: async (_spec, attempt) => [first, second][attempt - 1]?.gate, - resolve: () => ({ spec: kind }), - targets: [iphone17, { ...iphone17, osVersion: ">=26" }], - }); - - await rig.pool.pass(); - await flush(); - expect(rig.provisionCalls).toHaveLength(1); - first.finish(); - await flush(); - expect(rig.provisionCalls).toHaveLength(2); - second.finish(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toHaveLength(2); - expect(rig.pool.targets()).toHaveLength(1); - expect(rig.pool.targets()[0]).toMatchObject({ count: 2, ready: 2 }); - }); - - it("does nothing for a target while an operator reset holds acquisition closed", async () => { - const rig = harness([], { maintenance: () => true, targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.resolveCalls).toEqual([]); - }); - - it("creates nothing for a target the capacity strategy refuses, and reports why", async () => { - const rig = harness([], { refuseProvision: true, targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "device-limit" })]); - }); - - it("creates nothing, and reports no failure, when the budget refuses the reservation after the policy allowed the creation", async () => { - const rig = harness([], { refuseReservation: true, targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.sink.records.filter((record) => record.level === "error")).toEqual([]); - // A declined creation asks for no pass of its own: the same view would propose it again. - expect(rig.resolveCalls).toHaveLength(1); - // Not a failure: the target is not held back, and the next pass tries it again. - await rig.pool.pass(); - expect(rig.resolveCalls).toHaveLength(2); - expect(rig.pool.targets()[0]?.short).toBeUndefined(); - }); - - it("boots nothing, and asks for no pass of its own, when the device's boot claim is refused", async () => { - const rig = harness([ofKind("shut", "shutdown")], { refuseClaim: true, targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.bootCalls).toEqual([]); - expect(rig.resolveCalls).toHaveLength(1); - expect(rig.reservations.every((reservation) => reservation.released === 1)).toBe(true); - }); - - it("asks for no pass of its own when the lifecycle declines a target's boot", async () => { - const rig = harness([ofKind("shut", "shutdown")], { - boot: async () => undefined, - targets: [iphone17], - }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.bootCalls).toEqual(["shut"]); - expect(rig.resolveCalls).toHaveLength(1); - expect(rig.kick).not.toHaveBeenCalled(); - }); - - it("reports and logs nothing before its first pass", () => { - const rig = harness([ofKind("ready", "ready")], { targets: [iphone17] }); - - expect(rig.pool.targets()).toEqual([]); - expect(rig.sink.records).toEqual([]); - }); - - it("resolves the targets itself when asked which devices they keep before any pass has run", async () => { - const rig = harness([ofKind("ready", "ready")], { targets: [iphone17] }); - - expect([...(await rig.pool.targeted())]).toEqual(["ready"]); - expect(rig.resolveCalls).toHaveLength(1); - // Resolved once: a later ask reads what is there, and a pass resolves again on its own. - await rig.pool.targeted(); - expect(rig.resolveCalls).toHaveLength(1); - }); - - it("stops acting on the rest of a pass's proposals once a drain begins", async () => { - let rig: ReturnType | undefined; - rig = harness( - [ - device("a", "ready", 90 * minute), - device("b", "ready", 80 * minute), - device("c", "ready", 70 * minute), - ], - { - limit: 1, - shutdown: async (target) => { - void rig?.pool.drain(); - return { ...target, state: "shutdown" }; - }, - }, - ); - - await rig.pool.pass(); - - expect(rig.shutdownCalls).toEqual(["a"]); - }); - - it("resolves no target, reports none and names no device with warmPool disabled", async () => { - const rig = harness([ofKind("ready", "ready")], { - enabled: false, - resolve: () => ({ message: "not installed", refusal: "runtime-missing" }), - targets: [iphone17], - }); - - await rig.pool.pass(); - - expect(rig.pool.targets()).toEqual([]); - expect([...(await rig.pool.targeted())]).toEqual([]); - expect(rig.resolveCalls).toEqual([]); - expect( - rig.sink.records.filter((record) => record.message === "a warm pool target is short"), - ).toEqual([]); - }); - - it("keeps a target's last resolved spec when one pass cannot resolve it, and drops it for a settled refusal", async () => { - let answer: TargetResolution = { spec: kind }; - const stale = { ...ofKind("stale", "ready"), readyAt: now - 11 * minute }; - const rig = harness([stale], { resolve: () => answer, targets: [{ ...iphone17, count: 2 }] }); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.shutdownCalls).toEqual([]); - expect(rig.provisionCalls).toHaveLength(1); - rig.lease("new-1"); - - // The target is one short, but the spec it may no longer have plans no creation for it. - answer = { message: "simctl timed out", refusal: "unresolvable" }; - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - expect(rig.shutdownCalls).toEqual([]); - expect([...(await rig.pool.targeted())]).toEqual(["stale"]); - expect(rig.pool.targets()).toEqual([ - expect.objectContaining({ message: "simctl timed out", ready: 1, short: "unresolvable" }), - ]); - const line = rig.sink.records.filter( - (record) => record.message === "a warm pool target is short", - ); - expect(line[0]?.fields).toMatchObject({ ready: 1, short: "unresolvable" }); - - answer = { message: "gone", refusal: "runtime-missing" }; - await rig.pool.pass(); - expect(rig.shutdownCalls).toEqual(["stale"]); - expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "runtime-missing" })]); - - // The spec it once had is gone with a settled refusal: a later failed read has none to keep. - answer = { message: "simctl timed out", refusal: "unresolvable" }; - rig.state.devices = rig.state.devices.map((item) => - item.id === "stale" ? { ...item, state: "ready" as const } : item, - ); - await rig.pool.pass(); - expect(rig.pool.targets()).toEqual([ - expect.objectContaining({ ready: 0, short: "unresolvable" }), - ]); - expect([...(await rig.pool.targeted())]).toEqual([]); - }); - - it("starts nothing once closed, before it drains", async () => { - const rig = harness([], { targets: [iphone17] }); - - rig.pool.close(); - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.provisionCalls).toEqual([]); - expect(rig.resolveCalls).toEqual([]); - }); - - it("does not count a claimed device as ready in the report of a target it cannot resolve", async () => { - let answer: TargetResolution = { spec: kind }; - const rig = harness([ofKind("busy", "ready")], { resolve: () => answer, targets: [iphone17] }); - await rig.pool.pass(); - expect(rig.pool.targets()).toEqual([expect.objectContaining({ ready: 1 })]); - - rig.claims.tryClaim("busy", "eviction"); - answer = { message: "simctl timed out", refusal: "unresolvable" }; - await rig.pool.pass(); - - expect(rig.pool.targets()).toEqual([ - expect.objectContaining({ ready: 0, short: "unresolvable" }), - ]); - }); - - it("starts nothing new once draining, though a creation that was running ends", async () => { - const gate = held(); - const rig = harness([], { - provision: async () => gate.gate, - targets: [{ ...iphone17, count: 3 }], - }); - await rig.pool.pass(); - await flush(); - expect(rig.provisionCalls).toHaveLength(1); - - const draining = rig.pool.drain(); - gate.finish(); - await draining; - const resolved = rig.resolveCalls.length; - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.resolveCalls).toHaveLength(resolved); - - expect(rig.provisionCalls).toHaveLength(1); - expect(rig.state.devices).toHaveLength(1); - expect(rig.clock.pendingTimerCount).toBe(0); - }); - - it("logs a boot or creation that throws outside the driver, holds the target back and asks for no immediate retry", async () => { - const rig = harness([], { reserveThrows: true, targets: [iphone17] }); - - await rig.pool.pass(); - // Bounded by turns, not by settle: a pool that retried at once would never settle. - await flush(); - - expect(rig.reserveAttempts.count).toBe(1); - const errors = rig.sink.records.filter((record) => record.level === "error"); - expect(errors).toHaveLength(1); - expect(errors[0]).toMatchObject({ - fields: { step: "flight" }, - message: "a warm pool boot or creation failed", - }); - expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "boot-failed" })]); - expect(rig.clock.pendingTimerCount).toBe(1); - }); - - it("reports ram-budget and boots nothing when the capacity strategy refuses the shut-down device's boot", async () => { - const rig = harness([ofKind("shut", "shutdown")], { refuseBoot: true, targets: [iphone17] }); - - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.bootCalls).toEqual([]); - expect(rig.pool.targets()).toEqual([expect.objectContaining({ short: "ram-budget" })]); - }); - - it("counts a creation still running toward its target, so a second pass starts no second one even with room for two boots", async () => { - const gate = held(); - const rig = harness([], { - maxConcurrentBoots: 2, - provision: async () => gate.gate, - targets: [iphone17], - }); - - await rig.pool.pass(); - await flush(); - await rig.pool.pass(); - await flush(); - - expect(rig.provisionCalls).toHaveLength(1); - expect(rig.sink.records.filter((record) => record.level === "error")).toEqual([]); - gate.finish(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - }); - - it("asks for no pass and arms no timer once disposed, for a creation that ends afterwards", async () => { - const gate = held(); - const rig = harness([], { provision: async () => gate.gate, targets: [iphone17] }); - await rig.pool.pass(); - await flush(); - - rig.pool.dispose(); - gate.fail(new Error("the runtime would not boot")); - await flush(); - - expect(rig.resolveCalls).toHaveLength(1); - expect(rig.clock.pendingTimerCount).toBe(0); - }); - - it("cancels the retry timer on dispose, and arms none when a pass that was running ends after it", async () => { - let hold: Promise | undefined; - const rig = harness([], { - provision: async () => { - throw new Error("no luck"); - }, - resolve: async (request) => { - await hold; - return { - spec: { - physical: false, - model: request.model ?? "", - osVersion: "26.0", - platform: request.platform, - }, - }; - }, - targets: [iphone17], - }); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.clock.pendingTimerCount).toBe(1); - - let release: () => void = () => undefined; - hold = new Promise((resolve) => { - release = resolve; - }); - const running = rig.pool.pass(); - rig.pool.dispose(); - expect(rig.clock.pendingTimerCount).toBe(0); - release(); - await running; - - expect(rig.clock.pendingTimerCount).toBe(0); - }); - - it("keeps one retry timer however many passes run while a target is held back", async () => { - const rig = harness([], { - provision: async () => { - throw new Error("no luck"); - }, - targets: [iphone17], - }); - await rig.pool.pass(); - await rig.pool.settle(); - - await rig.pool.pass(); - await rig.pool.pass(); - await rig.pool.settle(); - - expect(rig.clock.pendingTimerCount).toBe(1); - }); - - it("arms the timer for a second held-back target when the first one's retry succeeds, so no tick is needed", async () => { - let rig: ReturnType | undefined; - rig = harness([], { - provision: async (spec, attempt) => { - // Attempt 1 (iPhone 17) fails at once, attempt 2 (iPhone 16) fails ten seconds later, - // attempt 3 (iPhone 17 again) succeeds, attempt 4 (iPhone 16 again) is the one under test. - if (attempt === 2) rig?.clock.advance(10_000); - if (attempt <= 2) throw new Error(`no luck for ${spec.model}`); - }, - targets: [iphone17, { ...iphone17, model: "iPhone 16" }], - }); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual(["iPhone 17", "iPhone 16"]); - - // iPhone 17 may be tried at 60s and succeeds; iPhone 16 failed ten seconds later. - rig.clock.advance(50_000); - await rig.pool.settle(); - expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual([ - "iPhone 17", - "iPhone 16", - "iPhone 17", - ]); - - rig.clock.advance(10_000); - await rig.pool.settle(); - expect(rig.provisionCalls.map((call) => call.spec.model)).toEqual([ - "iPhone 17", - "iPhone 16", - "iPhone 17", - "iPhone 16", - ]); - }); - - it("clears the schedule when a boot of a shut-down device succeeds, so a later failure waits one minute again", async () => { - let bootAttempts = 0; - let creating = false; - let rig: ReturnType | undefined; - rig = harness([ofKind("shut", "shutdown")], { - boot: async (target) => { - bootAttempts += 1; - if (bootAttempts === 1) throw new Error("simulator did not boot"); - const ready = { ...target, state: "ready" as const }; - if (rig !== undefined) rig.state.devices = [ready]; - return ready; - }, - provision: async () => { - if (creating) throw new Error("no luck"); - }, - targets: [iphone17], - }); - await rig.pool.pass(); - await rig.pool.settle(); - expect(bootAttempts).toBe(1); - - // The boot is retried after a minute and succeeds. - rig.clock.advance(minute); - await rig.pool.settle(); - expect(bootAttempts).toBe(2); - - // The device is then leased, the target is short again, and a creation fails: one minute - // later it is tried again, not two. - creating = true; - rig.lease("shut"); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(1); - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.provisionCalls).toHaveLength(2); - }); - - describe("figures", () => { - const wanted = { - mode: "full", - model: "iPhone 17", - origin: "configured", - osVersion: "26.0", - platform: "ios", - } as const; - - it("lists a target with its kind, count, ready and booting, and no reason while it fills", async () => { - let finish: () => void = () => undefined; - const rig = harness([ofKind("ready", "ready")], { - limit: 5, - provision: () => - new Promise((resolve) => { - finish = resolve; - }), - targets: [{ ...iphone17, count: 2 }], - }); - - await rig.pool.pass(); - - expect(rig.pool.figures()).toStrictEqual({ - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - targets: [{ ...wanted, booting: 1, count: 2, ready: 1 }], - }); - finish(); - await rig.pool.settle(); - }); - - it("lists a target the resolver refuses with the kind it was configured with and the refusal as short", async () => { - const rig = harness([], { - resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), - targets: [{ ...iphone17, osVersion: "27.0" }], - }); - - await rig.pool.pass(); - - expect(rig.pool.figures().targets).toStrictEqual([ - { ...wanted, booting: 0, count: 1, osVersion: "27.0", ready: 0, short: "runtime-missing" }, - ]); - }); - - it("lists every configured target as short disabled with the pool off, before and after a pass", async () => { - const rig = harness([ofKind("ready", "ready")], { - enabled: false, - targets: [iphone17, { ...iphone17, mode: "slim", model: "iPhone 16" }], - }); - const expected = { - enabled: false, - reserveRunning: { android: 0, ios: 0 }, - targets: [ - { ...wanted, booting: 0, count: 1, ready: 0, short: "disabled" }, - { - ...wanted, - booting: 0, - count: 1, - mode: "slim", - model: "iPhone 16", - ready: 0, - short: "disabled", - }, - ], - }; - - expect(rig.pool.figures()).toStrictEqual(expected); - await rig.pool.pass(); - expect(rig.pool.figures()).toStrictEqual(expected); - }); - - it("lists every configured target before the first pass with no reason, so a status read just after a start is not empty", () => { - const rig = harness([], { targets: [iphone17] }); - - expect(rig.pool.figures()).toStrictEqual({ - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - targets: [{ ...wanted, booting: 0, count: 1, ready: 0 }], - }); - }); - - it("names the platform's default mode for a target that names none, before a pass, with the pool off, refused and resolved", async () => { - const { osVersion: _osVersion, ...unversioned } = iphone17; - const modes = async (options: { enabled?: boolean; refuse?: boolean; pass: boolean }) => { - const rig = harness([], { - defaultMode: "slim", - ...(options.enabled === undefined ? {} : { enabled: options.enabled }), - ...(options.refuse === true - ? { resolve: () => ({ message: "why", refusal: "no-driver" as const }) } - : {}), - targets: [unversioned], - }); - if (options.pass) await rig.pool.pass(); - return rig.pool.figures().targets.map((target) => target.mode); - }; - - expect(await modes({ pass: false })).toEqual(["slim"]); - expect(await modes({ enabled: false, pass: true })).toEqual(["slim"]); - expect(await modes({ pass: true, refuse: true })).toEqual(["slim"]); - expect(await modes({ pass: true })).toEqual(["slim"]); - }); - - it("counts only the boots of the spec a target keeps while its resolution is refused unresolvable, and none for one that never resolved", async () => { - const creation = held(); - let refuse17 = false; - const rig = harness([], { - limit: 5, - maxConcurrentBoots: 2, - provision: () => creation.gate, - resolve: (request) => { - if (request.model === "iPhone 15") return { message: "why", refusal: "no-driver" }; - if (request.model === "iPhone 17" && refuse17) { - return { message: "why", refusal: "unresolvable" }; - } - return { - spec: { - physical: false, - model: request.model ?? "", - osVersion: "26.0", - platform: request.platform, - }, - }; - }, - targets: [ - iphone17, - { ...iphone17, model: "iPhone 16" }, - { ...iphone17, model: "iPhone 15" }, - ], - }); - const figures = () => - Object.fromEntries(rig.pool.figures().targets.map((target) => [target.model, target])); - await rig.pool.pass(); - expect(figures()["iPhone 17"]?.booting).toBe(1); - - refuse17 = true; - await rig.pool.pass(); - - expect(figures()["iPhone 17"]).toMatchObject({ booting: 1, short: "unresolvable" }); - expect(figures()["iPhone 16"]).toMatchObject({ booting: 1 }); - expect(figures()["iPhone 15"]).toMatchObject({ booting: 0, short: "no-driver" }); - creation.finish(); - await rig.pool.settle(); - }); - - it("leaves out osVersion for a refused target that names none", async () => { - const { osVersion: _osVersion, ...unversioned } = iphone17; - const rig = harness([], { - resolve: () => ({ message: "no driver", refusal: "no-driver" }), - targets: [unversioned], - }); - - await rig.pool.pass(); - - expect(rig.pool.figures().targets[0]).toStrictEqual({ - booting: 0, - count: 1, - mode: "full", - model: "iPhone 17", - origin: "configured", - platform: "ios", - ready: 0, - short: "no-driver", - }); - }); - }); - - describe("warm-pool.target-missed", () => { - const missed = (rig: ReturnType): unknown[] => { - const payloads: unknown[] = []; - rig.eventBus.subscribe("warm-pool.target-missed", (envelope) => { - payloads.push(envelope.payload); - }); - return payloads; - }; - - it("fires once when a target first ends a pass short, with its kind, count, ready and reason", async () => { - const rig = harness([], { - resolve: () => ({ message: "iOS 27.0 is not installed", refusal: "runtime-missing" }), - targets: [{ ...iphone17, osVersion: "27.0" }], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - - expect(payloads).toStrictEqual([ - { - count: 1, - mode: "full", - model: "iPhone 17", - origin: "configured", - osVersion: "27.0", - platform: "ios", - ready: 0, - reason: "runtime-missing", - }, - ]); - }); - - it("does not fire again on the next pass that leaves the target short", async () => { - const rig = harness([], { - resolve: () => ({ message: "why", refusal: "runtime-missing" }), - targets: [iphone17], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - await rig.pool.pass(); - await rig.pool.pass(); - - expect(payloads).toHaveLength(1); - }); - - it("names the pool as the emitter, and leaves osVersion out for a target that names none", async () => { - const { osVersion: _osVersion, ...unversioned } = iphone17; - const rig = harness([], { - resolve: () => ({ message: "no driver", refusal: "no-driver" }), - targets: [unversioned], - }); - const seen: { module: string; payload: unknown }[] = []; - rig.eventBus.subscribe("warm-pool.target-missed", (envelope) => { - seen.push({ module: envelope.module, payload: envelope.payload }); - }); - - await rig.pool.pass(); - - expect(seen).toStrictEqual([ - { - module: "warm-pool", - payload: { - count: 1, - mode: "full", - model: "iPhone 17", - origin: "configured", - platform: "ios", - ready: 0, - reason: "no-driver", - }, - }, - ]); - }); - - it("fires again when the target was met in between, ready reaching its count, and is missed again", async () => { - let refuse = true; - const rig = harness([ofKind("ready", "ready")], { - resolve: (request) => - refuse - ? { message: "why", refusal: "runtime-missing" } - : { - spec: { - physical: false, - model: request.model ?? "", - osVersion: "26.0", - platform: request.platform, - }, - }, - targets: [iphone17], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - refuse = false; - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.pool.figures().targets[0]).toMatchObject({ count: 1, ready: 1 }); - refuse = true; - await rig.pool.pass(); - - expect(payloads).toHaveLength(2); - }); - - it("does not fire again for a target that is only not short on a pass, filling or retrying, until it is met", async () => { - const rig = harness([], { - limit: 5, - provision: async () => { - throw new Error("no luck"); - }, - targets: [iphone17], - }); - const payloads = missed(rig); - - // The first creation fails and the target is short `boot-failed`; the retry a minute later - // is a pass that is not short (a creation runs), and it fails again. - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.pool.figures().targets[0]?.short).toBe("boot-failed"); - rig.clock.advance(minute); - await rig.pool.settle(); - rig.clock.advance(2 * minute); - await rig.pool.settle(); - - expect(rig.provisionCalls).toHaveLength(3); - expect(payloads).toStrictEqual([expect.objectContaining({ reason: "boot-failed" })]); - }); - - it("fires once for a target with no OS that is refused one pass and short the next, though its kind changes", async () => { - const { osVersion: _osVersion, ...unversioned } = iphone17; - let refuse = true; - const rig = harness([], { - limit: 0, - resolve: (request) => - refuse - ? { message: "why", refusal: "unresolvable" } - : { - spec: { - physical: false, - model: request.model ?? "", - osVersion: "26.0", - platform: request.platform, - }, - }, - targets: [unversioned], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - refuse = false; - await rig.pool.pass(); - - expect(rig.pool.figures().targets[0]).toMatchObject({ - osVersion: "26.0", - short: "running-limit", - }); - expect(payloads).toStrictEqual([expect.objectContaining({ reason: "unresolvable" })]); - }); - - it("fires once for two targets that resolve to one spec, and on no later pass", async () => { - const rig = harness([], { - limit: 0, - targets: [iphone17, { ...iphone17, count: 2 }], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - await rig.pool.pass(); - - expect(payloads).toStrictEqual([ - expect.objectContaining({ count: 3, reason: "running-limit" }), - ]); - }); - - it("fires for each of two configured targets of one kind that are both refused, not once for the kind", async () => { - const rig = harness([], { - resolve: () => ({ message: "why", refusal: "runtime-missing" }), - targets: [iphone17, { ...iphone17, count: 2 }], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - await rig.pool.pass(); - - expect(payloads).toStrictEqual([ - expect.objectContaining({ count: 1, model: "iPhone 17", reason: "runtime-missing" }), - expect.objectContaining({ count: 2, model: "iPhone 17", reason: "runtime-missing" }), - ]); - }); - - it("keeps each configured target's edge apart: one met and one short, then the met one missed, fires for it too", async () => { - const rig = harness([ofKind("ready", "ready")], { - limit: 0, - targets: [iphone17, { ...iphone17, model: "iPhone 16" }], - }); - const payloads = missed(rig); - - await rig.pool.pass(); - rig.lease("ready"); - await rig.pool.pass(); - - expect(payloads.map((payload) => (payload as { model: string }).model)).toEqual([ - "iPhone 16", - "iPhone 17", - ]); - }); - - it("fires nothing for a target that is filling, and nothing with the pool off", async () => { - const filling = harness([], { limit: 5, targets: [iphone17] }); - const fillingPayloads = missed(filling); - await filling.pool.pass(); - await filling.pool.settle(); - const off = harness([], { enabled: false, targets: [iphone17] }); - const offPayloads = missed(off); - await off.pool.pass(); - - expect(fillingPayloads).toEqual([]); - expect(offPayloads).toEqual([]); - }); - }); - - describe("targeted", () => { - it("names, once a pass has resolved the targets, the ready unleased devices of the kind up to the count, oldest first", async () => { - const rig = harness( - [ofKind("newer", "ready", 20), ofKind("older", "ready", 10), ofKind("oldest", "ready", 5)], - { limit: 5, targets: [{ ...iphone17, count: 2 }] }, - ); - await rig.pool.pass(); - - expect([...(await rig.pool.targeted())].sort()).toEqual(["older", "oldest"]); - }); - - it("names a learned kind's ready devices up to its count before the first pass, and again for a pool started over the same record", async () => { - const spec = ofKind("x", "ready").spec; - new LearnedKinds(learnedState).learn(spec, "lease-1", now - minute); - new LearnedKinds(learnedState).learn(spec, "lease-2", now - minute); - const devices = [ofKind("newer", "ready", 20), ofKind("older", "ready", 10)]; - - const before = harness(devices, { limit: 5 }); - const restarted = harness(devices, { limit: 5 }); - const off = harness(devices, { enabled: false, limit: 5 }); - - expect([...(await before.pool.targeted())].sort()).toEqual(["newer", "older"]); - expect([...(await restarted.pool.targeted())].sort()).toEqual(["newer", "older"]); - expect([...(await off.pool.targeted())]).toEqual([]); - }); - - it("does not name a device that is leased or claimed", async () => { - const rig = harness([ofKind("leased", "ready", 1), ofKind("claimed", "ready", 2)], { - limit: 5, - targets: [{ ...iphone17, count: 2 }], - }); - await rig.pool.pass(); - rig.lease("leased"); - rig.claims.tryClaim("claimed", "boot"); - - expect([...(await rig.pool.targeted())]).toEqual([]); - }); - }); -}); diff --git a/src/core/warm-pool/test-converger.ts b/src/core/warm-pool/test-converger.ts new file mode 100644 index 000000000..aeca37941 --- /dev/null +++ b/src/core/warm-pool/test-converger.ts @@ -0,0 +1,42 @@ +import type { VirtualDevice } from "../domain.js"; +import type { WarmTarget } from "./config.js"; + +export const iphone17: WarmTarget = { + count: 1, + model: "iPhone 17", + osVersion: "26.0", + platform: "ios", +}; +export const kind = { + model: "iPhone 17", + osVersion: "26.0", + physical: false, + platform: "ios", +} as const; +export const ofKind = ( + id: string, + state: VirtualDevice["state"], + createdAt = 1, +): VirtualDevice => ({ + physical: false, + createdAt, + driverData: {}, + driverDeviceId: `driver-${id}`, + id, + mode: "full", + spec: kind, + state, +}); +/** A creation the test holds open until it calls `finish`, or fails with `fail`. */ +export function held() { + let finish: () => void = () => undefined; + let fail: (error: Error) => void = () => undefined; + const gate = new Promise((resolve, reject) => { + finish = resolve; + fail = reject; + }); + return { fail, finish, gate }; +} +export const flush = async (): Promise => { + for (let turn = 0; turn < 30; turn += 1) await Promise.resolve(); +}; From 018025571e6226805f96a3c4d515f8b83ca16e2e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:50 +0100 Subject: [PATCH 52/83] test(core): split registry.test.ts by topic --- src/core/registry-component-records.test.ts | 66 ++ src/core/registry-leases.test.ts | 411 +++++++ src/core/registry-quarantine-cause.test.ts | 129 +++ src/core/registry-quarantine.test.ts | 495 ++++++++ src/core/registry-transition-event.test.ts | 49 + src/core/registry.test.ts | 1150 +------------------ src/core/test-registry-corrupt-row.ts | 29 + 7 files changed, 1181 insertions(+), 1148 deletions(-) create mode 100644 src/core/registry-component-records.test.ts create mode 100644 src/core/registry-leases.test.ts create mode 100644 src/core/registry-quarantine-cause.test.ts create mode 100644 src/core/registry-quarantine.test.ts create mode 100644 src/core/registry-transition-event.test.ts create mode 100644 src/core/test-registry-corrupt-row.ts diff --git a/src/core/registry-component-records.test.ts b/src/core/registry-component-records.test.ts new file mode 100644 index 000000000..8cdcb8d97 --- /dev/null +++ b/src/core/registry-component-records.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { type StateDatabase } from "../state/index.js"; +import { loadTestRegistry, openTestState } from "./testing.js"; + +describe("Registry: component records", () => { + async function load(state: StateDatabase) { + const clock = new FakeClock(1_000); + return loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + state, + }); + } + + it("keeps one record per platform and version, the latest install replacing the earlier one", async () => { + const registry = await load(await openTestState()); + + await registry.recordComponent({ + installedAt: 1, + platform: "android", + receipt: { stamp: "first" }, + version: "35", + }); + await registry.recordComponent({ + installedAt: 2, + platform: "ios", + receipt: { image: "IMG" }, + version: "35", + }); + await registry.recordComponent({ + installedAt: 3, + platform: "android", + receipt: { stamp: "second" }, + version: "35", + }); + + expect(registry.snapshot.components).toEqual([ + { installedAt: 2, platform: "ios", receipt: { image: "IMG" }, version: "35" }, + { installedAt: 3, platform: "android", receipt: { stamp: "second" }, version: "35" }, + ]); + }); + + it("deletes the record of one platform and version, keeps the others, and the deletion survives a reload", async () => { + const state = await openTestState(); + const registry = await load(state); + for (const [platform, version] of [ + ["ios", "26.4"], + ["ios", "27.0"], + ["android", "26.4"], + ] as const) { + await registry.recordComponent({ installedAt: 1, platform, receipt: { version }, version }); + } + + await registry.deleteComponent("ios", "26.4"); + + const kept = [ + { installedAt: 1, platform: "ios", receipt: { version: "27.0" }, version: "27.0" }, + { installedAt: 1, platform: "android", receipt: { version: "26.4" }, version: "26.4" }, + ]; + expect(registry.snapshot.components).toEqual(kept); + expect((await load(state)).snapshot.components).toEqual(kept); + }); +}); diff --git a/src/core/registry-leases.test.ts b/src/core/registry-leases.test.ts new file mode 100644 index 000000000..e8dddb091 --- /dev/null +++ b/src/core/registry-leases.test.ts @@ -0,0 +1,411 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { loadTestRegistry, openTestState } from "./testing.js"; +import { Registry, UnknownLeaseError } from "./index.js"; +import { seededWorld, spec, totalChanges } from "./test-registry.js"; + +describe("Registry", () => { + it("reloads a lease written for a request with the request granted, holding the leased device, environment, lease and timing it was written with", async () => { + const clock = new FakeClock(1_000); + const suffixes = ["device", "lease"]; + const state = await openTestState(); + const options = { + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, + state, + }; + const registry = await loadTestRegistry(options); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.createLeaseRequest({ + id: "req_1", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + }); + const timing = { + estimatedBootMs: 2, + estimatedProvisionMs: 1, + estimatedReadyMs: 4, + estimatedReclaimMs: 3, + }; + + const lease = await registry.createLease({ + deviceId: device.id, + ownerId: "agent-1", + request: { environment: { SIMLOCK_UDID: "abc" }, id: "req_1", timing }, + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }); + const reloaded = await loadTestRegistry(options); + + expect(reloaded.snapshot.leases).toEqual([lease]); + expect(reloaded.leaseRequests()).toMatchObject([ + { + grant: { + device: reloaded.snapshot.devices[0], + environment: { SIMLOCK_UDID: "abc" }, + lease, + timing, + }, + id: "req_1", + state: "granted", + }, + ]); + expect(reloaded.snapshot.devices[0]).toMatchObject({ id: device.id, state: "leased" }); + expect(lease.id).toBe("lse_lease"); + }); + + it("leaves a request already settled as it was when a lease is written for it", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "x" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.createLeaseRequest({ + id: "req_1", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + }); + await registry.settleLeaseRequest("req_1", { + failure: { code: "INTERNAL", message: "gone" }, + state: "failed", + }); + + await registry.createLease({ + deviceId: device.id, + ownerId: "agent-1", + request: { + environment: {}, + id: "req_1", + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 0, + }, + }, + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }); + + expect(registry.snapshot.leases).toHaveLength(1); + expect(registry.leaseRequests()).toMatchObject([ + { failure: { code: "INTERNAL", message: "gone" }, id: "req_1", state: "failed" }, + ]); + }); + + it("writes a device the same way whether it is marked missing alone or together with its lease's end", async () => { + const marked = async (viaLease: boolean) => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.markRecoveryAttempt(device.id, 1_500); + if (viaLease) { + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.endLeaseAndMarkDeviceMissing(lease.id, "doctor", () => undefined); + } else { + await registry.markDeviceMissing(device.id, "doctor"); + } + return registry.snapshot.devices[0]; + }; + + const viaLease = await marked(true); + const alone = await marked(false); + + expect(alone).toEqual(viaLease); + expect(viaLease).toEqual({ + physical: false, + createdAt: 1_000, + deletedAt: 1_000, + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + id: "dev_test", + leaseIdentity: "reusable", + mode: "full", + readyAt: 1_000, + spec, + state: "deleted", + }); + }); + + it("keeps idChosenByRequester true on the lease of a stored grant across a reload", async () => { + const clock = new FakeClock(1_000); + const state = await openTestState(); + const options = { + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "x" }, + state, + }; + const registry = await loadTestRegistry(options); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.createLeaseRequest({ + id: "req_1", + leaseId: "ad-7f3a", + ownerId: "agent-1", + request: { platform: "ios" }, + requesterId: "agent-1", + }); + await registry.createLease({ + deviceId: device.id, + leaseId: "ad-7f3a", + ownerId: "agent-1", + request: { + environment: {}, + id: "req_1", + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 0, + }, + }, + requesterId: "agent-1", + ttlDeadline: 2_000, + ttlMs: 60_000, + }); + + const reloaded = await loadTestRegistry(options); + + expect(reloaded.leaseRequests()[0]?.grant?.lease).toMatchObject({ + id: "ad-7f3a", + idChosenByRequester: true, + }); + }); + + it("writes idChosenByRequester true for a lease created with a leaseId, false without one, and keeps both across a reload", async () => { + const clock = new FakeClock(1_000); + let next = 0; + const state = await openTestState(); + const options = { + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => String((next += 1)) }, + state, + }; + const registry = await loadTestRegistry(options); + const ready = async (driverDeviceId: string) => { + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId, + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + return device.id; + }; + const base = { ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 2_000, ttlMs: 60_000 }; + + const chosen = await registry.createLease({ + ...base, + deviceId: await ready("one"), + leaseId: "ad-7f3a", + }); + const generated = await registry.createLease({ ...base, deviceId: await ready("two") }); + + expect(chosen).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); + expect(generated).toMatchObject({ + id: expect.stringMatching(/^lse_/), + idChosenByRequester: false, + }); + const reloaded = await loadTestRegistry(options); + expect(reloaded.snapshot.leases).toEqual([chosen, generated]); + }); + + it("removes a lease and marks its device deleted in one commit, announcing the lease before device.deleted, both after the commit", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const suffixes = ["device", "lease"]; + const state = await openTestState(); + const options = { + clock, + eventBus: bus, + idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, + state, + }; + const registry = await loadTestRegistry(options); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.markRecoveryAttempt(device.id, 1_500); + const operations: string[] = []; + const stored = () => { + const row = state.database + .prepare( + "SELECT (SELECT COUNT(*) FROM leases) AS leases, (SELECT state FROM devices) AS device", + ) + .get(); + return `${String(row?.leases)} leases stored, device ${String(row?.device)}`; + }; + bus.subscribe("device.deleted", () => operations.push(`device.deleted: ${stored()}`)); + + const ended = await registry.endLeaseAndMarkDeviceMissing(lease.id, "doctor", () => + operations.push(`lease announced: ${stored()}`), + ); + + expect(operations).toEqual([ + "lease announced: 0 leases stored, device deleted", + "device.deleted: 0 leases stored, device deleted", + ]); + expect(ended.lease).toEqual(lease); + expect(ended.device).toMatchObject({ id: device.id, state: "deleted" }); + expect(ended.device).not.toHaveProperty("recoveryAttempts"); + expect(registry.snapshot.leases).toEqual([]); + expect(registry.snapshot.devices).toEqual([ended.device]); + expect((await loadTestRegistry(options)).snapshot).toEqual(registry.snapshot); + expect(bus.replay().filter((entry) => entry.event === "device.deleted")).toMatchObject([ + { payload: { deviceId: device.id, initiator: "doctor" } }, + ]); + }); + + it("ends only the named lease and marks only its device deleted, leaving every other lease and device as they were, with the event's module registry", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + let next = 0; + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => String(next++) }, + }); + const leases = []; + for (const name of ["one", "two"]) { + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: `driver_${name}`, + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + leases.push( + await registry.createLease({ + deviceId: device.id, + requesterId: name, + ownerId: name, + ttlMs: 60_000, + ttlDeadline: 2_000, + }), + ); + } + const [first, second] = leases; + if (first === undefined || second === undefined) throw new Error("leases not created"); + + await registry.endLeaseAndMarkDeviceMissing(second.id, "doctor", () => undefined); + + expect(registry.snapshot.leases).toEqual([first]); + expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toEqual([ + [first.deviceId, "leased"], + [second.deviceId, "deleted"], + ]); + expect(bus.replay().filter((entry) => entry.event === "device.deleted")).toMatchObject([ + { module: "registry", payload: { deviceId: second.deviceId, initiator: "doctor" } }, + ]); + }); + + it.each([ + [ + "endLeaseAndMarkDeviceMissing", + (registry: Registry, announce: () => void) => + registry.endLeaseAndMarkDeviceMissing("lse_nope", "doctor", announce), + ], + [ + "endLeaseAndMarkDeviceAbsent", + (registry: Registry, announce: () => void) => + registry.endLeaseAndMarkDeviceAbsent("lse_nope", "locked", announce), + ], + ])( + "%s refuses to end a lease it does not hold, writing nothing and announcing nothing", + async (_method, endLease) => { + const { enroll, lease, registry, state } = await seededWorld(); + await enroll("physical"); + await lease("dev_physical"); + const devicesBefore = structuredClone(registry.snapshot.devices); + const leasesBefore = structuredClone(registry.snapshot.leases); + const changesBefore = totalChanges(state); + let announced = false; + + await expect( + endLease(registry, () => { + announced = true; + }), + ).rejects.toThrow(UnknownLeaseError); + + expect(announced).toBe(false); + expect(totalChanges(state)).toBe(changesBefore); + expect(registry.snapshot.devices).toEqual(devicesBefore); + expect(registry.snapshot.leases).toEqual(leasesBefore); + }, + ); +}); diff --git a/src/core/registry-quarantine-cause.test.ts b/src/core/registry-quarantine-cause.test.ts new file mode 100644 index 000000000..fa892cf06 --- /dev/null +++ b/src/core/registry-quarantine-cause.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from "vitest"; +import { type StateDatabase } from "../state/index.js"; +import { Registry } from "./index.js"; +import { physicalInput, seededWorld } from "./test-registry.js"; + +describe("Registry: quarantineCause", () => { + const purgeCause = { + kind: "purge-failed" as const, + message: "DriverCrashError: simctl erase exited 164", + }; + + async function reclaimingVirtual() { + const world = await seededWorld(); + await world.register("virtual"); + await world.lease("dev_virtual", "lse_virtual"); + await world.registry.beginRelease("lse_virtual"); + return world; + } + + async function quarantinedVirtual() { + const world = await reclaimingVirtual(); + await world.registry.enterQuarantine("dev_virtual", 5_000, purgeCause); + return world; + } + + async function quarantinedPhysical() { + const world = await seededWorld(); + await world.enroll("physical"); + await world.lease("dev_physical", "lse_physical"); + await world.registry.beginRelease("lse_physical"); + await world.registry.enterQuarantine("dev_physical", 5_000, purgeCause); + return world; + } + + const storedCauseKinds = (world: { state: StateDatabase }) => + world.state.database.prepare("SELECT quarantine_cause_kind FROM devices").all(); + + it("a quarantined record with each kind, written by enterQuarantine and loaded by a new Registry, has its cause", async () => { + const causes = [ + { kind: "purge-failed", message: "Error: purge exploded" }, + { kind: "shutdown-failed", message: "Error: shutdown exploded" }, + { kind: "delete-failed", message: "Error: delete exploded" }, + { kind: "stalled", stalledState: "provisioning" }, + { kind: "stalled", stalledState: "reclaiming" }, + ] as const; + + for (const cause of causes) { + const { registry, reload } = await reclaimingVirtual(); + if (cause.kind === "delete-failed") { + await registry.completeReclaimWithoutPurge("dev_virtual"); + } + const entered = await registry.enterQuarantine("dev_virtual", 5_000, cause); + expect(entered.quarantineCause, cause.kind).toEqual(cause); + + const reloaded = await reload(); + expect(reloaded.snapshot.devices[0], cause.kind).toMatchObject({ + quarantineCause: cause, + state: "quarantined", + }); + } + }); + + describe("every exit from quarantined removes it", () => { + const readyEvent = (id: string) => ({ + event: "device.ready" as const, + payload: { bootDuration: 0, deviceId: id }, + }); + const exits: readonly (readonly [string, (registry: Registry) => Promise])[] = [ + ["recoverFromQuarantine to ready", (r) => r.recoverFromQuarantine("dev_virtual", "ready")], + [ + "recoverFromQuarantine to shutdown", + (r) => r.recoverFromQuarantine("dev_virtual", "shutdown"), + ], + ["deleteQuarantined", (r) => r.deleteQuarantined("dev_virtual", "lease-end")], + ["markDeviceMissing", (r) => r.markDeviceMissing("dev_virtual", "doctor")], + [ + "transitionDevice to ready", + (r) => r.transitionDevice("dev_virtual", "ready", readyEvent("dev_virtual")), + ], + [ + "transitionDevice to shutdown", + (r) => + r.transitionDevice("dev_virtual", "shutdown", { + event: "device.shutdown", + payload: { deviceId: "dev_virtual", initiator: "idle" }, + }), + ], + [ + "transitionDevice to deleted", + (r) => + r.transitionDevice("dev_virtual", "deleted", { + event: "device.deleted", + payload: { deviceId: "dev_virtual", initiator: "lease-end" }, + }), + ], + ]; + + it.each(exits)("%s", async (_name, exit) => { + const world = await quarantinedVirtual(); + const { registry } = world; + expect(registry.snapshot.devices[0]).toHaveProperty("quarantineCause"); + + const result = await exit(registry); + + expect(result).not.toHaveProperty("quarantineCause"); + expect(registry.snapshot.devices[0]).not.toHaveProperty("quarantineCause"); + expect(storedCauseKinds(world)).toEqual([{ quarantine_cause_kind: null }]); + }); + + it.each([ + [ + "reenrollPhysical", + (r: Registry) => + r.reenrollPhysical("dev_physical", { ...physicalInput, apps: ["com.example.one"] }), + ], + ["unenrollPhysical", (r: Registry) => r.unenrollPhysical("dev_physical")], + ])("%s", async (_name, exit) => { + const world = await quarantinedPhysical(); + const { registry } = world; + expect(registry.snapshot.devices[0]).toHaveProperty("quarantineCause"); + + const result = await exit(registry); + + expect(result).not.toHaveProperty("quarantineCause"); + expect(registry.snapshot.devices[0]).not.toHaveProperty("quarantineCause"); + expect(storedCauseKinds(world)).toEqual([{ quarantine_cause_kind: null }]); + }); + }); +}); diff --git a/src/core/registry-quarantine.test.ts b/src/core/registry-quarantine.test.ts new file mode 100644 index 000000000..ec534273f --- /dev/null +++ b/src/core/registry-quarantine.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { loadTestRegistry, openTestState } from "./testing.js"; +import { RegistryEventError, RegistryLoadError, UnknownDeviceError } from "./index.js"; +import { spec } from "./test-registry.js"; +import { loadAfterCorruptingDevice } from "./test-registry-corrupt-row.js"; + +describe("Registry", () => { + it("enters quarantine from reclaiming, tracking attempts and the next retry deadline", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.beginRelease(lease.id); + + const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + expect(quarantined).toMatchObject({ + quarantineAttempts: 0, + quarantineNextRetryAt: 5_000, + quarantinedAt: 1_000, + state: "quarantined", + }); + await expect( + registry.enterQuarantine(device.id, 6_000, { kind: "purge-failed" }), + ).rejects.toThrow(RegistryEventError); + }); + + it("enters quarantine from provisioning, its stalled-transition entry point", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + + const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + expect(quarantined).toMatchObject({ + quarantineAttempts: 0, + quarantineNextRetryAt: 5_000, + quarantinedAt: 1_000, + state: "quarantined", + }); + }); + + it("enters quarantine from shutdown, a spent fresh device's failed-delete entry point", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + leaseIdentity: { android: "reusable", ios: "fresh" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.beginRelease(lease.id); + clock.advance(7_000); + const shutDown = await registry.completeReclaimWithoutPurge(device.id); + // Every path into `shutdown` stamps it, the reclaim's own included. + expect(shutDown.shutdownAt).toBe(8_000); + + const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + expect(quarantined).toMatchObject({ quarantineNextRetryAt: 5_000, state: "quarantined" }); + }); + + it("refuses to quarantine a device that is still leased", async () => { + // Quarantine is a post-release disposition: it is only ever entered from `reclaiming`, which + // a device reaches by having its lease released. A leased device reaching it would mean + // pulling a device out from under its holder -- the one thing safety rule 2 forbids outright. + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + + await expect( + registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }), + ).rejects.toThrow(RegistryEventError); + expect(registry.snapshot.devices[0]?.state).toBe("leased"); + }); + + it("records a quarantine retry failure without leaving quarantined", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.beginRelease(lease.id); + await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + const retried = await registry.recordQuarantineRetryFailure(device.id, 1, 15_000); + + expect(retried).toMatchObject({ + quarantineAttempts: 1, + quarantineNextRetryAt: 15_000, + state: "quarantined", + }); + await expect(registry.recordQuarantineRetryFailure("dev_missing", 1, 15_000)).rejects.toThrow( + UnknownDeviceError, + ); + }); + + it("recovers a quarantined device and clears its quarantine bookkeeping", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.beginRelease(lease.id); + await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + const recovered = await registry.recoverFromQuarantine(device.id, "ready"); + + expect(recovered).toEqual({ + physical: false, + createdAt: 1_000, + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + id: device.id, + lastLeaseEndedAt: 1_000, + leaseIdentity: "reusable", + readyAt: 1_000, + spec, + mode: "full", + state: "ready", + }); + await expect(registry.recoverFromQuarantine(device.id, "ready")).rejects.toThrow( + RegistryEventError, + ); + }); + + it("deletes a quarantined device and emits device.deleted with the caller's initiator", async () => { + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const events: unknown[] = []; + bus.subscribe("device.deleted", (envelope) => events.push(envelope.payload)); + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.beginRelease(lease.id); + await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); + + const deleted = await registry.deleteQuarantined(device.id, "lease-end"); + + expect(deleted.state).toBe("deleted"); + expect(events).toEqual([{ deviceId: device.id, initiator: "lease-end", physical: false }]); + }); + + it("rejects mutations for a device that is not registered", async () => { + const clock = new FakeClock(); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + + await expect(registry.transitionDevice("dev_missing", "ready")).rejects.toThrow( + UnknownDeviceError, + ); + await expect( + registry.transitionDevice("dev_missing", "deleted", { + event: "device.deleted", + payload: { deviceId: "dev_missing", initiator: "test" }, + }), + ).rejects.toThrow(UnknownDeviceError); + }); + + it("records the first recovery attempt, then keeps its recoveringSince and increments the count on retry", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + const first = await registry.markRecoveryAttempt(device.id, 1_500); + expect(first).toMatchObject({ recoveringSince: 1_500, recoveryAttempts: 1 }); + + const updated = await registry.markRecoveryAttempt(device.id, 2_000); + + expect(updated).toMatchObject({ recoveringSince: 1_500, recoveryAttempts: 2 }); + }); + + it("clears both recovery markers", async () => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.markRecoveryAttempt(device.id, 1_500); + + const cleared = await registry.clearRecovery(device.id); + + expect(cleared.recoveringSince).toBeUndefined(); + expect(cleared.recoveryAttempts).toBeUndefined(); + }); + + it("persists readyAt and shutdownAt across a reload", async () => { + const clock = new FakeClock(1_000); + const state = await openTestState(); + const options = { + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + state, + }; + const registry = await loadTestRegistry(options); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 5, deviceId: device.id }, + }); + + clock.advance(5_000); + await registry.transitionDevice(device.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: device.id, initiator: "test" }, + }); + const reloaded = await loadTestRegistry(options); + + expect(reloaded.snapshot.devices[0]?.readyAt).toBe(1_000); + expect(reloaded.snapshot.devices[0]?.shutdownAt).toBe(6_000); + }); + + it.each(["mode", "spec_mode"])( + "fails the load when a stored %s is neither slim nor full", + async (column) => { + await expect(loadAfterCorruptingDevice(`${column} = 'reduced'`)).rejects.toThrow( + RegistryLoadError, + ); + }, + ); + + it.each([ + ["no mode", spec], + ["a slim mode", { ...spec, mode: "slim" as const }], + ])("registers a new device as full, whether its spec carries %s", async (_label, deviceSpec) => { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + }); + + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec: deviceSpec, + }); + + expect(device.spec).toEqual(deviceSpec); + expect(device.mode).toBe("full"); + }); + + it("clears recovery markers when a lease ends", async () => { + const clock = new FakeClock(1_000); + const suffixes = ["device", "lease"]; + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_device", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + await registry.markRecoveryAttempt(device.id, 1_200); + + const released = await registry.beginRelease(lease.id); + + expect(released.device.recoveringSince).toBeUndefined(); + expect(released.device.recoveryAttempts).toBeUndefined(); + expect(released.device.state).toBe("reclaiming"); + expect(registry.snapshot.devices[0]?.recoveringSince).toBeUndefined(); + expect(registry.snapshot.devices[0]?.recoveryAttempts).toBeUndefined(); + }); + + describe("a reclaim put off at a daemon start", () => { + async function leased() { + const clock = new FakeClock(1_000); + const state = await openTestState(); + const options = { + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "test" }, + state, + }; + const registry = await loadTestRegistry(options); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_test", + provisionDuration: 0, + spec, + }); + await registry.transitionDevice(device.id, "ready", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }); + const lease = await registry.createLease({ + deviceId: device.id, + requesterId: "agent-1", + ownerId: "agent-1", + ttlMs: 60_000, + ttlDeadline: 2_000, + }); + return { device, lease, options, registry }; + } + + it("names the lease on the reclaiming device when the release defers it, and the name survives a reload", async () => { + const { lease, options, registry } = await leased(); + + const released = await registry.beginRelease(lease.id, { deferReclaim: true }); + + expect(released.device).toMatchObject({ + deferredReclaimLeaseId: lease.id, + state: "reclaiming", + }); + expect((await loadTestRegistry(options)).snapshot.devices[0]).toMatchObject({ + deferredReclaimLeaseId: lease.id, + }); + }); + + it("names no lease on a release that does not defer", async () => { + const { lease, registry } = await leased(); + + const released = await registry.beginRelease(lease.id); + + expect(released.device).not.toHaveProperty("deferredReclaimLeaseId"); + }); + + it.each([ + ["completeReclaimWithoutPurge", "shutdown"], + ["markDeviceMissing", "deleted"], + ] as const)( + "drops the name when %s takes the device out of reclaiming", + async (exit, expected) => { + const { device, lease, registry } = await leased(); + await registry.beginRelease(lease.id, { deferReclaim: true }); + + if (exit === "markDeviceMissing") await registry.markDeviceMissing(device.id, "doctor"); + else await registry.completeReclaimWithoutPurge(device.id); + + expect(registry.snapshot.devices[0]).toMatchObject({ state: expected }); + expect(registry.snapshot.devices[0]).not.toHaveProperty("deferredReclaimLeaseId"); + }, + ); + }); +}); diff --git a/src/core/registry-transition-event.test.ts b/src/core/registry-transition-event.test.ts new file mode 100644 index 000000000..773e77941 --- /dev/null +++ b/src/core/registry-transition-event.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { loadTestRegistry } from "./testing.js"; +import { spec } from "./test-registry.js"; + +describe("Registry: the device event a transition carries", () => { + async function provisioned() { + const clock = new FakeClock(1_000); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "x" }, + }); + const device = await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_x", + provisionDuration: 0, + spec, + }); + return { device, registry }; + } + + it("names the event a transition requires when it comes without one or with another", async () => { + const { device, registry } = await provisioned(); + + await expect(registry.transitionDevice(device.id, "ready")).rejects.toThrow( + "Transition from provisioning to ready requires device.ready device event", + ); + await expect( + registry.transitionDevice(device.id, "ready", { + event: "device.shutdown", + payload: { deviceId: device.id, initiator: "test" }, + }), + ).rejects.toThrow("requires device.ready device event"); + expect(registry.snapshot.devices[0]?.state).toBe("provisioning"); + }); + + it("says a transition requires no event when it comes with one", async () => { + const { device, registry } = await provisioned(); + + await expect( + registry.transitionDevice(device.id, "quarantined", { + event: "device.ready", + payload: { bootDuration: 0, deviceId: device.id }, + }), + ).rejects.toThrow("Transition from provisioning to quarantined requires no device event"); + }); +}); diff --git a/src/core/registry.test.ts b/src/core/registry.test.ts index 63b7b6d9c..31c24ee99 100644 --- a/src/core/registry.test.ts +++ b/src/core/registry.test.ts @@ -1,9 +1,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; - import { describe, expect, it } from "vitest"; - import { EventBus } from "../bus/index.js"; import { FakeClock, NodeFilesystem, NodeSqliteOpener, NoopLogger } from "../ports/index.js"; import { type StateDatabase, openStateDatabase } from "../state/index.js"; @@ -17,95 +15,9 @@ import { registryOwner, RegistryEventError, RegistryLoadError, - UnknownDeviceError, - UnknownLeaseError, } from "./index.js"; -import { physicalInput, seededWorld, spec, totalChanges } from "./test-registry.js"; - -async function loadAfterCorruptingDevice(assignment: string): Promise { - const clock = new FakeClock(1_000); - const state = await openTestState(); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "stored" }, - state, - }); - await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_stored", - provisionDuration: 0, - spec, - }); - state.database.prepare(`UPDATE devices SET ${assignment}`).run(); - return loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "new" }, - state, - }); -} - -describe("Registry: component records", () => { - async function load(state: StateDatabase) { - const clock = new FakeClock(1_000); - return loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - state, - }); - } - - it("keeps one record per platform and version, the latest install replacing the earlier one", async () => { - const registry = await load(await openTestState()); - - await registry.recordComponent({ - installedAt: 1, - platform: "android", - receipt: { stamp: "first" }, - version: "35", - }); - await registry.recordComponent({ - installedAt: 2, - platform: "ios", - receipt: { image: "IMG" }, - version: "35", - }); - await registry.recordComponent({ - installedAt: 3, - platform: "android", - receipt: { stamp: "second" }, - version: "35", - }); - - expect(registry.snapshot.components).toEqual([ - { installedAt: 2, platform: "ios", receipt: { image: "IMG" }, version: "35" }, - { installedAt: 3, platform: "android", receipt: { stamp: "second" }, version: "35" }, - ]); - }); - - it("deletes the record of one platform and version, keeps the others, and the deletion survives a reload", async () => { - const state = await openTestState(); - const registry = await load(state); - for (const [platform, version] of [ - ["ios", "26.4"], - ["ios", "27.0"], - ["android", "26.4"], - ] as const) { - await registry.recordComponent({ installedAt: 1, platform, receipt: { version }, version }); - } - - await registry.deleteComponent("ios", "26.4"); - - const kept = [ - { installedAt: 1, platform: "ios", receipt: { version: "27.0" }, version: "27.0" }, - { installedAt: 1, platform: "android", receipt: { version: "26.4" }, version: "26.4" }, - ]; - expect(registry.snapshot.components).toEqual(kept); - expect((await load(state)).snapshot.components).toEqual(kept); - }); -}); +import { spec } from "./test-registry.js"; +import { loadAfterCorruptingDevice } from "./test-registry-corrupt-row.js"; describe("Registry", () => { it("loads an empty registry from a new state.db", async () => { @@ -731,1062 +643,4 @@ describe("Registry", () => { }), ).rejects.toThrow(RegistryEventError); }); - - it("reloads a lease written for a request with the request granted, holding the leased device, environment, lease and timing it was written with", async () => { - const clock = new FakeClock(1_000); - const suffixes = ["device", "lease"]; - const state = await openTestState(); - const options = { - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, - state, - }; - const registry = await loadTestRegistry(options); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_device", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - await registry.createLeaseRequest({ - id: "req_1", - ownerId: "agent-1", - request: { platform: "ios" }, - requesterId: "agent-1", - }); - const timing = { - estimatedBootMs: 2, - estimatedProvisionMs: 1, - estimatedReadyMs: 4, - estimatedReclaimMs: 3, - }; - - const lease = await registry.createLease({ - deviceId: device.id, - ownerId: "agent-1", - request: { environment: { SIMLOCK_UDID: "abc" }, id: "req_1", timing }, - requesterId: "agent-1", - ttlDeadline: 2_000, - ttlMs: 60_000, - }); - const reloaded = await loadTestRegistry(options); - - expect(reloaded.snapshot.leases).toEqual([lease]); - expect(reloaded.leaseRequests()).toMatchObject([ - { - grant: { - device: reloaded.snapshot.devices[0], - environment: { SIMLOCK_UDID: "abc" }, - lease, - timing, - }, - id: "req_1", - state: "granted", - }, - ]); - expect(reloaded.snapshot.devices[0]).toMatchObject({ id: device.id, state: "leased" }); - expect(lease.id).toBe("lse_lease"); - }); - - it("leaves a request already settled as it was when a lease is written for it", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "x" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_device", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - await registry.createLeaseRequest({ - id: "req_1", - ownerId: "agent-1", - request: { platform: "ios" }, - requesterId: "agent-1", - }); - await registry.settleLeaseRequest("req_1", { - failure: { code: "INTERNAL", message: "gone" }, - state: "failed", - }); - - await registry.createLease({ - deviceId: device.id, - ownerId: "agent-1", - request: { - environment: {}, - id: "req_1", - timing: { - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReadyMs: 0, - estimatedReclaimMs: 0, - }, - }, - requesterId: "agent-1", - ttlDeadline: 2_000, - ttlMs: 60_000, - }); - - expect(registry.snapshot.leases).toHaveLength(1); - expect(registry.leaseRequests()).toMatchObject([ - { failure: { code: "INTERNAL", message: "gone" }, id: "req_1", state: "failed" }, - ]); - }); - - it("enters quarantine from reclaiming, tracking attempts and the next retry deadline", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.beginRelease(lease.id); - - const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - expect(quarantined).toMatchObject({ - quarantineAttempts: 0, - quarantineNextRetryAt: 5_000, - quarantinedAt: 1_000, - state: "quarantined", - }); - await expect( - registry.enterQuarantine(device.id, 6_000, { kind: "purge-failed" }), - ).rejects.toThrow(RegistryEventError); - }); - - it("enters quarantine from provisioning, its stalled-transition entry point", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - - const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - expect(quarantined).toMatchObject({ - quarantineAttempts: 0, - quarantineNextRetryAt: 5_000, - quarantinedAt: 1_000, - state: "quarantined", - }); - }); - - it("enters quarantine from shutdown, a spent fresh device's failed-delete entry point", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - leaseIdentity: { android: "reusable", ios: "fresh" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.beginRelease(lease.id); - clock.advance(7_000); - const shutDown = await registry.completeReclaimWithoutPurge(device.id); - // Every path into `shutdown` stamps it, the reclaim's own included. - expect(shutDown.shutdownAt).toBe(8_000); - - const quarantined = await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - expect(quarantined).toMatchObject({ quarantineNextRetryAt: 5_000, state: "quarantined" }); - }); - - it("refuses to quarantine a device that is still leased", async () => { - // Quarantine is a post-release disposition: it is only ever entered from `reclaiming`, which - // a device reaches by having its lease released. A leased device reaching it would mean - // pulling a device out from under its holder -- the one thing safety rule 2 forbids outright. - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - - await expect( - registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }), - ).rejects.toThrow(RegistryEventError); - expect(registry.snapshot.devices[0]?.state).toBe("leased"); - }); - - it("records a quarantine retry failure without leaving quarantined", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.beginRelease(lease.id); - await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - const retried = await registry.recordQuarantineRetryFailure(device.id, 1, 15_000); - - expect(retried).toMatchObject({ - quarantineAttempts: 1, - quarantineNextRetryAt: 15_000, - state: "quarantined", - }); - await expect(registry.recordQuarantineRetryFailure("dev_missing", 1, 15_000)).rejects.toThrow( - UnknownDeviceError, - ); - }); - - it("recovers a quarantined device and clears its quarantine bookkeeping", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.beginRelease(lease.id); - await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - const recovered = await registry.recoverFromQuarantine(device.id, "ready"); - - expect(recovered).toEqual({ - physical: false, - createdAt: 1_000, - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - id: device.id, - lastLeaseEndedAt: 1_000, - leaseIdentity: "reusable", - readyAt: 1_000, - spec, - mode: "full", - state: "ready", - }); - await expect(registry.recoverFromQuarantine(device.id, "ready")).rejects.toThrow( - RegistryEventError, - ); - }); - - it("deletes a quarantined device and emits device.deleted with the caller's initiator", async () => { - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - const events: unknown[] = []; - bus.subscribe("device.deleted", (envelope) => events.push(envelope.payload)); - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.beginRelease(lease.id); - await registry.enterQuarantine(device.id, 5_000, { kind: "purge-failed" }); - - const deleted = await registry.deleteQuarantined(device.id, "lease-end"); - - expect(deleted.state).toBe("deleted"); - expect(events).toEqual([{ deviceId: device.id, initiator: "lease-end", physical: false }]); - }); - - it("rejects mutations for a device that is not registered", async () => { - const clock = new FakeClock(); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - - await expect(registry.transitionDevice("dev_missing", "ready")).rejects.toThrow( - UnknownDeviceError, - ); - await expect( - registry.transitionDevice("dev_missing", "deleted", { - event: "device.deleted", - payload: { deviceId: "dev_missing", initiator: "test" }, - }), - ).rejects.toThrow(UnknownDeviceError); - }); - - it("records the first recovery attempt, then keeps its recoveringSince and increments the count on retry", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - const first = await registry.markRecoveryAttempt(device.id, 1_500); - expect(first).toMatchObject({ recoveringSince: 1_500, recoveryAttempts: 1 }); - - const updated = await registry.markRecoveryAttempt(device.id, 2_000); - - expect(updated).toMatchObject({ recoveringSince: 1_500, recoveryAttempts: 2 }); - }); - - it("clears both recovery markers", async () => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.markRecoveryAttempt(device.id, 1_500); - - const cleared = await registry.clearRecovery(device.id); - - expect(cleared.recoveringSince).toBeUndefined(); - expect(cleared.recoveryAttempts).toBeUndefined(); - }); - - it("persists readyAt and shutdownAt across a reload", async () => { - const clock = new FakeClock(1_000); - const state = await openTestState(); - const options = { - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - state, - }; - const registry = await loadTestRegistry(options); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 5, deviceId: device.id }, - }); - - clock.advance(5_000); - await registry.transitionDevice(device.id, "shutdown", { - event: "device.shutdown", - payload: { deviceId: device.id, initiator: "test" }, - }); - const reloaded = await loadTestRegistry(options); - - expect(reloaded.snapshot.devices[0]?.readyAt).toBe(1_000); - expect(reloaded.snapshot.devices[0]?.shutdownAt).toBe(6_000); - }); - - it.each(["mode", "spec_mode"])( - "fails the load when a stored %s is neither slim nor full", - async (column) => { - await expect(loadAfterCorruptingDevice(`${column} = 'reduced'`)).rejects.toThrow( - RegistryLoadError, - ); - }, - ); - - it.each([ - ["no mode", spec], - ["a slim mode", { ...spec, mode: "slim" as const }], - ])("registers a new device as full, whether its spec carries %s", async (_label, deviceSpec) => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec: deviceSpec, - }); - - expect(device.spec).toEqual(deviceSpec); - expect(device.mode).toBe("full"); - }); - - it("clears recovery markers when a lease ends", async () => { - const clock = new FakeClock(1_000); - const suffixes = ["device", "lease"]; - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_device", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.markRecoveryAttempt(device.id, 1_200); - - const released = await registry.beginRelease(lease.id); - - expect(released.device.recoveringSince).toBeUndefined(); - expect(released.device.recoveryAttempts).toBeUndefined(); - expect(released.device.state).toBe("reclaiming"); - expect(registry.snapshot.devices[0]?.recoveringSince).toBeUndefined(); - expect(registry.snapshot.devices[0]?.recoveryAttempts).toBeUndefined(); - }); - - describe("a reclaim put off at a daemon start", () => { - async function leased() { - const clock = new FakeClock(1_000); - const state = await openTestState(); - const options = { - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - state, - }; - const registry = await loadTestRegistry(options); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - return { device, lease, options, registry }; - } - - it("names the lease on the reclaiming device when the release defers it, and the name survives a reload", async () => { - const { lease, options, registry } = await leased(); - - const released = await registry.beginRelease(lease.id, { deferReclaim: true }); - - expect(released.device).toMatchObject({ - deferredReclaimLeaseId: lease.id, - state: "reclaiming", - }); - expect((await loadTestRegistry(options)).snapshot.devices[0]).toMatchObject({ - deferredReclaimLeaseId: lease.id, - }); - }); - - it("names no lease on a release that does not defer", async () => { - const { lease, registry } = await leased(); - - const released = await registry.beginRelease(lease.id); - - expect(released.device).not.toHaveProperty("deferredReclaimLeaseId"); - }); - - it.each([ - ["completeReclaimWithoutPurge", "shutdown"], - ["markDeviceMissing", "deleted"], - ] as const)( - "drops the name when %s takes the device out of reclaiming", - async (exit, expected) => { - const { device, lease, registry } = await leased(); - await registry.beginRelease(lease.id, { deferReclaim: true }); - - if (exit === "markDeviceMissing") await registry.markDeviceMissing(device.id, "doctor"); - else await registry.completeReclaimWithoutPurge(device.id); - - expect(registry.snapshot.devices[0]).toMatchObject({ state: expected }); - expect(registry.snapshot.devices[0]).not.toHaveProperty("deferredReclaimLeaseId"); - }, - ); - }); - - it("writes a device the same way whether it is marked missing alone or together with its lease's end", async () => { - const marked = async (viaLease: boolean) => { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "test" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - await registry.markRecoveryAttempt(device.id, 1_500); - if (viaLease) { - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.endLeaseAndMarkDeviceMissing(lease.id, "doctor", () => undefined); - } else { - await registry.markDeviceMissing(device.id, "doctor"); - } - return registry.snapshot.devices[0]; - }; - - const viaLease = await marked(true); - const alone = await marked(false); - - expect(alone).toEqual(viaLease); - expect(viaLease).toEqual({ - physical: false, - createdAt: 1_000, - deletedAt: 1_000, - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_test", - id: "dev_test", - leaseIdentity: "reusable", - mode: "full", - readyAt: 1_000, - spec, - state: "deleted", - }); - }); - - it("keeps idChosenByRequester true on the lease of a stored grant across a reload", async () => { - const clock = new FakeClock(1_000); - const state = await openTestState(); - const options = { - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "x" }, - state, - }; - const registry = await loadTestRegistry(options); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_device", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - await registry.createLeaseRequest({ - id: "req_1", - leaseId: "ad-7f3a", - ownerId: "agent-1", - request: { platform: "ios" }, - requesterId: "agent-1", - }); - await registry.createLease({ - deviceId: device.id, - leaseId: "ad-7f3a", - ownerId: "agent-1", - request: { - environment: {}, - id: "req_1", - timing: { - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReadyMs: 0, - estimatedReclaimMs: 0, - }, - }, - requesterId: "agent-1", - ttlDeadline: 2_000, - ttlMs: 60_000, - }); - - const reloaded = await loadTestRegistry(options); - - expect(reloaded.leaseRequests()[0]?.grant?.lease).toMatchObject({ - id: "ad-7f3a", - idChosenByRequester: true, - }); - }); - - it("writes idChosenByRequester true for a lease created with a leaseId, false without one, and keeps both across a reload", async () => { - const clock = new FakeClock(1_000); - let next = 0; - const state = await openTestState(); - const options = { - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => String((next += 1)) }, - state, - }; - const registry = await loadTestRegistry(options); - const ready = async (driverDeviceId: string) => { - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId, - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - return device.id; - }; - const base = { ownerId: "agent-1", requesterId: "agent-1", ttlDeadline: 2_000, ttlMs: 60_000 }; - - const chosen = await registry.createLease({ - ...base, - deviceId: await ready("one"), - leaseId: "ad-7f3a", - }); - const generated = await registry.createLease({ ...base, deviceId: await ready("two") }); - - expect(chosen).toMatchObject({ id: "ad-7f3a", idChosenByRequester: true }); - expect(generated).toMatchObject({ - id: expect.stringMatching(/^lse_/), - idChosenByRequester: false, - }); - const reloaded = await loadTestRegistry(options); - expect(reloaded.snapshot.leases).toEqual([chosen, generated]); - }); - - it("removes a lease and marks its device deleted in one commit, announcing the lease before device.deleted, both after the commit", async () => { - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - const suffixes = ["device", "lease"]; - const state = await openTestState(); - const options = { - clock, - eventBus: bus, - idGenerator: { generate: () => suffixes.shift() ?? "unexpected" }, - state, - }; - const registry = await loadTestRegistry(options); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_device", - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - const lease = await registry.createLease({ - deviceId: device.id, - requesterId: "agent-1", - ownerId: "agent-1", - ttlMs: 60_000, - ttlDeadline: 2_000, - }); - await registry.markRecoveryAttempt(device.id, 1_500); - const operations: string[] = []; - const stored = () => { - const row = state.database - .prepare( - "SELECT (SELECT COUNT(*) FROM leases) AS leases, (SELECT state FROM devices) AS device", - ) - .get(); - return `${String(row?.leases)} leases stored, device ${String(row?.device)}`; - }; - bus.subscribe("device.deleted", () => operations.push(`device.deleted: ${stored()}`)); - - const ended = await registry.endLeaseAndMarkDeviceMissing(lease.id, "doctor", () => - operations.push(`lease announced: ${stored()}`), - ); - - expect(operations).toEqual([ - "lease announced: 0 leases stored, device deleted", - "device.deleted: 0 leases stored, device deleted", - ]); - expect(ended.lease).toEqual(lease); - expect(ended.device).toMatchObject({ id: device.id, state: "deleted" }); - expect(ended.device).not.toHaveProperty("recoveryAttempts"); - expect(registry.snapshot.leases).toEqual([]); - expect(registry.snapshot.devices).toEqual([ended.device]); - expect((await loadTestRegistry(options)).snapshot).toEqual(registry.snapshot); - expect(bus.replay().filter((entry) => entry.event === "device.deleted")).toMatchObject([ - { payload: { deviceId: device.id, initiator: "doctor" } }, - ]); - }); - - it("ends only the named lease and marks only its device deleted, leaving every other lease and device as they were, with the event's module registry", async () => { - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - let next = 0; - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => String(next++) }, - }); - const leases = []; - for (const name of ["one", "two"]) { - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: `driver_${name}`, - provisionDuration: 0, - spec, - }); - await registry.transitionDevice(device.id, "ready", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }); - leases.push( - await registry.createLease({ - deviceId: device.id, - requesterId: name, - ownerId: name, - ttlMs: 60_000, - ttlDeadline: 2_000, - }), - ); - } - const [first, second] = leases; - if (first === undefined || second === undefined) throw new Error("leases not created"); - - await registry.endLeaseAndMarkDeviceMissing(second.id, "doctor", () => undefined); - - expect(registry.snapshot.leases).toEqual([first]); - expect(registry.snapshot.devices.map((device) => [device.id, device.state])).toEqual([ - [first.deviceId, "leased"], - [second.deviceId, "deleted"], - ]); - expect(bus.replay().filter((entry) => entry.event === "device.deleted")).toMatchObject([ - { module: "registry", payload: { deviceId: second.deviceId, initiator: "doctor" } }, - ]); - }); - - it.each([ - [ - "endLeaseAndMarkDeviceMissing", - (registry: Registry, announce: () => void) => - registry.endLeaseAndMarkDeviceMissing("lse_nope", "doctor", announce), - ], - [ - "endLeaseAndMarkDeviceAbsent", - (registry: Registry, announce: () => void) => - registry.endLeaseAndMarkDeviceAbsent("lse_nope", "locked", announce), - ], - ])( - "%s refuses to end a lease it does not hold, writing nothing and announcing nothing", - async (_method, endLease) => { - const { enroll, lease, registry, state } = await seededWorld(); - await enroll("physical"); - await lease("dev_physical"); - const devicesBefore = structuredClone(registry.snapshot.devices); - const leasesBefore = structuredClone(registry.snapshot.leases); - const changesBefore = totalChanges(state); - let announced = false; - - await expect( - endLease(registry, () => { - announced = true; - }), - ).rejects.toThrow(UnknownLeaseError); - - expect(announced).toBe(false); - expect(totalChanges(state)).toBe(changesBefore); - expect(registry.snapshot.devices).toEqual(devicesBefore); - expect(registry.snapshot.leases).toEqual(leasesBefore); - }, - ); -}); - -describe("Registry: the device event a transition carries", () => { - async function provisioned() { - const clock = new FakeClock(1_000); - const registry = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => "x" }, - }); - const device = await registry.registerDevice({ - driverData: { fakeDeviceId: "fake-1" }, - driverDeviceId: "driver_x", - provisionDuration: 0, - spec, - }); - return { device, registry }; - } - - it("names the event a transition requires when it comes without one or with another", async () => { - const { device, registry } = await provisioned(); - - await expect(registry.transitionDevice(device.id, "ready")).rejects.toThrow( - "Transition from provisioning to ready requires device.ready device event", - ); - await expect( - registry.transitionDevice(device.id, "ready", { - event: "device.shutdown", - payload: { deviceId: device.id, initiator: "test" }, - }), - ).rejects.toThrow("requires device.ready device event"); - expect(registry.snapshot.devices[0]?.state).toBe("provisioning"); - }); - - it("says a transition requires no event when it comes with one", async () => { - const { device, registry } = await provisioned(); - - await expect( - registry.transitionDevice(device.id, "quarantined", { - event: "device.ready", - payload: { bootDuration: 0, deviceId: device.id }, - }), - ).rejects.toThrow("Transition from provisioning to quarantined requires no device event"); - }); -}); - -describe("Registry: quarantineCause", () => { - const purgeCause = { - kind: "purge-failed" as const, - message: "DriverCrashError: simctl erase exited 164", - }; - - async function reclaimingVirtual() { - const world = await seededWorld(); - await world.register("virtual"); - await world.lease("dev_virtual", "lse_virtual"); - await world.registry.beginRelease("lse_virtual"); - return world; - } - - async function quarantinedVirtual() { - const world = await reclaimingVirtual(); - await world.registry.enterQuarantine("dev_virtual", 5_000, purgeCause); - return world; - } - - async function quarantinedPhysical() { - const world = await seededWorld(); - await world.enroll("physical"); - await world.lease("dev_physical", "lse_physical"); - await world.registry.beginRelease("lse_physical"); - await world.registry.enterQuarantine("dev_physical", 5_000, purgeCause); - return world; - } - - const storedCauseKinds = (world: { state: StateDatabase }) => - world.state.database.prepare("SELECT quarantine_cause_kind FROM devices").all(); - - it("a quarantined record with each kind, written by enterQuarantine and loaded by a new Registry, has its cause", async () => { - const causes = [ - { kind: "purge-failed", message: "Error: purge exploded" }, - { kind: "shutdown-failed", message: "Error: shutdown exploded" }, - { kind: "delete-failed", message: "Error: delete exploded" }, - { kind: "stalled", stalledState: "provisioning" }, - { kind: "stalled", stalledState: "reclaiming" }, - ] as const; - - for (const cause of causes) { - const { registry, reload } = await reclaimingVirtual(); - if (cause.kind === "delete-failed") { - await registry.completeReclaimWithoutPurge("dev_virtual"); - } - const entered = await registry.enterQuarantine("dev_virtual", 5_000, cause); - expect(entered.quarantineCause, cause.kind).toEqual(cause); - - const reloaded = await reload(); - expect(reloaded.snapshot.devices[0], cause.kind).toMatchObject({ - quarantineCause: cause, - state: "quarantined", - }); - } - }); - - describe("every exit from quarantined removes it", () => { - const readyEvent = (id: string) => ({ - event: "device.ready" as const, - payload: { bootDuration: 0, deviceId: id }, - }); - const exits: readonly (readonly [string, (registry: Registry) => Promise])[] = [ - ["recoverFromQuarantine to ready", (r) => r.recoverFromQuarantine("dev_virtual", "ready")], - [ - "recoverFromQuarantine to shutdown", - (r) => r.recoverFromQuarantine("dev_virtual", "shutdown"), - ], - ["deleteQuarantined", (r) => r.deleteQuarantined("dev_virtual", "lease-end")], - ["markDeviceMissing", (r) => r.markDeviceMissing("dev_virtual", "doctor")], - [ - "transitionDevice to ready", - (r) => r.transitionDevice("dev_virtual", "ready", readyEvent("dev_virtual")), - ], - [ - "transitionDevice to shutdown", - (r) => - r.transitionDevice("dev_virtual", "shutdown", { - event: "device.shutdown", - payload: { deviceId: "dev_virtual", initiator: "idle" }, - }), - ], - [ - "transitionDevice to deleted", - (r) => - r.transitionDevice("dev_virtual", "deleted", { - event: "device.deleted", - payload: { deviceId: "dev_virtual", initiator: "lease-end" }, - }), - ], - ]; - - it.each(exits)("%s", async (_name, exit) => { - const world = await quarantinedVirtual(); - const { registry } = world; - expect(registry.snapshot.devices[0]).toHaveProperty("quarantineCause"); - - const result = await exit(registry); - - expect(result).not.toHaveProperty("quarantineCause"); - expect(registry.snapshot.devices[0]).not.toHaveProperty("quarantineCause"); - expect(storedCauseKinds(world)).toEqual([{ quarantine_cause_kind: null }]); - }); - - it.each([ - [ - "reenrollPhysical", - (r: Registry) => - r.reenrollPhysical("dev_physical", { ...physicalInput, apps: ["com.example.one"] }), - ], - ["unenrollPhysical", (r: Registry) => r.unenrollPhysical("dev_physical")], - ])("%s", async (_name, exit) => { - const world = await quarantinedPhysical(); - const { registry } = world; - expect(registry.snapshot.devices[0]).toHaveProperty("quarantineCause"); - - const result = await exit(registry); - - expect(result).not.toHaveProperty("quarantineCause"); - expect(registry.snapshot.devices[0]).not.toHaveProperty("quarantineCause"); - expect(storedCauseKinds(world)).toEqual([{ quarantine_cause_kind: null }]); - }); - }); }); diff --git a/src/core/test-registry-corrupt-row.ts b/src/core/test-registry-corrupt-row.ts new file mode 100644 index 000000000..b978dd7a0 --- /dev/null +++ b/src/core/test-registry-corrupt-row.ts @@ -0,0 +1,29 @@ +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { loadTestRegistry, openTestState } from "./testing.js"; +import { Registry } from "./index.js"; +import { spec } from "./test-registry.js"; + +export async function loadAfterCorruptingDevice(assignment: string): Promise { + const clock = new FakeClock(1_000); + const state = await openTestState(); + const registry = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "stored" }, + state, + }); + await registry.registerDevice({ + driverData: { fakeDeviceId: "fake-1" }, + driverDeviceId: "driver_stored", + provisionDuration: 0, + spec, + }); + state.database.prepare(`UPDATE devices SET ${assignment}`).run(); + return loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => "new" }, + state, + }); +} From e0be5196b457600a7de7f72141218d52ca5ffd48 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:32:12 +0100 Subject: [PATCH 53/83] test(core): split quarantine-coordinator.test.ts by topic --- src/core/quarantine-coordinator-cause.test.ts | 308 +++++++ ...tine-coordinator-physical-presence.test.ts | 334 +++++++ src/core/quarantine-coordinator.test.ts | 814 +----------------- src/core/test-quarantine-coordinator.ts | 158 ++++ 4 files changed, 813 insertions(+), 801 deletions(-) create mode 100644 src/core/quarantine-coordinator-cause.test.ts create mode 100644 src/core/quarantine-coordinator-physical-presence.test.ts create mode 100644 src/core/test-quarantine-coordinator.ts diff --git a/src/core/quarantine-coordinator-cause.test.ts b/src/core/quarantine-coordinator-cause.test.ts new file mode 100644 index 000000000..0e1e8dde5 --- /dev/null +++ b/src/core/quarantine-coordinator-cause.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, it, vi } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock } from "../ports/index.js"; +import { DeviceOperationClaims } from "./device-operation-claims.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import type { VirtualSpec } from "./domain.js"; +import { FakeDriver, loadTestRegistry, openTestState } from "./testing.js"; +import { QuarantineCoordinator } from "./quarantine-coordinator.js"; +import { SerializedDecision } from "./serialized-decision.js"; +import { spec, retryConfig, flush } from "./test-quarantine-coordinator.js"; + +// why: index.ts does not export the coordinator, so its retry timers, claims and give-up order are an ordering the surface cannot force. +describe("QuarantineCoordinator quarantine cause", () => { + const androidSpec: VirtualSpec = { + physical: false, + model: "Pixel 8", + osVersion: "35", + platform: "android", + }; + const PHYSICAL_ID = "00008110-001A2B3C4D5E"; + + type StartState = "provisioning" | "reclaiming" | "shutdown"; + + async function build( + options: { + readonly physical?: boolean; + readonly platform?: "ios" | "android"; + readonly state?: StartState; + readonly fresh?: boolean; + } = {}, + ) { + const platform = options.platform ?? "ios"; + const clock = new FakeClock(1_000); + const bus = new EventBus(clock); + const driver = new FakeDriver({ clock, platform }); + const state = options.state ?? "reclaiming"; + const testState = await openTestState(); + const seeding = await loadTestRegistry({ + clock, + eventBus: new EventBus(clock), + idGenerator: { generate: () => (options.physical === true ? "physical" : "virtual") }, + state: testState, + }); + let id: string; + if (options.physical === true) { + driver.setPhysicalPresence(PHYSICAL_ID, true); + id = "dev_physical"; + await seeding.enrollPhysical({ + apps: [], + class: "phone", + driverDeviceId: PHYSICAL_ID, + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: "ios", + }); + testState.database + .prepare( + "UPDATE devices SET created_at = 600, last_lease_ended_at = 700, ready_at = NULL, state = ?", + ) + .run(state); + } else { + const virtualSpec = platform === "android" ? androidSpec : spec; + const provisioned = await driver.provision(virtualSpec); + id = "dev_virtual"; + await seeding.registerDevice({ + driverData: provisioned.driverData, + driverDeviceId: provisioned.deviceId, + provisionDuration: 0, + spec: virtualSpec, + }); + testState.database.prepare("UPDATE devices SET created_at = 1, state = ?").run(state); + if (options.fresh === true) { + testState.database + .prepare("UPDATE devices SET last_lease_ended_at = 900, lease_identity = 'fresh'") + .run(); + } + } + const registry = await loadTestRegistry({ + clock, + eventBus: bus, + idGenerator: { generate: () => "id" }, + state: testState, + }); + const claims = new DeviceOperationClaims(); + const coordinator = new QuarantineCoordinator({ + claims, + clock, + config: retryConfig, + decisions: new SerializedDecision(), + drivers: new DriverCatalog([driver]), + eventBus: bus, + notifyAvailability: vi.fn(), + registry, + }); + const device = () => registry.snapshot.devices.find((candidate) => candidate.id === id); + const quarantinedPayload = () => + bus.replay().find((event) => event.event === "device.quarantined")?.payload; + return { bus, claims, clock, coordinator, device, driver, id, quarantinedPayload, registry }; + } + + function failure( + id: string, + kind: "purge-failed" | "shutdown-failed" | "delete-failed", + error: string, + ) { + return { + attemptedStrategy: kind === "purge-failed" ? ("wipe" as const) : ("delete" as const), + deviceId: id, + duration: 5, + error, + kind, + leaseId: "lease-1", + }; + } + + it.each(["ios", "android"] as const)( + "device.quarantine-abandoned and device.quarantine-stranded carry the device's own platform, for an %s device", + async (platform) => { + const abandoned = await build({ platform }); + abandoned.driver.failOn("destroy", 1, new Error("still stuck")); + abandoned.driver.failOn("destroy", 2, new Error("still stuck")); + await abandoned.coordinator.enter(failure(abandoned.id, "purge-failed", "boom")); + abandoned.clock.advance(retryConfig.retryBackoffMs); + await flush(); + abandoned.clock.advance(2_000); + await flush(); + + const stranded = await build({ platform }); + stranded.driver.failOn("destroy", 1, new Error("still stuck")); + stranded.driver.failOn("destroy", 2, new Error("still stuck")); + stranded.driver.failOn("destroy", 3, new Error("destroy blew up")); + await stranded.coordinator.enter(failure(stranded.id, "purge-failed", "boom")); + stranded.clock.advance(retryConfig.retryBackoffMs); + await flush(); + stranded.clock.advance(2_000); + await flush(); + + const payloadOf = (harness: { readonly bus: EventBus }, name: string) => + harness.bus.replay().find((event) => event.event === name)?.payload; + expect([ + payloadOf(abandoned, "device.quarantine-abandoned"), + payloadOf(stranded, "device.quarantine-stranded"), + ]).toMatchObject([{ platform }, { platform }]); + }, + ); + + it.each([ + ["purge-failed", "reclaiming"], + ["shutdown-failed", "reclaiming"], + ["delete-failed", "shutdown"], + ] as const)( + "enter writes quarantineCause kind %s and its message on the record and on device.quarantined", + async (kind, state) => { + const harness = await build({ fresh: kind !== "purge-failed", state }); + + await harness.coordinator.enter(failure(harness.id, kind, "DriverCrashError: it broke")); + + expect(harness.device()).toMatchObject({ state: "quarantined" }); + expect(harness.device()?.quarantineCause).toEqual({ + kind, + message: "DriverCrashError: it broke", + }); + expect(harness.quarantinedPayload()).toMatchObject({ + kind, + message: "DriverCrashError: it broke", + }); + expect(harness.quarantinedPayload()).not.toHaveProperty("stalledState"); + }, + ); + + it("cleans the message: control characters become spaces, it is trimmed, cut at 300 with a final ellipsis, and left out when empty; device.purge-failed keeps the full text", async () => { + const long = (length: number) => `Error: ${"x".repeat(length - 7)}`; + const cases: readonly (readonly [string, string, string | undefined])[] = [ + ["control characters", "\n Error: a\r\nb\u0000c
d \t", "Error: a b c d"], + ["300 characters", long(300), long(300)], + ["301 characters", long(301), `${long(301).slice(0, 299)}…`], + ["a high surrogate at 299", `${"x".repeat(298)}😀yyy`, `${"x".repeat(298)}…`], + ["only whitespace and controls", " \n\t\u0007 ", undefined], + ]; + + for (const [name, error, expected] of cases) { + const harness = await build(); + + await harness.coordinator.enter(failure(harness.id, "purge-failed", error)); + + const cause = harness.device()?.quarantineCause; + expect(cause, name).toBeDefined(); + expect(cause?.message, name).toBe(expected); + if (expected === undefined) expect(cause, name).toEqual({ kind: "purge-failed" }); + else expect(expected.length, name).toBeLessThanOrEqual(300); + expect(harness.quarantinedPayload(), name).toMatchObject({ kind: "purge-failed" }); + expect((harness.quarantinedPayload() as { message?: string }).message, name).toBe(expected); + const purgeFailed = harness.bus + .replay() + .find((event) => event.event === "device.purge-failed"); + expect(purgeFailed?.payload, name).toMatchObject({ error }); + } + }); + + it.each([ + ["provisioning", false], + ["reclaiming", false], + ["reclaiming", true], + ] as const)( + "enterFromStalledTransition of a device stalled %s (physical %s) writes kind stalled with its stalledState and no message, on the record and on the event", + async (stalledState, physical) => { + const harness = await build({ physical, state: stalledState }); + + await harness.coordinator.enterFromStalledTransition(harness.id, stalledState); + + expect(harness.device()?.state).toBe("quarantined"); + expect(harness.device()?.quarantineCause).toEqual({ kind: "stalled", stalledState }); + expect(harness.quarantinedPayload()).toMatchObject({ kind: "stalled", stalledState }); + expect(harness.quarantinedPayload()).not.toHaveProperty("message"); + }, + ); + + it("enterFromStalledTransition of a device in another state, the lease engine's hand-over after a failed destroy, writes kind delete-failed", async () => { + const harness = await build({ physical: false, state: "shutdown" }); + + const entered = await harness.coordinator.enterFromStalledTransition(harness.id, "shutdown"); + + expect(entered).toBe(true); + expect(harness.device()?.state).toBe("quarantined"); + expect(harness.device()?.quarantineCause).toEqual({ kind: "delete-failed" }); + expect(harness.quarantinedPayload()).toMatchObject({ kind: "delete-failed" }); + }); + + it.each(["ios", "android"] as const)( + "device.quarantined carries the %s device's platform from enter and from enterFromStalledTransition", + async (platform) => { + const entered = await build({ platform }); + await entered.coordinator.enter(failure(entered.id, "purge-failed", "Error: boom")); + const stalled = await build({ platform }); + await stalled.coordinator.enterFromStalledTransition(stalled.id, "reclaiming"); + + expect(entered.quarantinedPayload()).toMatchObject({ platform }); + expect(stalled.quarantinedPayload()).toMatchObject({ platform }); + }, + ); + + it("a stalled entry that writes nothing, because the device is claimed or has moved on, leaves no cause", async () => { + const claimed = await build(); + const held = claimed.claims.tryClaim(claimed.id, "boot"); + await claimed.coordinator.enterFromStalledTransition(claimed.id, "reclaiming"); + held?.release(); + const moved = await build({ state: "provisioning" }); + await moved.coordinator.enterFromStalledTransition(moved.id, "reclaiming"); + + for (const harness of [claimed, moved]) { + expect(harness.device()?.state).not.toBe("quarantined"); + expect(harness.device()).not.toHaveProperty("quarantineCause"); + expect(harness.bus.replay()).toEqual([]); + } + }); + + it("a failed delete retry keeps the cause written at entry", async () => { + const harness = await build({ fresh: true, state: "shutdown" }); + harness.driver.failOn("destroy", 1, new Error("still stuck")); + await harness.coordinator.enter(failure(harness.id, "delete-failed", "Error: first")); + + harness.clock.advance(retryConfig.retryBackoffMs); + await flush(); + + expect(harness.device()).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); + expect(harness.device()?.quarantineCause).toEqual({ + kind: "delete-failed", + message: "Error: first", + }); + }); + + it("a physical retry that waits for the device to be present keeps the cause written at entry", async () => { + const harness = await build({ physical: true }); + await harness.coordinator.enter(failure(harness.id, "purge-failed", "Error: first")); + harness.driver.setPhysicalPresence(PHYSICAL_ID, false); + + harness.clock.advance(retryConfig.retryBackoffMs); + await flush(); + + expect(harness.device()).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); + expect(harness.device()?.quarantineCause).toEqual({ + kind: "purge-failed", + message: "Error: first", + }); + }); + + it("a stranded give-up keeps the cause written at entry", async () => { + const harness = await build(); + harness.driver.failOn("destroy", 1, new Error("still stuck")); + harness.driver.failOn("destroy", 2, new Error("still stuck")); + harness.driver.failOn("destroy", 3, new Error("cannot destroy")); + await harness.coordinator.enter(failure(harness.id, "purge-failed", "Error: first")); + + harness.clock.advance(retryConfig.retryBackoffMs); + await flush(); + harness.clock.advance(2_000); + await flush(); + + expect(harness.bus.replay().map((event) => event.event)).toContain( + "device.quarantine-stranded", + ); + expect(harness.device()).toMatchObject({ quarantineAttempts: 2, state: "quarantined" }); + expect(harness.device()?.quarantineCause).toEqual({ + kind: "purge-failed", + message: "Error: first", + }); + }); +}); diff --git a/src/core/quarantine-coordinator-physical-presence.test.ts b/src/core/quarantine-coordinator-physical-presence.test.ts new file mode 100644 index 000000000..7484efa71 --- /dev/null +++ b/src/core/quarantine-coordinator-physical-presence.test.ts @@ -0,0 +1,334 @@ +import { describe, expect, it, vi } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, FakeSystemStats, MemoryFilesystem } from "../ports/index.js"; +import { type FakePhysicalDevice } from "./fake-driver.js"; +import { FakeDriver, loadTestRegistry, testComponentWiring, testStateOf } from "./testing.js"; +import { createCore, loadConfig } from "./index.js"; +import type { PhysicalDevice } from "./domain.js"; +import { retryConfig, createHarness, flush } from "./test-quarantine-coordinator.js"; + +// why: index.ts does not export the coordinator, so its retry timers, claims and give-up order are an ordering the surface cannot force. +describe("QuarantineCoordinator and enrollment of a quarantined physical device (ADR 0022 §2)", () => { + const FIRST = "00008110-001A2B3C4D5E"; + const SECOND = "00008110-00FF00FF00FF"; + const DEVICES: readonly FakePhysicalDevice[] = [ + { apps: [], ids: [FIRST], model: "iPhone 15 Pro", osVersion: "18.4" }, + { apps: [], ids: [SECOND], model: "iPhone 15 Pro", osVersion: "18.4" }, + ]; + + async function build() { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const driver = new FakeDriver({ clock, physicalDevices: DEVICES, platform: "ios" }); + for (const id of [FIRST, SECOND]) driver.setPhysicalPresence(id, true); + const filesystem = new MemoryFilesystem(); + const systemStats = new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * 1024 ** 3 }); + let next = 1; + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: { generate: () => `${next++}` }, + }); + const config = await loadConfig({ filesystem, systemStats }); + const core = createCore({ + clock, + config, + drivers: [driver], + eventBus, + registry, + state: testStateOf(registry), + systemStats, + ...testComponentWiring({ clock, drivers: [driver], eventBus, registry }), + }); + return { clock, config, core, driver, registry }; + } + + type Harness = Awaited>; + + async function enrollQuarantined(harness: Harness, driverId: string): Promise { + const { device } = await harness.core.enrollment.add("ios", driverId); + const lease = await harness.registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlDeadline: 61_000, + ttlMs: 60_000, + }); + await harness.registry.beginRelease(lease.id); + await harness.core.quarantine.enterFromStalledTransition(device.id, "reclaiming"); + return device.id; + } + + function reclaimedIds(harness: Harness): unknown[] { + return harness.driver.calls + .filter((call) => call.operation === "reclaim") + .map((call) => (call.arguments[0] as { deviceId: string }).deviceId); + } + + function record(harness: Harness, id: string): PhysicalDevice | undefined { + return harness.registry.snapshot.devices.find( + (device): device is PhysicalDevice => device.physical && device.id === id, + ); + } + + it("makes a re-enrolled quarantined physical device ready with no quarantine fields, and no driver reclaim call when its next retry falls due", async () => { + const harness = await build(); + const enrolled = await enrollQuarantined(harness, FIRST); + await enrollQuarantined(harness, SECOND); + const backoff = harness.config.warmPool.quarantine.retryBackoffMs; + + await harness.core.enrollment.add("ios", FIRST); + harness.clock.advance(backoff); + await flush(); + + expect(record(harness, enrolled)).toMatchObject({ id: enrolled, state: "ready" }); + expect(record(harness, enrolled)).not.toHaveProperty("quarantineNextRetryAt"); + expect(reclaimedIds(harness)).toEqual([SECOND]); + }); + + it("makes a removed quarantined physical device deleted, and no driver call when its next retry falls due", async () => { + const harness = await build(); + const removed = await enrollQuarantined(harness, FIRST); + await enrollQuarantined(harness, SECOND); + const backoff = harness.config.warmPool.quarantine.retryBackoffMs; + + await harness.core.enrollment.remove(FIRST); + harness.clock.advance(backoff); + await flush(); + + expect(record(harness, removed)?.state).toBe("deleted"); + expect(reclaimedIds(harness)).toEqual([SECOND]); + expect( + harness.driver.calls.filter( + (call) => + call.operation !== "inspectPhysical" && + (call.arguments[0] as { deviceId?: string } | undefined)?.deviceId === FIRST, + ), + ).toEqual([]); + }); + + it.each(["add", "remove"] as const)( + "answers DEVICE_BUSY reclaiming to %s of a physical device whose quarantine retry is inside its driver reclaim", + async (operation) => { + const harness = await build(); + await enrollQuarantined(harness, FIRST); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const reached = vi.fn(); + vi.spyOn(harness.driver, "reclaim").mockImplementation(async () => { + reached(); + await gate; + return { state: "ready", strategy: "uninstall" }; + }); + + harness.clock.advance(harness.config.warmPool.quarantine.retryBackoffMs); + await flush(); + expect(reached).toHaveBeenCalledTimes(1); + + const attempt = + operation === "add" + ? harness.core.enrollment.add("ios", FIRST) + : harness.core.enrollment.remove(FIRST); + await expect(attempt).rejects.toMatchObject({ + name: "DeviceBusyError", + reason: "reclaiming", + }); + release(); + await flush(); + }, + ); + + it("makes no driver call when a physical quarantine retry falls due while an add holds the claim, keeps its attempt count, and runs one retryBackoffMs later when the add was refused", async () => { + const harness = await build(); + const id = await enrollQuarantined(harness, FIRST); + const backoff = harness.config.warmPool.quarantine.retryBackoffMs; + const original = harness.driver.inspectPhysical.bind(harness.driver); + let reads = 0; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const held = vi.fn(); + vi.spyOn(harness.driver, "inspectPhysical").mockImplementation(async (identifier) => { + reads += 1; + if (reads < 2) return original(identifier); + held(); + await gate; + return { enrollable: false, message: "locked: unlock it", reason: "locked" }; + }); + const adding = harness.core.enrollment.add("ios", FIRST); + const refused = expect(adding).rejects.toMatchObject({ name: "DeviceNotEnrollableError" }); + await vi.waitFor(() => expect(held).toHaveBeenCalled()); + + harness.clock.advance(backoff); + await flush(); + + expect(reclaimedIds(harness)).toEqual([]); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); + release(); + await refused; + harness.clock.advance(backoff - 1); + await flush(); + expect(reclaimedIds(harness)).toEqual([]); + harness.clock.advance(1); + await flush(); + + expect(reclaimedIds(harness)).toEqual([FIRST]); + }); +}); + +describe("QuarantineCoordinator retry of a physical device that is not present (ADR 0022 §2)", () => { + const FIRST = "00008110-001A2B3C4D5E"; + const PROBE_INTERVAL_MS = 7_000; + const DEVICES: readonly FakePhysicalDevice[] = [ + { apps: [], ids: [FIRST], model: "iPhone 15 Pro", osVersion: "18.4" }, + ]; + + async function build() { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const driver = new FakeDriver({ clock, physicalDevices: DEVICES, platform: "ios" }); + const filesystem = new MemoryFilesystem(); + const systemStats = new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * 1024 ** 3 }); + let next = 1; + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: { generate: () => `${next++}` }, + }); + const config = await loadConfig({ + filesystem, + overrides: { health: { probeIntervalMs: PROBE_INTERVAL_MS } }, + systemStats, + }); + const core = createCore({ + clock, + config, + drivers: [driver], + eventBus, + registry, + state: testStateOf(registry), + systemStats, + ...testComponentWiring({ clock, drivers: [driver], eventBus, registry }), + }); + return { clock, config, core, driver, eventBus, registry }; + } + + type Harness = Awaited>; + + async function enrollQuarantined(harness: Harness): Promise { + const { device } = await harness.core.enrollment.add("ios", FIRST); + const lease = await harness.registry.createLease({ + deviceId: device.id, + ownerId: "agent", + requesterId: "agent", + ttlDeadline: 61_000, + ttlMs: 60_000, + }); + await harness.registry.beginRelease(lease.id); + await harness.core.quarantine.enterFromStalledTransition(device.id, "reclaiming"); + return device.id; + } + + async function advance(harness: Harness, ms: number): Promise { + harness.clock.advance(ms); + for (let round = 0; round < 5; round += 1) await flush(); + } + + function record(harness: Harness, id: string): PhysicalDevice | undefined { + return harness.registry.snapshot.devices.find( + (device): device is PhysicalDevice => device.physical && device.id === id, + ); + } + + function verbCalls(harness: Harness, operation: "reclaim" | "destroy"): unknown[] { + return harness.driver.calls.filter((call) => call.operation === operation); + } + + it("a physical retry while not present makes no reclaim or destroy call, keeps quarantineAttempts, writes quarantineNextRetryAt as now plus health.probeIntervalMs, and re-arms for that time; a virtual retry reads no presence and deletes the device", async () => { + const harness = await build(); + const id = await enrollQuarantined(harness); + const backoff = harness.config.warmPool.quarantine.retryBackoffMs; + harness.driver.setPhysicalPresence(FIRST, false, { reason: "locked" }); + + await advance(harness, backoff); + + expect(verbCalls(harness, "reclaim")).toEqual([]); + expect(verbCalls(harness, "destroy")).toEqual([]); + expect(record(harness, id)).toMatchObject({ + quarantineAttempts: 0, + quarantineNextRetryAt: 1_000 + backoff + PROBE_INTERVAL_MS, + state: "quarantined", + }); + await advance(harness, PROBE_INTERVAL_MS - 1); + expect(verbCalls(harness, "reclaim")).toEqual([]); + harness.driver.setPhysicalPresence(FIRST, true); + await advance(harness, 1); + expect(verbCalls(harness, "reclaim")).toHaveLength(1); + expect(record(harness, id)?.state).toBe("ready"); + + const clock = new FakeClock(1_000); + const driver = new FakeDriver({ clock, platform: "ios" }); + const virtual = await createHarness({ clock, driver, target: { state: "reclaiming" } }); + await virtual.coordinator.enterFromStalledTransition(virtual.target.id, "reclaiming"); + virtual.clock.advance(retryConfig.retryBackoffMs); + await flush(); + expect(virtual.driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); + expect(virtual.driver.calls.map((call) => call.operation)).not.toContain("listPresentPhysical"); + expect(virtual.registry.snapshot.virtualDevices[0]?.state).toBe("deleted"); + }); + + it("a physical retry that runs while present and fails counts one attempt", async () => { + const harness = await build(); + const id = await enrollQuarantined(harness); + harness.driver.failOn("reclaim", 1, new Error("uninstall exploded")); + harness.driver.setPhysicalPresence(FIRST, false); + await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); + expect(verbCalls(harness, "reclaim")).toEqual([]); + + harness.driver.setPhysicalPresence(FIRST, true); + await advance(harness, PROBE_INTERVAL_MS); + + expect(verbCalls(harness, "reclaim")).toHaveLength(1); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); + }); + + it("a physical retry that fails while present, then finds the device gone, keeps the one attempt it counted", async () => { + const harness = await build(); + const id = await enrollQuarantined(harness); + harness.driver.failOn("reclaim", 1, new Error("uninstall exploded")); + harness.driver.setPhysicalPresence(FIRST, true); + await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); + + harness.driver.setPhysicalPresence(FIRST, false); + await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs * 2); + + expect(verbCalls(harness, "reclaim")).toHaveLength(1); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); + }); + + it("a physical retry whose presence read rejects makes no reclaim or destroy call, keeps quarantineAttempts, and re-arms after health.probeIntervalMs", async () => { + const harness = await build(); + const id = await enrollQuarantined(harness); + harness.driver.setPhysicalPresence(FIRST, true); + const read = vi + .spyOn(harness.driver, "listPresentPhysical") + .mockRejectedValueOnce(new Error("link down")); + + await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); + + expect(read).toHaveBeenCalledTimes(1); + expect(verbCalls(harness, "reclaim")).toEqual([]); + expect(verbCalls(harness, "destroy")).toEqual([]); + expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); + await advance(harness, PROBE_INTERVAL_MS - 1); + expect(verbCalls(harness, "reclaim")).toEqual([]); + await advance(harness, 1); + expect(verbCalls(harness, "reclaim")).toHaveLength(1); + expect(record(harness, id)?.state).toBe("ready"); + }); +}); diff --git a/src/core/quarantine-coordinator.test.ts b/src/core/quarantine-coordinator.test.ts index ffa68ca4c..88c36708a 100644 --- a/src/core/quarantine-coordinator.test.ts +++ b/src/core/quarantine-coordinator.test.ts @@ -1,187 +1,26 @@ import { describe, expect, it, vi } from "vitest"; - import { EventBus, type EventEnvelope } from "../bus/index.js"; -import { - FakeClock, - FakeSystemStats, - JsonLinesLogger, - type Logger, - MemoryFilesystem, - MemoryLogSink, -} from "../ports/index.js"; +import { FakeClock, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; import { DeviceOperationClaims } from "./device-operation-claims.js"; import { DriverCatalog } from "./driver-catalog.js"; -import type { VirtualDevice, VirtualSpec } from "./domain.js"; -import { type FakePhysicalDevice } from "./fake-driver.js"; -import { - FakeDriver, - loadTestRegistry, - openTestState, - testComponentWiring, - testStateOf, -} from "./testing.js"; -import { createCore, loadConfig } from "./index.js"; -import { - QuarantineCoordinator, - type QuarantineRegistry, - type QuarantineRetryConfig, -} from "./quarantine-coordinator.js"; -import type { PhysicalDevice } from "./domain.js"; +import type { VirtualDevice } from "./domain.js"; +import { FakeDriver, loadTestRegistry, openTestState } from "./testing.js"; +import { QuarantineCoordinator } from "./quarantine-coordinator.js"; import { SerializedDecision } from "./serialized-decision.js"; - -const spec: VirtualSpec = { - physical: false, - model: "iPhone 16", - osVersion: "26.5", - platform: "ios", -}; - -const retryConfig: QuarantineRetryConfig = { - maxRetries: 2, - retryBackoffMs: 1_000, - retryBackoffMultiplier: 2, - maxRetryBackoffMs: 5_000, -}; - -class FakeRegistry implements QuarantineRegistry { - #devices: VirtualDevice[]; - readonly deletions: string[] = []; - - constructor(devices: readonly VirtualDevice[]) { - this.#devices = [...devices]; - } - - get snapshot(): { - readonly devices: readonly VirtualDevice[]; - readonly virtualDevices: readonly VirtualDevice[]; - } { - const copies = this.#devices.map((device) => ({ ...device })); - return { devices: copies, virtualDevices: copies }; - } - - async enterQuarantine(deviceId: string, nextRetryAt: number): Promise { - return this.#update(deviceId, (device) => ({ - ...device, - quarantineAttempts: 0, - quarantineNextRetryAt: nextRetryAt, - state: "quarantined", - })); - } - - async recordQuarantineRetryFailure( - deviceId: string, - attempts: number, - nextRetryAt: number, - ): Promise { - return this.#update(deviceId, (device) => ({ - ...device, - quarantineAttempts: attempts, - quarantineNextRetryAt: nextRetryAt, - })); - } - - async strandQuarantine(deviceId: string, attempts: number): Promise { - return this.#update(deviceId, (device) => { - const { quarantineNextRetryAt: _quarantineNextRetryAt, ...rest } = device; - return { ...rest, quarantineAttempts: attempts }; - }); - } - - async recoverFromQuarantine(deviceId: string, to: "ready" | "shutdown"): Promise { - return this.#update(deviceId, (device) => { - const { - quarantineAttempts: _quarantineAttempts, - quarantineNextRetryAt: _quarantineNextRetryAt, - quarantinedAt: _quarantinedAt, - ...rest - } = device; - return { ...rest, state: to }; - }); - } - - async deleteQuarantined(deviceId: string, initiator: string): Promise { - this.deletions.push(initiator); - return this.#update(deviceId, (device) => { - const { - quarantineAttempts: _quarantineAttempts, - quarantineNextRetryAt: _quarantineNextRetryAt, - quarantinedAt: _quarantinedAt, - ...rest - } = device; - return { ...rest, state: "deleted" }; - }); - } - - /** Test-only escape hatch to simulate a device leaving quarantine out of band. */ - forceState(deviceId: string, state: VirtualDevice["state"]): void { - this.#update(deviceId, (device) => ({ ...device, state })); - } - - #update(deviceId: string, apply: (device: VirtualDevice) => VirtualDevice): VirtualDevice { - const index = this.#devices.findIndex((device) => device.id === deviceId); - const current = this.#devices[index]; - if (index === -1 || current === undefined) throw new Error(`missing device: ${deviceId}`); - const updated = apply(current); - this.#devices[index] = updated; - return { ...updated }; - } -} - -function device(id: string, driverDeviceId: string): VirtualDevice { - return { - physical: false, - createdAt: 1, - driverData: {}, - driverDeviceId, - id, - mode: "full", - spec, - state: "quarantined", - }; -} - -async function createHarness( - options: { - readonly clock?: FakeClock; - readonly config?: QuarantineRetryConfig; - readonly driver?: FakeDriver; - readonly logger?: Logger; - readonly others?: readonly VirtualDevice[]; - readonly physicalDeviceReady?: (deviceId: string) => void; - readonly target?: Pick & { - readonly state?: VirtualDevice["state"]; - }; - } = {}, -) { - const clock = options.clock ?? new FakeClock(1_000); - const bus = new EventBus(clock); - const driver = options.driver ?? new FakeDriver({ clock, platform: "ios" }); - const driverDevice = await driver.provision(spec); - const target = { ...device("dev_1", driverDevice.deviceId), ...options.target }; - const registry = new FakeRegistry([...(options.others ?? []), target]); - const notifyAvailability = vi.fn(); - const claims = new DeviceOperationClaims(); - const coordinator = new QuarantineCoordinator({ - claims, - clock, - config: options.config ?? retryConfig, - decisions: new SerializedDecision(), - drivers: new DriverCatalog([driver]), - eventBus: bus, - ...(options.logger === undefined ? {} : { logger: options.logger }), - notifyAvailability, - ...(options.physicalDeviceReady === undefined - ? {} - : { physicalDeviceReady: options.physicalDeviceReady }), - registry, - }); - return { bus, claims, clock, coordinator, driver, notifyAvailability, registry, target }; -} +import { + spec, + retryConfig, + FakeRegistry, + device, + createHarness, + flush, +} from "./test-quarantine-coordinator.js"; function eventsOf(bus: EventBus): readonly EventEnvelope[] { return bus.replay(); } +// why: index.ts does not export the coordinator, so its retry timers, claims and give-up order are an ordering the surface cannot force. describe("QuarantineCoordinator", () => { it("commits quarantine entry, emits purge-failed and quarantined, and does not wake availability", async () => { const harness = await createHarness(); @@ -587,11 +426,6 @@ describe("QuarantineCoordinator", () => { }); }); -/** Lets the microtask queue (the fire-and-forget retry chain) settle after a timer fires. */ -async function flush(): Promise { - for (let count = 0; count < 100; count += 1) await Promise.resolve(); -} - describe("QuarantineCoordinator for a physical device (ADR 0022 §4)", () => { const physicalRecord = { driverDeviceId: "00008110-001A2B3C4D5E" }; @@ -874,331 +708,6 @@ describe("QuarantineCoordinator retry claim (ADR 0022 §2)", () => { }); }); -describe("QuarantineCoordinator and enrollment of a quarantined physical device (ADR 0022 §2)", () => { - const FIRST = "00008110-001A2B3C4D5E"; - const SECOND = "00008110-00FF00FF00FF"; - const DEVICES: readonly FakePhysicalDevice[] = [ - { apps: [], ids: [FIRST], model: "iPhone 15 Pro", osVersion: "18.4" }, - { apps: [], ids: [SECOND], model: "iPhone 15 Pro", osVersion: "18.4" }, - ]; - - async function build() { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const driver = new FakeDriver({ clock, physicalDevices: DEVICES, platform: "ios" }); - for (const id of [FIRST, SECOND]) driver.setPhysicalPresence(id, true); - const filesystem = new MemoryFilesystem(); - const systemStats = new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * 1024 ** 3 }); - let next = 1; - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: { generate: () => `${next++}` }, - }); - const config = await loadConfig({ filesystem, systemStats }); - const core = createCore({ - clock, - config, - drivers: [driver], - eventBus, - registry, - state: testStateOf(registry), - systemStats, - ...testComponentWiring({ clock, drivers: [driver], eventBus, registry }), - }); - return { clock, config, core, driver, registry }; - } - - type Harness = Awaited>; - - async function enrollQuarantined(harness: Harness, driverId: string): Promise { - const { device } = await harness.core.enrollment.add("ios", driverId); - const lease = await harness.registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlDeadline: 61_000, - ttlMs: 60_000, - }); - await harness.registry.beginRelease(lease.id); - await harness.core.quarantine.enterFromStalledTransition(device.id, "reclaiming"); - return device.id; - } - - function reclaimedIds(harness: Harness): unknown[] { - return harness.driver.calls - .filter((call) => call.operation === "reclaim") - .map((call) => (call.arguments[0] as { deviceId: string }).deviceId); - } - - function record(harness: Harness, id: string): PhysicalDevice | undefined { - return harness.registry.snapshot.devices.find( - (device): device is PhysicalDevice => device.physical && device.id === id, - ); - } - - it("makes a re-enrolled quarantined physical device ready with no quarantine fields, and no driver reclaim call when its next retry falls due", async () => { - const harness = await build(); - const enrolled = await enrollQuarantined(harness, FIRST); - await enrollQuarantined(harness, SECOND); - const backoff = harness.config.warmPool.quarantine.retryBackoffMs; - - await harness.core.enrollment.add("ios", FIRST); - harness.clock.advance(backoff); - await flush(); - - expect(record(harness, enrolled)).toMatchObject({ id: enrolled, state: "ready" }); - expect(record(harness, enrolled)).not.toHaveProperty("quarantineNextRetryAt"); - expect(reclaimedIds(harness)).toEqual([SECOND]); - }); - - it("makes a removed quarantined physical device deleted, and no driver call when its next retry falls due", async () => { - const harness = await build(); - const removed = await enrollQuarantined(harness, FIRST); - await enrollQuarantined(harness, SECOND); - const backoff = harness.config.warmPool.quarantine.retryBackoffMs; - - await harness.core.enrollment.remove(FIRST); - harness.clock.advance(backoff); - await flush(); - - expect(record(harness, removed)?.state).toBe("deleted"); - expect(reclaimedIds(harness)).toEqual([SECOND]); - expect( - harness.driver.calls.filter( - (call) => - call.operation !== "inspectPhysical" && - (call.arguments[0] as { deviceId?: string } | undefined)?.deviceId === FIRST, - ), - ).toEqual([]); - }); - - it.each(["add", "remove"] as const)( - "answers DEVICE_BUSY reclaiming to %s of a physical device whose quarantine retry is inside its driver reclaim", - async (operation) => { - const harness = await build(); - await enrollQuarantined(harness, FIRST); - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const reached = vi.fn(); - vi.spyOn(harness.driver, "reclaim").mockImplementation(async () => { - reached(); - await gate; - return { state: "ready", strategy: "uninstall" }; - }); - - harness.clock.advance(harness.config.warmPool.quarantine.retryBackoffMs); - await flush(); - expect(reached).toHaveBeenCalledTimes(1); - - const attempt = - operation === "add" - ? harness.core.enrollment.add("ios", FIRST) - : harness.core.enrollment.remove(FIRST); - await expect(attempt).rejects.toMatchObject({ - name: "DeviceBusyError", - reason: "reclaiming", - }); - release(); - await flush(); - }, - ); - - it("makes no driver call when a physical quarantine retry falls due while an add holds the claim, keeps its attempt count, and runs one retryBackoffMs later when the add was refused", async () => { - const harness = await build(); - const id = await enrollQuarantined(harness, FIRST); - const backoff = harness.config.warmPool.quarantine.retryBackoffMs; - const original = harness.driver.inspectPhysical.bind(harness.driver); - let reads = 0; - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const held = vi.fn(); - vi.spyOn(harness.driver, "inspectPhysical").mockImplementation(async (identifier) => { - reads += 1; - if (reads < 2) return original(identifier); - held(); - await gate; - return { enrollable: false, message: "locked: unlock it", reason: "locked" }; - }); - const adding = harness.core.enrollment.add("ios", FIRST); - const refused = expect(adding).rejects.toMatchObject({ name: "DeviceNotEnrollableError" }); - await vi.waitFor(() => expect(held).toHaveBeenCalled()); - - harness.clock.advance(backoff); - await flush(); - - expect(reclaimedIds(harness)).toEqual([]); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); - release(); - await refused; - harness.clock.advance(backoff - 1); - await flush(); - expect(reclaimedIds(harness)).toEqual([]); - harness.clock.advance(1); - await flush(); - - expect(reclaimedIds(harness)).toEqual([FIRST]); - }); -}); - -describe("QuarantineCoordinator retry of a physical device that is not present (ADR 0022 §2)", () => { - const FIRST = "00008110-001A2B3C4D5E"; - const PROBE_INTERVAL_MS = 7_000; - const DEVICES: readonly FakePhysicalDevice[] = [ - { apps: [], ids: [FIRST], model: "iPhone 15 Pro", osVersion: "18.4" }, - ]; - - async function build() { - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const driver = new FakeDriver({ clock, physicalDevices: DEVICES, platform: "ios" }); - const filesystem = new MemoryFilesystem(); - const systemStats = new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * 1024 ** 3 }); - let next = 1; - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: { generate: () => `${next++}` }, - }); - const config = await loadConfig({ - filesystem, - overrides: { health: { probeIntervalMs: PROBE_INTERVAL_MS } }, - systemStats, - }); - const core = createCore({ - clock, - config, - drivers: [driver], - eventBus, - registry, - state: testStateOf(registry), - systemStats, - ...testComponentWiring({ clock, drivers: [driver], eventBus, registry }), - }); - return { clock, config, core, driver, eventBus, registry }; - } - - type Harness = Awaited>; - - async function enrollQuarantined(harness: Harness): Promise { - const { device } = await harness.core.enrollment.add("ios", FIRST); - const lease = await harness.registry.createLease({ - deviceId: device.id, - ownerId: "agent", - requesterId: "agent", - ttlDeadline: 61_000, - ttlMs: 60_000, - }); - await harness.registry.beginRelease(lease.id); - await harness.core.quarantine.enterFromStalledTransition(device.id, "reclaiming"); - return device.id; - } - - async function advance(harness: Harness, ms: number): Promise { - harness.clock.advance(ms); - for (let round = 0; round < 5; round += 1) await flush(); - } - - function record(harness: Harness, id: string): PhysicalDevice | undefined { - return harness.registry.snapshot.devices.find( - (device): device is PhysicalDevice => device.physical && device.id === id, - ); - } - - function verbCalls(harness: Harness, operation: "reclaim" | "destroy"): unknown[] { - return harness.driver.calls.filter((call) => call.operation === operation); - } - - it("a physical retry while not present makes no reclaim or destroy call, keeps quarantineAttempts, writes quarantineNextRetryAt as now plus health.probeIntervalMs, and re-arms for that time; a virtual retry reads no presence and deletes the device", async () => { - const harness = await build(); - const id = await enrollQuarantined(harness); - const backoff = harness.config.warmPool.quarantine.retryBackoffMs; - harness.driver.setPhysicalPresence(FIRST, false, { reason: "locked" }); - - await advance(harness, backoff); - - expect(verbCalls(harness, "reclaim")).toEqual([]); - expect(verbCalls(harness, "destroy")).toEqual([]); - expect(record(harness, id)).toMatchObject({ - quarantineAttempts: 0, - quarantineNextRetryAt: 1_000 + backoff + PROBE_INTERVAL_MS, - state: "quarantined", - }); - await advance(harness, PROBE_INTERVAL_MS - 1); - expect(verbCalls(harness, "reclaim")).toEqual([]); - harness.driver.setPhysicalPresence(FIRST, true); - await advance(harness, 1); - expect(verbCalls(harness, "reclaim")).toHaveLength(1); - expect(record(harness, id)?.state).toBe("ready"); - - const clock = new FakeClock(1_000); - const driver = new FakeDriver({ clock, platform: "ios" }); - const virtual = await createHarness({ clock, driver, target: { state: "reclaiming" } }); - await virtual.coordinator.enterFromStalledTransition(virtual.target.id, "reclaiming"); - virtual.clock.advance(retryConfig.retryBackoffMs); - await flush(); - expect(virtual.driver.calls.filter((call) => call.operation === "destroy")).toHaveLength(1); - expect(virtual.driver.calls.map((call) => call.operation)).not.toContain("listPresentPhysical"); - expect(virtual.registry.snapshot.virtualDevices[0]?.state).toBe("deleted"); - }); - - it("a physical retry that runs while present and fails counts one attempt", async () => { - const harness = await build(); - const id = await enrollQuarantined(harness); - harness.driver.failOn("reclaim", 1, new Error("uninstall exploded")); - harness.driver.setPhysicalPresence(FIRST, false); - await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); - expect(verbCalls(harness, "reclaim")).toEqual([]); - - harness.driver.setPhysicalPresence(FIRST, true); - await advance(harness, PROBE_INTERVAL_MS); - - expect(verbCalls(harness, "reclaim")).toHaveLength(1); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); - }); - - it("a physical retry that fails while present, then finds the device gone, keeps the one attempt it counted", async () => { - const harness = await build(); - const id = await enrollQuarantined(harness); - harness.driver.failOn("reclaim", 1, new Error("uninstall exploded")); - harness.driver.setPhysicalPresence(FIRST, true); - await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); - - harness.driver.setPhysicalPresence(FIRST, false); - await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs * 2); - - expect(verbCalls(harness, "reclaim")).toHaveLength(1); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); - }); - - it("a physical retry whose presence read rejects makes no reclaim or destroy call, keeps quarantineAttempts, and re-arms after health.probeIntervalMs", async () => { - const harness = await build(); - const id = await enrollQuarantined(harness); - harness.driver.setPhysicalPresence(FIRST, true); - const read = vi - .spyOn(harness.driver, "listPresentPhysical") - .mockRejectedValueOnce(new Error("link down")); - - await advance(harness, harness.config.warmPool.quarantine.retryBackoffMs); - - expect(read).toHaveBeenCalledTimes(1); - expect(verbCalls(harness, "reclaim")).toEqual([]); - expect(verbCalls(harness, "destroy")).toEqual([]); - expect(record(harness, id)).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); - await advance(harness, PROBE_INTERVAL_MS - 1); - expect(verbCalls(harness, "reclaim")).toEqual([]); - await advance(harness, 1); - expect(verbCalls(harness, "reclaim")).toHaveLength(1); - expect(record(harness, id)?.state).toBe("ready"); - }); -}); - describe("QuarantineCoordinator enterFromStalledTransition under the device's reclaim claim (ADR 0022 §8)", () => { type StalledEntry = (deviceId: string, findingState: string) => Promise; @@ -1275,300 +784,3 @@ describe("QuarantineCoordinator enterFromStalledTransition under the device's re expect(failing.claims.isClaimed(failing.target.id)).toBe(false); }); }); - -describe("QuarantineCoordinator quarantine cause", () => { - const androidSpec: VirtualSpec = { - physical: false, - model: "Pixel 8", - osVersion: "35", - platform: "android", - }; - const PHYSICAL_ID = "00008110-001A2B3C4D5E"; - - type StartState = "provisioning" | "reclaiming" | "shutdown"; - - async function build( - options: { - readonly physical?: boolean; - readonly platform?: "ios" | "android"; - readonly state?: StartState; - readonly fresh?: boolean; - } = {}, - ) { - const platform = options.platform ?? "ios"; - const clock = new FakeClock(1_000); - const bus = new EventBus(clock); - const driver = new FakeDriver({ clock, platform }); - const state = options.state ?? "reclaiming"; - const testState = await openTestState(); - const seeding = await loadTestRegistry({ - clock, - eventBus: new EventBus(clock), - idGenerator: { generate: () => (options.physical === true ? "physical" : "virtual") }, - state: testState, - }); - let id: string; - if (options.physical === true) { - driver.setPhysicalPresence(PHYSICAL_ID, true); - id = "dev_physical"; - await seeding.enrollPhysical({ - apps: [], - class: "phone", - driverDeviceId: PHYSICAL_ID, - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: "ios", - }); - testState.database - .prepare( - "UPDATE devices SET created_at = 600, last_lease_ended_at = 700, ready_at = NULL, state = ?", - ) - .run(state); - } else { - const virtualSpec = platform === "android" ? androidSpec : spec; - const provisioned = await driver.provision(virtualSpec); - id = "dev_virtual"; - await seeding.registerDevice({ - driverData: provisioned.driverData, - driverDeviceId: provisioned.deviceId, - provisionDuration: 0, - spec: virtualSpec, - }); - testState.database.prepare("UPDATE devices SET created_at = 1, state = ?").run(state); - if (options.fresh === true) { - testState.database - .prepare("UPDATE devices SET last_lease_ended_at = 900, lease_identity = 'fresh'") - .run(); - } - } - const registry = await loadTestRegistry({ - clock, - eventBus: bus, - idGenerator: { generate: () => "id" }, - state: testState, - }); - const claims = new DeviceOperationClaims(); - const coordinator = new QuarantineCoordinator({ - claims, - clock, - config: retryConfig, - decisions: new SerializedDecision(), - drivers: new DriverCatalog([driver]), - eventBus: bus, - notifyAvailability: vi.fn(), - registry, - }); - const device = () => registry.snapshot.devices.find((candidate) => candidate.id === id); - const quarantinedPayload = () => - bus.replay().find((event) => event.event === "device.quarantined")?.payload; - return { bus, claims, clock, coordinator, device, driver, id, quarantinedPayload, registry }; - } - - function failure( - id: string, - kind: "purge-failed" | "shutdown-failed" | "delete-failed", - error: string, - ) { - return { - attemptedStrategy: kind === "purge-failed" ? ("wipe" as const) : ("delete" as const), - deviceId: id, - duration: 5, - error, - kind, - leaseId: "lease-1", - }; - } - - it.each(["ios", "android"] as const)( - "device.quarantine-abandoned and device.quarantine-stranded carry the device's own platform, for an %s device", - async (platform) => { - const abandoned = await build({ platform }); - abandoned.driver.failOn("destroy", 1, new Error("still stuck")); - abandoned.driver.failOn("destroy", 2, new Error("still stuck")); - await abandoned.coordinator.enter(failure(abandoned.id, "purge-failed", "boom")); - abandoned.clock.advance(retryConfig.retryBackoffMs); - await flush(); - abandoned.clock.advance(2_000); - await flush(); - - const stranded = await build({ platform }); - stranded.driver.failOn("destroy", 1, new Error("still stuck")); - stranded.driver.failOn("destroy", 2, new Error("still stuck")); - stranded.driver.failOn("destroy", 3, new Error("destroy blew up")); - await stranded.coordinator.enter(failure(stranded.id, "purge-failed", "boom")); - stranded.clock.advance(retryConfig.retryBackoffMs); - await flush(); - stranded.clock.advance(2_000); - await flush(); - - const payloadOf = (harness: { readonly bus: EventBus }, name: string) => - harness.bus.replay().find((event) => event.event === name)?.payload; - expect([ - payloadOf(abandoned, "device.quarantine-abandoned"), - payloadOf(stranded, "device.quarantine-stranded"), - ]).toMatchObject([{ platform }, { platform }]); - }, - ); - - it.each([ - ["purge-failed", "reclaiming"], - ["shutdown-failed", "reclaiming"], - ["delete-failed", "shutdown"], - ] as const)( - "enter writes quarantineCause kind %s and its message on the record and on device.quarantined", - async (kind, state) => { - const harness = await build({ fresh: kind !== "purge-failed", state }); - - await harness.coordinator.enter(failure(harness.id, kind, "DriverCrashError: it broke")); - - expect(harness.device()).toMatchObject({ state: "quarantined" }); - expect(harness.device()?.quarantineCause).toEqual({ - kind, - message: "DriverCrashError: it broke", - }); - expect(harness.quarantinedPayload()).toMatchObject({ - kind, - message: "DriverCrashError: it broke", - }); - expect(harness.quarantinedPayload()).not.toHaveProperty("stalledState"); - }, - ); - - it("cleans the message: control characters become spaces, it is trimmed, cut at 300 with a final ellipsis, and left out when empty; device.purge-failed keeps the full text", async () => { - const long = (length: number) => `Error: ${"x".repeat(length - 7)}`; - const cases: readonly (readonly [string, string, string | undefined])[] = [ - ["control characters", "\n Error: a\r\nb\u0000c
d \t", "Error: a b c d"], - ["300 characters", long(300), long(300)], - ["301 characters", long(301), `${long(301).slice(0, 299)}…`], - ["a high surrogate at 299", `${"x".repeat(298)}😀yyy`, `${"x".repeat(298)}…`], - ["only whitespace and controls", " \n\t\u0007 ", undefined], - ]; - - for (const [name, error, expected] of cases) { - const harness = await build(); - - await harness.coordinator.enter(failure(harness.id, "purge-failed", error)); - - const cause = harness.device()?.quarantineCause; - expect(cause, name).toBeDefined(); - expect(cause?.message, name).toBe(expected); - if (expected === undefined) expect(cause, name).toEqual({ kind: "purge-failed" }); - else expect(expected.length, name).toBeLessThanOrEqual(300); - expect(harness.quarantinedPayload(), name).toMatchObject({ kind: "purge-failed" }); - expect((harness.quarantinedPayload() as { message?: string }).message, name).toBe(expected); - const purgeFailed = harness.bus - .replay() - .find((event) => event.event === "device.purge-failed"); - expect(purgeFailed?.payload, name).toMatchObject({ error }); - } - }); - - it.each([ - ["provisioning", false], - ["reclaiming", false], - ["reclaiming", true], - ] as const)( - "enterFromStalledTransition of a device stalled %s (physical %s) writes kind stalled with its stalledState and no message, on the record and on the event", - async (stalledState, physical) => { - const harness = await build({ physical, state: stalledState }); - - await harness.coordinator.enterFromStalledTransition(harness.id, stalledState); - - expect(harness.device()?.state).toBe("quarantined"); - expect(harness.device()?.quarantineCause).toEqual({ kind: "stalled", stalledState }); - expect(harness.quarantinedPayload()).toMatchObject({ kind: "stalled", stalledState }); - expect(harness.quarantinedPayload()).not.toHaveProperty("message"); - }, - ); - - it("enterFromStalledTransition of a device in another state, the lease engine's hand-over after a failed destroy, writes kind delete-failed", async () => { - const harness = await build({ physical: false, state: "shutdown" }); - - const entered = await harness.coordinator.enterFromStalledTransition(harness.id, "shutdown"); - - expect(entered).toBe(true); - expect(harness.device()?.state).toBe("quarantined"); - expect(harness.device()?.quarantineCause).toEqual({ kind: "delete-failed" }); - expect(harness.quarantinedPayload()).toMatchObject({ kind: "delete-failed" }); - }); - - it.each(["ios", "android"] as const)( - "device.quarantined carries the %s device's platform from enter and from enterFromStalledTransition", - async (platform) => { - const entered = await build({ platform }); - await entered.coordinator.enter(failure(entered.id, "purge-failed", "Error: boom")); - const stalled = await build({ platform }); - await stalled.coordinator.enterFromStalledTransition(stalled.id, "reclaiming"); - - expect(entered.quarantinedPayload()).toMatchObject({ platform }); - expect(stalled.quarantinedPayload()).toMatchObject({ platform }); - }, - ); - - it("a stalled entry that writes nothing, because the device is claimed or has moved on, leaves no cause", async () => { - const claimed = await build(); - const held = claimed.claims.tryClaim(claimed.id, "boot"); - await claimed.coordinator.enterFromStalledTransition(claimed.id, "reclaiming"); - held?.release(); - const moved = await build({ state: "provisioning" }); - await moved.coordinator.enterFromStalledTransition(moved.id, "reclaiming"); - - for (const harness of [claimed, moved]) { - expect(harness.device()?.state).not.toBe("quarantined"); - expect(harness.device()).not.toHaveProperty("quarantineCause"); - expect(harness.bus.replay()).toEqual([]); - } - }); - - it("a failed delete retry keeps the cause written at entry", async () => { - const harness = await build({ fresh: true, state: "shutdown" }); - harness.driver.failOn("destroy", 1, new Error("still stuck")); - await harness.coordinator.enter(failure(harness.id, "delete-failed", "Error: first")); - - harness.clock.advance(retryConfig.retryBackoffMs); - await flush(); - - expect(harness.device()).toMatchObject({ quarantineAttempts: 1, state: "quarantined" }); - expect(harness.device()?.quarantineCause).toEqual({ - kind: "delete-failed", - message: "Error: first", - }); - }); - - it("a physical retry that waits for the device to be present keeps the cause written at entry", async () => { - const harness = await build({ physical: true }); - await harness.coordinator.enter(failure(harness.id, "purge-failed", "Error: first")); - harness.driver.setPhysicalPresence(PHYSICAL_ID, false); - - harness.clock.advance(retryConfig.retryBackoffMs); - await flush(); - - expect(harness.device()).toMatchObject({ quarantineAttempts: 0, state: "quarantined" }); - expect(harness.device()?.quarantineCause).toEqual({ - kind: "purge-failed", - message: "Error: first", - }); - }); - - it("a stranded give-up keeps the cause written at entry", async () => { - const harness = await build(); - harness.driver.failOn("destroy", 1, new Error("still stuck")); - harness.driver.failOn("destroy", 2, new Error("still stuck")); - harness.driver.failOn("destroy", 3, new Error("cannot destroy")); - await harness.coordinator.enter(failure(harness.id, "purge-failed", "Error: first")); - - harness.clock.advance(retryConfig.retryBackoffMs); - await flush(); - harness.clock.advance(2_000); - await flush(); - - expect(harness.bus.replay().map((event) => event.event)).toContain( - "device.quarantine-stranded", - ); - expect(harness.device()).toMatchObject({ quarantineAttempts: 2, state: "quarantined" }); - expect(harness.device()?.quarantineCause).toEqual({ - kind: "purge-failed", - message: "Error: first", - }); - }); -}); diff --git a/src/core/test-quarantine-coordinator.ts b/src/core/test-quarantine-coordinator.ts new file mode 100644 index 000000000..c4e6b540b --- /dev/null +++ b/src/core/test-quarantine-coordinator.ts @@ -0,0 +1,158 @@ +import { vi, type Mock } from "vitest"; +import { EventBus } from "../bus/index.js"; +import { FakeClock, type Logger } from "../ports/index.js"; +import { DeviceOperationClaims } from "./device-operation-claims.js"; +import { DriverCatalog } from "./driver-catalog.js"; +import type { VirtualDevice, VirtualSpec } from "./domain.js"; +import { FakeDriver } from "./testing.js"; +import { + QuarantineCoordinator, + type QuarantineRegistry, + type QuarantineRetryConfig, +} from "./quarantine-coordinator.js"; +import { SerializedDecision } from "./serialized-decision.js"; + +export const spec: VirtualSpec = { + physical: false, + model: "iPhone 16", + osVersion: "26.5", + platform: "ios", +}; + +export const retryConfig: QuarantineRetryConfig = { + maxRetries: 2, + retryBackoffMs: 1_000, + retryBackoffMultiplier: 2, + maxRetryBackoffMs: 5_000, +}; + +export class FakeRegistry implements QuarantineRegistry { + #devices: VirtualDevice[]; + readonly deletions: string[] = []; + constructor(devices: readonly VirtualDevice[]) { + this.#devices = [...devices]; + } + get snapshot(): { + readonly devices: readonly VirtualDevice[]; + readonly virtualDevices: readonly VirtualDevice[]; + } { + const copies = this.#devices.map((device) => ({ ...device })); + return { devices: copies, virtualDevices: copies }; + } + async enterQuarantine(deviceId: string, nextRetryAt: number): Promise { + return this.#update(deviceId, (device) => ({ + ...device, + quarantineAttempts: 0, + quarantineNextRetryAt: nextRetryAt, + state: "quarantined", + })); + } + async recordQuarantineRetryFailure( + deviceId: string, + attempts: number, + nextRetryAt: number, + ): Promise { + return this.#update(deviceId, (device) => ({ + ...device, + quarantineAttempts: attempts, + quarantineNextRetryAt: nextRetryAt, + })); + } + async strandQuarantine(deviceId: string, attempts: number): Promise { + return this.#update(deviceId, (device) => { + const { quarantineNextRetryAt: _quarantineNextRetryAt, ...rest } = device; + return { ...rest, quarantineAttempts: attempts }; + }); + } + async recoverFromQuarantine(deviceId: string, to: "ready" | "shutdown"): Promise { + return this.#update(deviceId, (device) => { + const { + quarantineAttempts: _quarantineAttempts, + quarantineNextRetryAt: _quarantineNextRetryAt, + quarantinedAt: _quarantinedAt, + ...rest + } = device; + return { ...rest, state: to }; + }); + } + async deleteQuarantined(deviceId: string, initiator: string): Promise { + this.deletions.push(initiator); + return this.#update(deviceId, (device) => { + const { + quarantineAttempts: _quarantineAttempts, + quarantineNextRetryAt: _quarantineNextRetryAt, + quarantinedAt: _quarantinedAt, + ...rest + } = device; + return { ...rest, state: "deleted" }; + }); + } + /** Test-only escape hatch to simulate a device leaving quarantine out of band. */ + forceState(deviceId: string, state: VirtualDevice["state"]): void { + this.#update(deviceId, (device) => ({ ...device, state })); + } + #update(deviceId: string, apply: (device: VirtualDevice) => VirtualDevice): VirtualDevice { + const index = this.#devices.findIndex((device) => device.id === deviceId); + const current = this.#devices[index]; + if (index === -1 || current === undefined) throw new Error(`missing device: ${deviceId}`); + const updated = apply(current); + this.#devices[index] = updated; + return { ...updated }; + } +} + +export function device(id: string, driverDeviceId: string): VirtualDevice { + return { + physical: false, + createdAt: 1, + driverData: {}, + driverDeviceId, + id, + mode: "full", + spec, + state: "quarantined", + }; +} + +export async function createHarness( + options: { + readonly clock?: FakeClock; + readonly config?: QuarantineRetryConfig; + readonly driver?: FakeDriver; + readonly logger?: Logger; + readonly others?: readonly VirtualDevice[]; + readonly physicalDeviceReady?: (deviceId: string) => void; + readonly target?: Pick & { + readonly state?: VirtualDevice["state"]; + }; + } = {}, +) { + const clock = options.clock ?? new FakeClock(1_000); + const bus = new EventBus(clock); + const driver = options.driver ?? new FakeDriver({ clock, platform: "ios" }); + const driverDevice = await driver.provision(spec); + const target = { ...device("dev_1", driverDevice.deviceId), ...options.target }; + const registry = new FakeRegistry([...(options.others ?? []), target]); + const notifyAvailability: Mock<() => void> = vi.fn(); + const claims = new DeviceOperationClaims(); + const coordinator = new QuarantineCoordinator({ + claims, + clock, + config: options.config ?? retryConfig, + decisions: new SerializedDecision(), + drivers: new DriverCatalog([driver]), + eventBus: bus, + ...(options.logger === undefined ? {} : { logger: options.logger }), + notifyAvailability, + ...(options.physicalDeviceReady === undefined + ? {} + : { physicalDeviceReady: options.physicalDeviceReady }), + registry, + }); + return { bus, claims, clock, coordinator, driver, notifyAvailability, registry, target }; +} + +/** Lets the microtask queue (the fire-and-forget retry chain) settle after a timer fires. */ +export async function flush(): Promise { + for (let count = 0; count < 100; count += 1) await Promise.resolve(); +} From 79c0a725ad0b944c6d024284e97982812f6cb59a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:32:43 +0100 Subject: [PATCH 54/83] test(core): split learned-kinds.test.ts by topic --- .../warm-pool/learned-kinds-forget.test.ts | 235 +++++++++++++++++ src/core/warm-pool/learned-kinds.test.ts | 238 +----------------- 2 files changed, 236 insertions(+), 237 deletions(-) create mode 100644 src/core/warm-pool/learned-kinds-forget.test.ts diff --git a/src/core/warm-pool/learned-kinds-forget.test.ts b/src/core/warm-pool/learned-kinds-forget.test.ts new file mode 100644 index 000000000..31ad14f36 --- /dev/null +++ b/src/core/warm-pool/learned-kinds-forget.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; +import { openMemoryState } from "../testing.js"; +import type { WarmTarget } from "./config.js"; +import type { WarmPool } from "./converger.js"; +import { LearnedKinds, warmPoolOwner } from "./learned.js"; +import { device, harness, iosListing, minute, now, waiter } from "./test-fixtures.js"; + +// why: the warm pool is a decomposed module (ADR 0063 section 2); these tests drive its converger and learned-kinds parts directly. +describe("warm pool learned kinds", () => { + it("forgetKinds forgets the kinds the match names, logs each, answers them, and asks for a pass only when a kind matched, leaving their devices to the idle rules", async () => { + const phone = device("kept", "ready", 0).spec; + const state = await openMemoryState([warmPoolOwner]); + new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); + new LearnedKinds(state).ended("l1", now - minute); + const pad = { + ...device("pad", "shutdown", 60 * minute), + spec: { + model: "iPad Pro", + osVersion: "26.0", + physical: false as const, + platform: "ios" as const, + }, + }; + const rig = harness([device("kept", "ready", minute), pad], { + catalog: async () => [iosListing], + state, + targets: [{ count: 1, model: "iPad Pro", osVersion: "26.0", platform: "ios" }], + }); + expect([...(await rig.pool.targeted())]).toEqual(["kept"]); + + expect(rig.pool.forgetKinds({ model: "iPhone 16", platform: "ios" })).toEqual([]); + await rig.pool.settle(); + expect(rig.bootCalls).toEqual([]); + expect(rig.pool.forgetKinds({ model: "iPhone 15", platform: "ios" })).toEqual([phone]); + await rig.pool.settle(); + + expect(rig.bootCalls).toHaveLength(1); + expect([...(await rig.pool.targeted())]).toEqual(["pad"]); + expect(rig.shutdownCalls).toEqual([]); + expect(new LearnedKinds(state).kinds(now)).toEqual([]); + expect( + rig.sink.records + .filter((record) => record.message.includes("by the operator")) + .map((record) => record.fields), + ).toEqual([{ model: "iPhone 15", osVersion: "26.0", platform: "ios" }]); + }); + + it("forgetKinds matches a learned baseline kind on the newest version's model, OS, mode and image tag, and never answers or deletes a baseline still counting", async () => { + const learnedBaseline = { + ...device("d1", "ready", 0).spec, + baseline: { name: "checkout", version: 1 }, + }; + const plain = device("d0", "ready", 0).spec; + const newest = { + imageTag: "google_apis", + mode: "slim", + model: "Pixel 8", + osVersion: "35", + platform: "android", + version: 2, + } as const; + const run = async ( + match: Parameters[0], + learnAfter = 1, + withPlain = false, + ) => { + const state = await openMemoryState([warmPoolOwner]); + new LearnedKinds(state).learn(learnedBaseline, "l1", now - minute, learnAfter); + if (withPlain) new LearnedKinds(state).learn(plain, "l0", now - minute); + const rig = harness([], { baselines: { checkout: newest }, state }); + const answered = rig.pool.forgetKinds(match); + return { + answered: answered.map((spec) => [spec.model, spec.baseline?.name]), + remaining: new LearnedKinds(state).kinds(now).length, + state, + }; + }; + + expect((await run({ model: "Pixel 8", platform: "android" })).answered).toEqual([ + ["Pixel 8", "checkout"], + ]); + expect((await run({ model: "Pixel 8", platform: "android" })).remaining).toBe(0); + expect((await run({ model: "iPhone 15", platform: "ios" })).answered).toEqual([]); + expect((await run({ model: "iPhone 15", platform: "ios" })).remaining).toBe(1); + // why: a plain kind that shares the baseline row's stored model is forgotten alone: the baseline's row is keyed on its name. + const beside = await run({ model: "iPhone 15", platform: "ios" }, 1, true); + expect(beside.answered).toEqual([["iPhone 15", undefined]]); + expect(beside.remaining).toBe(1); + expect( + (await run({ model: "Pixel 8", osVersion: "35", platform: "android" })).answered, + ).toHaveLength(1); + expect( + (await run({ model: "Pixel 8", osVersion: "34", platform: "android" })).answered, + ).toEqual([]); + expect( + (await run({ mode: "slim", model: "Pixel 8", platform: "android" })).answered, + ).toHaveLength(1); + expect((await run({ mode: "full", model: "Pixel 8", platform: "android" })).answered).toEqual( + [], + ); + expect( + (await run({ imageTag: "google_apis", model: "Pixel 8", platform: "android" })).answered, + ).toHaveLength(1); + expect( + (await run({ imageTag: "other", model: "Pixel 8", platform: "android" })).answered, + ).toEqual([]); + + const counting = await run({ model: "Pixel 8", platform: "android" }, 2); + expect(counting.answered).toEqual([]); + new LearnedKinds(counting.state).learn(learnedBaseline, "l2", now, 2); + expect(new LearnedKinds(counting.state).kinds(now)).toHaveLength(1); + }); + + it("forgetKinds keeps the retry backoff of a failing configured target of the forgotten kind's spec", async () => { + const phone = device("d1", "ready", 0).spec; + const state = await openMemoryState([warmPoolOwner]); + new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); + new LearnedKinds(state).ended("l1", now - minute); + const rig = harness([device("shut", "shutdown", 60 * minute)], { + boot: async () => { + throw new Error("simulator did not boot"); + }, + catalog: async () => [iosListing], + state, + targets: [{ count: 1, model: "iPhone 15", osVersion: "26.0", platform: "ios" }], + }); + await rig.pool.pass(); + await rig.pool.settle(); + expect(rig.bootCalls).toHaveLength(1); + + expect(rig.pool.forgetKinds({ model: "iPhone 15", platform: "ios" })).toEqual([phone]); + await rig.pool.settle(); + expect(rig.bootCalls).toHaveLength(1); + + rig.clock.advance(minute); + await rig.pool.settle(); + expect(rig.bootCalls).toHaveLength(2); + }); + + it("forgets a learned kind on its third failed boot or creation in a row with one warning, and a success in between, a forgetting or an operator's forget resets the count, and a configured target's kind is never forgotten", async () => { + const phone = device("d1", "ready", 0).spec; + const configured: WarmTarget = { + count: 1, + model: "iPhone 15", + osVersion: "26.0", + platform: "ios", + }; + const run = async ( + outcomes: readonly ("fail" | "ok")[], + creation: boolean, + options: { forgetAt?: number; relearnAt?: number; targets?: WarmTarget[] } = {}, + ) => { + const state = await openMemoryState([warmPoolOwner]); + new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); + new LearnedKinds(state).ended("l1", now - minute); + let attempt = 0; + const next = (): "fail" | "ok" => outcomes[attempt++] ?? "ok"; + const rig: ReturnType = harness( + creation ? [] : [device("shut", "shutdown", 60 * minute)], + { + boot: async (target) => { + if (next() === "fail") throw new Error("simulator did not boot"); + return { ...target, state: "ready" }; + }, + catalog: async () => [iosListing], + provision: async () => { + if (next() === "fail") throw new Error("creation failed"); + }, + state, + ...(options.targets === undefined ? {} : { targets: options.targets }), + }, + ); + for (let round = 0; round < outcomes.length; round += 1) { + if (round === options.forgetAt) + rig.pool.forgetKinds({ model: phone.model, platform: "ios" }); + if (round === options.relearnAt || round === options.forgetAt) { + new LearnedKinds(state).learn(phone, "l2", rig.clock.now() - 2 * minute); + new LearnedKinds(state).ended("l2", rig.clock.now() - minute); + } + await rig.pool.pass(); + await rig.pool.settle(); + rig.clock.advance(10 * minute); + rig.state.devices = rig.state.devices + .filter((item) => !item.id.startsWith("new-")) + .map((item) => ({ ...item, state: "shutdown" as const })); + } + return { + kinds: new LearnedKinds(state).kinds(rig.clock.now()).length, + warnings: rig.sink.records.filter( + (record) => + record.message === + "a warm pool learned kind failed three times in a row and was forgotten", + ), + }; + }; + + for (const creation of [false, true]) { + const third = await run(["fail", "fail", "fail"], creation); + expect(third.kinds, `creation ${creation}`).toBe(0); + expect(third.warnings).toHaveLength(1); + const reset = await run(["fail", "fail", "ok", "fail", "fail"], creation); + expect(reset.kinds, `creation ${creation}`).toBe(1); + expect(reset.warnings).toEqual([]); + const two = await run(["fail", "fail"], creation); + expect(two.kinds, `creation ${creation}`).toBe(1); + const again = await run(["fail", "fail", "fail", "fail"], creation, { relearnAt: 3 }); + expect(again.kinds, `creation ${creation}`).toBe(1); + const forgot = await run(["fail", "fail", "fail"], creation, { forgetAt: 2 }); + expect(forgot.kinds, `creation ${creation}`).toBe(1); + expect(forgot.warnings).toEqual([]); + const kept = await run(["fail", "fail", "fail"], creation, { targets: [configured] }); + expect(kept.kinds, `creation ${creation}`).toBe(1); + expect(kept.warnings).toEqual([]); + } + + const behind = { ...phone, model: "iPhone 16" }; + const waiters = harness([], { + maxConcurrentBoots: 2, + provision: async () => { + throw new Error("creation failed"); + }, + state: await openMemoryState([warmPoolOwner]), + waiting: () => [waiter({ ...phone, model: "iPhone 15" }, true), waiter(behind)], + }); + for (let round = 0; round < 3; round += 1) { + await waiters.pool.pass(); + await waiters.pool.settle(); + waiters.clock.advance(10 * minute); + } + expect(waiters.provisionCalls).toHaveLength(3); + expect( + waiters.sink.records.filter((record) => record.message.includes("was forgotten")), + ).toEqual([]); + }); +}); diff --git a/src/core/warm-pool/learned-kinds.test.ts b/src/core/warm-pool/learned-kinds.test.ts index f514eed12..5af5ed863 100644 --- a/src/core/warm-pool/learned-kinds.test.ts +++ b/src/core/warm-pool/learned-kinds.test.ts @@ -1,21 +1,11 @@ import { describe, expect, it, vi } from "vitest"; - import type { PlatformCatalog } from "../driver-catalog.js"; import type { VirtualDevice, VirtualSpec } from "../domain.js"; import type { StateDatabase } from "../../state/index.js"; import { openMemoryState } from "../testing.js"; import type { WarmTarget } from "./config.js"; -import type { WarmPool } from "./converger.js"; import { LearnedKinds, warmPoolOwner } from "./learned.js"; -import { - device, - harness, - iosListing, - minute, - now, - waiter, - type BaselineFixture, -} from "./test-fixtures.js"; +import { device, harness, iosListing, minute, now, type BaselineFixture } from "./test-fixtures.js"; // why: the warm pool is a decomposed module (ADR 0063 section 2); these tests drive its converger and learned-kinds parts directly. describe("warm pool learned kinds", () => { @@ -794,230 +784,4 @@ describe("warm pool learned kinds", () => { ).toEqual([{ lease_id: "again" }, { lease_id: "ghost" }]); }); }); - - it("forgetKinds forgets the kinds the match names, logs each, answers them, and asks for a pass only when a kind matched, leaving their devices to the idle rules", async () => { - const phone = device("kept", "ready", 0).spec; - const state = await openMemoryState([warmPoolOwner]); - new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); - new LearnedKinds(state).ended("l1", now - minute); - const pad = { - ...device("pad", "shutdown", 60 * minute), - spec: { - model: "iPad Pro", - osVersion: "26.0", - physical: false as const, - platform: "ios" as const, - }, - }; - const rig = harness([device("kept", "ready", minute), pad], { - catalog: async () => [iosListing], - state, - targets: [{ count: 1, model: "iPad Pro", osVersion: "26.0", platform: "ios" }], - }); - expect([...(await rig.pool.targeted())]).toEqual(["kept"]); - - expect(rig.pool.forgetKinds({ model: "iPhone 16", platform: "ios" })).toEqual([]); - await rig.pool.settle(); - expect(rig.bootCalls).toEqual([]); - expect(rig.pool.forgetKinds({ model: "iPhone 15", platform: "ios" })).toEqual([phone]); - await rig.pool.settle(); - - expect(rig.bootCalls).toHaveLength(1); - expect([...(await rig.pool.targeted())]).toEqual(["pad"]); - expect(rig.shutdownCalls).toEqual([]); - expect(new LearnedKinds(state).kinds(now)).toEqual([]); - expect( - rig.sink.records - .filter((record) => record.message.includes("by the operator")) - .map((record) => record.fields), - ).toEqual([{ model: "iPhone 15", osVersion: "26.0", platform: "ios" }]); - }); - - it("forgetKinds matches a learned baseline kind on the newest version's model, OS, mode and image tag, and never answers or deletes a baseline still counting", async () => { - const learnedBaseline = { - ...device("d1", "ready", 0).spec, - baseline: { name: "checkout", version: 1 }, - }; - const plain = device("d0", "ready", 0).spec; - const newest = { - imageTag: "google_apis", - mode: "slim", - model: "Pixel 8", - osVersion: "35", - platform: "android", - version: 2, - } as const; - const run = async ( - match: Parameters[0], - learnAfter = 1, - withPlain = false, - ) => { - const state = await openMemoryState([warmPoolOwner]); - new LearnedKinds(state).learn(learnedBaseline, "l1", now - minute, learnAfter); - if (withPlain) new LearnedKinds(state).learn(plain, "l0", now - minute); - const rig = harness([], { baselines: { checkout: newest }, state }); - const answered = rig.pool.forgetKinds(match); - return { - answered: answered.map((spec) => [spec.model, spec.baseline?.name]), - remaining: new LearnedKinds(state).kinds(now).length, - state, - }; - }; - - expect((await run({ model: "Pixel 8", platform: "android" })).answered).toEqual([ - ["Pixel 8", "checkout"], - ]); - expect((await run({ model: "Pixel 8", platform: "android" })).remaining).toBe(0); - expect((await run({ model: "iPhone 15", platform: "ios" })).answered).toEqual([]); - expect((await run({ model: "iPhone 15", platform: "ios" })).remaining).toBe(1); - // why: a plain kind that shares the baseline row's stored model is forgotten alone: the baseline's row is keyed on its name. - const beside = await run({ model: "iPhone 15", platform: "ios" }, 1, true); - expect(beside.answered).toEqual([["iPhone 15", undefined]]); - expect(beside.remaining).toBe(1); - expect( - (await run({ model: "Pixel 8", osVersion: "35", platform: "android" })).answered, - ).toHaveLength(1); - expect( - (await run({ model: "Pixel 8", osVersion: "34", platform: "android" })).answered, - ).toEqual([]); - expect( - (await run({ mode: "slim", model: "Pixel 8", platform: "android" })).answered, - ).toHaveLength(1); - expect((await run({ mode: "full", model: "Pixel 8", platform: "android" })).answered).toEqual( - [], - ); - expect( - (await run({ imageTag: "google_apis", model: "Pixel 8", platform: "android" })).answered, - ).toHaveLength(1); - expect( - (await run({ imageTag: "other", model: "Pixel 8", platform: "android" })).answered, - ).toEqual([]); - - const counting = await run({ model: "Pixel 8", platform: "android" }, 2); - expect(counting.answered).toEqual([]); - new LearnedKinds(counting.state).learn(learnedBaseline, "l2", now, 2); - expect(new LearnedKinds(counting.state).kinds(now)).toHaveLength(1); - }); - - it("forgetKinds keeps the retry backoff of a failing configured target of the forgotten kind's spec", async () => { - const phone = device("d1", "ready", 0).spec; - const state = await openMemoryState([warmPoolOwner]); - new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); - new LearnedKinds(state).ended("l1", now - minute); - const rig = harness([device("shut", "shutdown", 60 * minute)], { - boot: async () => { - throw new Error("simulator did not boot"); - }, - catalog: async () => [iosListing], - state, - targets: [{ count: 1, model: "iPhone 15", osVersion: "26.0", platform: "ios" }], - }); - await rig.pool.pass(); - await rig.pool.settle(); - expect(rig.bootCalls).toHaveLength(1); - - expect(rig.pool.forgetKinds({ model: "iPhone 15", platform: "ios" })).toEqual([phone]); - await rig.pool.settle(); - expect(rig.bootCalls).toHaveLength(1); - - rig.clock.advance(minute); - await rig.pool.settle(); - expect(rig.bootCalls).toHaveLength(2); - }); - - it("forgets a learned kind on its third failed boot or creation in a row with one warning, and a success in between, a forgetting or an operator's forget resets the count, and a configured target's kind is never forgotten", async () => { - const phone = device("d1", "ready", 0).spec; - const configured: WarmTarget = { - count: 1, - model: "iPhone 15", - osVersion: "26.0", - platform: "ios", - }; - const run = async ( - outcomes: readonly ("fail" | "ok")[], - creation: boolean, - options: { forgetAt?: number; relearnAt?: number; targets?: WarmTarget[] } = {}, - ) => { - const state = await openMemoryState([warmPoolOwner]); - new LearnedKinds(state).learn(phone, "l1", now - 2 * minute); - new LearnedKinds(state).ended("l1", now - minute); - let attempt = 0; - const next = (): "fail" | "ok" => outcomes[attempt++] ?? "ok"; - const rig: ReturnType = harness( - creation ? [] : [device("shut", "shutdown", 60 * minute)], - { - boot: async (target) => { - if (next() === "fail") throw new Error("simulator did not boot"); - return { ...target, state: "ready" }; - }, - catalog: async () => [iosListing], - provision: async () => { - if (next() === "fail") throw new Error("creation failed"); - }, - state, - ...(options.targets === undefined ? {} : { targets: options.targets }), - }, - ); - for (let round = 0; round < outcomes.length; round += 1) { - if (round === options.forgetAt) - rig.pool.forgetKinds({ model: phone.model, platform: "ios" }); - if (round === options.relearnAt || round === options.forgetAt) { - new LearnedKinds(state).learn(phone, "l2", rig.clock.now() - 2 * minute); - new LearnedKinds(state).ended("l2", rig.clock.now() - minute); - } - await rig.pool.pass(); - await rig.pool.settle(); - rig.clock.advance(10 * minute); - rig.state.devices = rig.state.devices - .filter((item) => !item.id.startsWith("new-")) - .map((item) => ({ ...item, state: "shutdown" as const })); - } - return { - kinds: new LearnedKinds(state).kinds(rig.clock.now()).length, - warnings: rig.sink.records.filter( - (record) => - record.message === - "a warm pool learned kind failed three times in a row and was forgotten", - ), - }; - }; - - for (const creation of [false, true]) { - const third = await run(["fail", "fail", "fail"], creation); - expect(third.kinds, `creation ${creation}`).toBe(0); - expect(third.warnings).toHaveLength(1); - const reset = await run(["fail", "fail", "ok", "fail", "fail"], creation); - expect(reset.kinds, `creation ${creation}`).toBe(1); - expect(reset.warnings).toEqual([]); - const two = await run(["fail", "fail"], creation); - expect(two.kinds, `creation ${creation}`).toBe(1); - const again = await run(["fail", "fail", "fail", "fail"], creation, { relearnAt: 3 }); - expect(again.kinds, `creation ${creation}`).toBe(1); - const forgot = await run(["fail", "fail", "fail"], creation, { forgetAt: 2 }); - expect(forgot.kinds, `creation ${creation}`).toBe(1); - expect(forgot.warnings).toEqual([]); - const kept = await run(["fail", "fail", "fail"], creation, { targets: [configured] }); - expect(kept.kinds, `creation ${creation}`).toBe(1); - expect(kept.warnings).toEqual([]); - } - - const behind = { ...phone, model: "iPhone 16" }; - const waiters = harness([], { - maxConcurrentBoots: 2, - provision: async () => { - throw new Error("creation failed"); - }, - state: await openMemoryState([warmPoolOwner]), - waiting: () => [waiter({ ...phone, model: "iPhone 15" }, true), waiter(behind)], - }); - for (let round = 0; round < 3; round += 1) { - await waiters.pool.pass(); - await waiters.pool.settle(); - waiters.clock.advance(10 * minute); - } - expect(waiters.provisionCalls).toHaveLength(3); - expect( - waiters.sink.records.filter((record) => record.message.includes("was forgotten")), - ).toEqual([]); - }); }); From bb161f23a96306d07603dbfd38b4f9ccb776294f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:28:51 +0100 Subject: [PATCH 55/83] test(http): split app-leases.test.ts by topic --- src/http/app-leases-files.test.ts | 595 ++++++++++ src/http/app-leases-lease-routes.test.ts | 746 ++++++++++++ src/http/app-leases.test.ts | 1341 +--------------------- 3 files changed, 1345 insertions(+), 1337 deletions(-) create mode 100644 src/http/app-leases-files.test.ts create mode 100644 src/http/app-leases-lease-routes.test.ts diff --git a/src/http/app-leases-files.test.ts b/src/http/app-leases-files.test.ts new file mode 100644 index 000000000..dc3a015c4 --- /dev/null +++ b/src/http/app-leases-files.test.ts @@ -0,0 +1,595 @@ +import { createHash, randomFillSync } from "node:crypto"; +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createLeaseFiles, type LeaseFilesLimits } from "../lease-files/index.js"; +import { NodeFilesystem } from "../ports/index.js"; +import { agentAuth, buildHarness, operatorAuth, otherAgentAuth } from "./test-app.js"; +import { makeLease, sequenceIdGenerator } from "./test-fakes.js"; + +const filesRoots: string[] = []; + +afterEach(async () => { + await Promise.all(filesRoots.splice(0).map((root) => rm(root, { force: true, recursive: true }))); +}); + +// why: a worker's HTTP app over a real lease-files store in a temporary folder, with lease `lse_1` held by `tok_agent`. +async function filesRig( + options: { + readonly limits?: LeaseFilesLimits; + readonly filesystem?: NodeFilesystem; + readonly diskPressureThreshold?: number; + } = {}, +) { + const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-lease-files-")); + filesRoots.push(dataDirectory); + const rig = buildHarness({ + leaseFiles: ({ clock, logger }) => + createLeaseFiles({ + clock, + dataDirectory, + diskPressureThreshold: options.diskPressureThreshold ?? 0, + filesystem: options.filesystem ?? new NodeFilesystem(), + findGrant: (leaseId) => + leaseId === "lse_1" ? { grantedAt: 1_000, id: leaseId, platform: "ios" } : undefined, + idGenerator: sequenceIdGenerator("up"), + logger, + ...(options.limits === undefined ? {} : { limits: options.limits }), + }), + }); + rig.registry.leases = [makeLease({ id: "lse_1", ownerId: "tok_agent" })]; + const put = (name: string, body: BodyInit, headers: Record = agentAuth) => + rig.app.request(`/v1/leases/lse_1/files/${name}`, { body, headers, method: "PUT" }); + const get = (name: string, headers: Record = agentAuth) => + rig.app.request(`/v1/leases/lse_1/files/${name}`, { headers }); + const remove = (name: string, headers: Record = agentAuth) => + rig.app.request(`/v1/leases/lse_1/files/${name}`, { headers, method: "DELETE" }); + const list = (headers: Record = agentAuth) => + rig.app.request("/v1/leases/lse_1/files", { headers }); + const leaseFolder = join( + dataDirectory, + "lease-files", + `${Buffer.from("lse_1").toString("hex")}-1000`, + ); + return { ...rig, dataDirectory, get, leaseFolder, list, put, remove }; +} + +// why: what a lease folder holds, temporary upload files included, so "nothing stored" means nothing on disk. +async function folderEntries(folder: string): Promise { + return readdir(folder).catch(() => []); +} + +// why: a request body the test feeds by hand: `push` resolves once the upload has taken the bytes and asks for more, so a test acts only after the store has seen them. +function liveBody() { + let controller!: ReadableStreamDefaultController; + let waiting: (() => void)[] = []; + let pulled!: () => void; + const firstPull = new Promise((resolve) => { + pulled = resolve; + }); + const stream = new ReadableStream( + { + start(c) { + controller = c; + }, + pull() { + pulled(); + for (const resume of waiting.splice(0)) resume(); + }, + }, + { highWaterMark: 0 }, + ); + return { + fail: (error: Error) => controller.error(error), + finish: () => controller.close(), + firstPull, + push: (bytes: Uint8Array) => + new Promise((resolve) => { + controller.enqueue(bytes); + waiting = [...waiting, resolve]; + }), + stream, + }; +} + +// why: a request the code under test must answer; when it never does, the assertion on `undefined` fails by name where an await would time the whole test out. +function answered( + response: Promise | Response, + waitMs = 2_000, +): Promise { + return Promise.race([ + Promise.resolve(response), + new Promise((resolve) => setTimeout(() => resolve(undefined), waitMs)), + ]); +} + +// why: counts the files opened and not yet closed, so a test sees a response that left one open. +class CountingFilesystem extends NodeFilesystem { + open = 0; + override async openFile(path: string) { + const file = await super.openFile(path); + this.open += 1; + let closed = false; + const release = () => { + if (!closed) this.open -= 1; + closed = true; + }; + return { + size: file.size, + read: (range?: { readonly start: number; readonly end: number }) => { + const stream = file.read(range); + const reader = stream.getReader(); + return new ReadableStream({ + async pull(controller) { + const piece = await reader.read(); + if (piece.done) { + release(); + controller.close(); + } else controller.enqueue(piece.value); + }, + async cancel(reason) { + release(); + await reader.cancel(reason); + }, + }); + }, + close: async () => { + release(); + await file.close(); + }, + }; + } +} + +class LowDiskFilesystem extends NodeFilesystem { + override diskFree(): Promise { + return Promise.resolve(5 * 1024 ** 3); + } +} + +// why: comparing megabytes with `toEqual` walks them one by one and takes seconds; their digests say the same. +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +describe("a lease's files on a worker", () => { + it("a lease holder PUTs a file and gets 201 with its name and size; a GET returns the same bytes", async () => { + const { get, put } = await filesRig(); + const patterned = (length: number): Uint8Array => { + const bytes = new Uint8Array(length); + randomFillSync(bytes.subarray(0, Math.min(length, 1 << 20))); + for (let at = 1 << 20; at < length; at += 1 << 20) bytes.copyWithin(at, 0, 1 << 20); + return bytes; + }; + const cases: [string, Uint8Array][] = [ + ["fixture.tar", patterned(3 * 1024 * 1024)], + ["fixture.tar", patterned(1000)], + ["empty.bin", new Uint8Array(0)], + ["big.bin", patterned(70_000_000)], + ]; + + for (const [name, bytes] of cases) { + const response = await put(name, bytes); + expect(response.status).toBe(201); + expect(await response.json()).toEqual({ name, size: bytes.length }); + const download = await get(name); + expect(download.status).toBe(200); + expect(sha256(new Uint8Array(await download.arrayBuffer()))).toBe(sha256(bytes)); + } + }); + + it("a lease holder PUTs a bad file name and gets 400 BAD_REQUEST with the name rule", async () => { + const { get, put, app } = await filesRig(); + const accepted = [ + "fixture.tar", + "app-release.apk", + "Documents.tar.deflate", + "A", + "a".repeat(128), + ]; + const refused = [ + ["a".repeat(129), "a".repeat(129)], + [".hidden", ".hidden"], + ["-rf", "-rf"], + ["my%20file.tar", "my file.tar"], + ["a%2Fb.tar", "a/b.tar"], + ["za%C5%BC%C3%B3%C5%82%C4%87.txt", "zażółć.txt"], + ] as const; + + for (const name of accepted) expect((await put(name, "x")).status).toBe(201); + for (const [encoded, decoded] of refused) { + for (const response of [await put(encoded, "x"), await get(encoded)]) { + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: { + code: "BAD_REQUEST", + message: `File ${decoded} must be 1 to 128 letters, digits, '.', '_' or '-', starting with a letter or digit`, + }, + }); + } + } + expect((await app.request("/v1/leases/lse_1/files/..", { headers: agentAuth })).status).toBe( + 404, + ); + }); + + it("a lease holder PUTs past a limit and gets the limit's code, with nothing stored and the previous file kept", async () => { + const limits = { fileBytes: 10, leaseBytes: 100, silenceMs: 330_000 }; + const big = new Uint8Array(11); + const rig = await filesRig({ limits }); + const refusal = async (response: Response) => + (await response.json()) as { error: { code: string; message: string } }; + const text = async (response: Response) => + new TextDecoder().decode(await response.arrayBuffer()); + + expect((await rig.put("kept.bin", "1234567890")).status).toBe(201); + const tooBig = await rig.put("kept.bin", big); + expect(tooBig.status).toBe(413); + expect((await refusal(tooBig)).error).toEqual({ + code: "FILE_TOO_LARGE", + message: "File kept.bin is over the 2 GiB limit for one file", + name: "kept.bin", + }); + expect(await text(await rig.get("kept.bin"))).toBe("1234567890"); + expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); + + let pulled = false; + const unread = () => + new ReadableStream( + { + pull(c) { + pulled = true; + c.close(); + }, + }, + { highWaterMark: 0 }, + ); + const declared = await rig.app.request("/v1/leases/lse_1/files/declared.bin", { + body: unread(), + duplex: "half", + headers: { ...agentAuth, "content-length": "11" }, + method: "PUT", + } as RequestInit); + expect(declared.status).toBe(413); + expect((await refusal(declared)).error.code).toBe("FILE_TOO_LARGE"); + expect(pulled).toBe(false); + expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); + + const roomy = { fileBytes: 1000, leaseBytes: 100, silenceMs: 330_000 }; + const full = await filesRig({ limits: roomy }); + await mkdir(full.leaseFolder, { recursive: true }); + await writeFile(join(full.leaseFolder, "my file.png"), new Uint8Array(60)); + expect((await full.put("fits.bin", new Uint8Array(40))).status).toBe(201); + await rm(join(full.leaseFolder, "fits.bin")); + for (const headers of [agentAuth, { ...agentAuth, "content-length": "50" }]) { + const over = await full.put("over.bin", new Uint8Array(50), headers); + expect(over.status).toBe(413); + expect((await refusal(over)).error).toEqual({ + code: "LEASE_FILES_FULL", + leaseId: "lse_1", + message: "Lease lse_1 holds 60 bytes of files; this upload would pass the 4 GiB limit", + usedBytes: 60, + }); + expect(await folderEntries(full.leaseFolder)).toEqual(["my file.png"]); + } + pulled = false; + const declaredFull = await full.app.request("/v1/leases/lse_1/files/over.bin", { + body: unread(), + duplex: "half", + headers: { ...agentAuth, "content-length": "50" }, + method: "PUT", + } as RequestInit); + expect(declaredFull.status).toBe(413); + expect((await refusal(declaredFull)).error.code).toBe("LEASE_FILES_FULL"); + expect(pulled).toBe(false); + + const together = await filesRig({ limits: roomy }); + const first = liveBody(); + const running = together.app.request("/v1/leases/lse_1/files/a.bin", { + body: first.stream, + duplex: "half", + headers: agentAuth, + method: "PUT", + } as RequestInit); + await first.push(new Uint8Array(60)); + const second = await together.put("b.bin", new Uint8Array(60)); + expect(second.status).toBe(413); + expect((await refusal(second)).error.code).toBe("LEASE_FILES_FULL"); + first.finish(); + expect((await running).status).toBe(201); + expect(await folderEntries(together.leaseFolder)).toEqual(["a.bin"]); + + const chunked = await filesRig({ limits: roomy }); + const chunks = liveBody(); + const chunkedRequest = chunked.app.request("/v1/leases/lse_1/files/chunks.bin", { + body: chunks.stream, + duplex: "half", + headers: agentAuth, + method: "PUT", + } as RequestInit); + for (let piece = 0; piece < 3; piece += 1) { + await Promise.race([chunks.push(new Uint8Array(30)), chunkedRequest]); + } + chunks.finish(); + expect((await chunkedRequest).status).toBe(201); + expect(await folderEntries(chunked.leaseFolder)).toEqual(["chunks.bin"]); + + const replacing = await filesRig({ limits: roomy }); + expect((await replacing.put("x.bin", new Uint8Array(80))).status).toBe(201); + expect((await replacing.put("x.bin", new Uint8Array(80))).status).toBe(201); + + const lowDisk = await filesRig({ + diskPressureThreshold: 10 * 1024 ** 3, + filesystem: new LowDiskFilesystem(), + }); + const noDisk = await lowDisk.put("fixture.tar", "x"); + expect(noDisk.status).toBe(422); + expect((await refusal(noDisk)).error).toEqual({ + availableBytes: 5 * 1024 ** 3, + code: "INSUFFICIENT_DISK_SPACE", + message: + "Not enough free disk space to store fixture.tar: the worker keeps 10.0 GiB free, only 5.0 GiB available", + platform: "ios", + requiredBytes: 10 * 1024 ** 3, + }); + expect(await folderEntries(lowDisk.leaseFolder)).toEqual([]); + }); + + it("a lease holder whose PUT is cut or goes silent keeps the previous file or none; a second PUT to a name being uploaded gets 409 FILE_BUSY", async () => { + const rig = await filesRig(); + const uploading = (name: string, body: ReadableStream) => + rig.app.request(`/v1/leases/lse_1/files/${name}`, { + body, + duplex: "half", + headers: agentAuth, + method: "PUT", + } as RequestInit); + const text = async (response: Response) => + new TextDecoder().decode(await response.arrayBuffer()); + const bytes = new TextEncoder().encode("new"); + + const cutFresh = liveBody(); + const cutting = uploading("fresh.bin", cutFresh.stream); + await cutFresh.push(bytes); + cutFresh.fail(new Error("socket hang up")); + expect(await answered(cutting)).toBeDefined(); + expect(await folderEntries(rig.leaseFolder)).toEqual([]); + + expect((await rig.put("kept.bin", "old")).status).toBe(201); + const cutKept = liveBody(); + const cuttingKept = uploading("kept.bin", cutKept.stream); + await cutKept.push(bytes); + cutKept.fail(new Error("socket hang up")); + expect(await answered(cuttingKept)).toBeDefined(); + expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); + expect(await text(await rig.get("kept.bin"))).toBe("old"); + + const silent = liveBody(); + const silentRequest = uploading("kept.bin", silent.stream); + await silent.push(bytes); + rig.clock.advance(329_999); + expect(await answered(silentRequest, 50)).toBeUndefined(); + rig.clock.advance(1); + const stalled = await answered(silentRequest); + expect(stalled?.status).toBe(408); + expect(await stalled?.json()).toEqual({ + error: { + code: "UPLOAD_STALLED", + message: "No bytes of kept.bin arrived for 330 s", + name: "kept.bin", + }, + }); + expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); + expect(await text(await rig.get("kept.bin"))).toBe("old"); + + const mute = liveBody(); + const muteRequest = uploading("mute.bin", mute.stream); + await mute.firstPull; + rig.clock.advance(329_999); + expect(await answered(muteRequest, 50)).toBeUndefined(); + rig.clock.advance(1); + expect((await answered(muteRequest))?.status).toBe(408); + + const moving = liveBody(); + const movingRequest = uploading("moving.bin", moving.stream); + for (const chunk of ["a", "b", "c"]) { + await moving.push(new TextEncoder().encode(chunk)); + rig.clock.advance(200_000); + } + moving.finish(); + expect((await movingRequest).status).toBe(201); + expect(await text(await rig.get("moving.bin"))).toBe("abc"); + + for (const [firstName, secondName] of [ + ["x.bin", "x.bin"], + ["A.tar", "a.tar"], + ] as const) { + const running = liveBody(); + const runningRequest = uploading(firstName, running.stream); + await running.push(bytes); + const refused = await rig.put(secondName, "other"); + expect(refused.status).toBe(409); + expect(await refused.json()).toEqual({ + error: { + code: "FILE_BUSY", + message: `File ${secondName} is being uploaded by another request`, + name: secondName, + }, + }); + running.finish(); + expect((await runningRequest).status).toBe(201); + } + }); + + it("a lease holder lists the lease's files and deletes one", async () => { + const { leaseFolder, list, put, remove } = await filesRig(); + const files = async () => ((await (await list()).json()) as { files: unknown[] }).files; + + expect((await list()).status).toBe(200); + expect(await files()).toEqual([]); + + expect((await put("b.txt", "bb")).status).toBe(201); + expect((await put("a.txt", "a")).status).toBe(201); + await mkdir(join(leaseFolder, "sub")); + await writeFile(join(leaseFolder, "sub", "inner.txt"), "x"); + await writeFile(join(leaseFolder, "my file.png"), "x"); + expect(await files()).toEqual([ + { name: "a.txt", size: 1 }, + { name: "b.txt", size: 2 }, + ]); + + const removed = await remove("a.txt"); + expect(removed.status).toBe(204); + expect(await files()).toEqual([{ name: "b.txt", size: 2 }]); + + const unknown = await remove("a.txt"); + expect(unknown.status).toBe(404); + expect(await unknown.json()).toEqual({ + error: { code: "UNKNOWN_FILE", message: "No file a.txt in this lease", name: "a.txt" }, + }); + }); + + it("a lease holder's HEAD answers a GET's headers with no body and leaves no file open", async () => { + const filesystem = new CountingFilesystem(); + const { app, put } = await filesRig({ filesystem }); + expect((await put("head.bin", "twelve bytes")).status).toBe(201); + const head = await app.request("/v1/leases/lse_1/files/head.bin", { + headers: agentAuth, + method: "HEAD", + }); + expect(head.status).toBe(200); + expect(head.headers.get("content-length")).toBe("12"); + expect(await head.text()).toBe(""); + expect(filesystem.open).toBe(0); + }); + + it("a lease holder's file names match ignoring case", async () => { + const { get, leaseFolder, list, put, remove } = await filesRig(); + const text = async (response: Response) => + new TextDecoder().decode(await response.arrayBuffer()); + const names = async () => + ((await (await list()).json()) as { files: { name: string }[] }).files.map( + (file) => file.name, + ); + + expect((await put("A.tar", "upper")).status).toBe(201); + expect(await text(await get("a.tar"))).toBe("upper"); + + expect((await put("a.tar", "lower")).status).toBe(201); + expect(await folderEntries(leaseFolder)).toEqual(["a.tar"]); + expect(await names()).toEqual(["a.tar"]); + expect(await text(await get("A.TAR"))).toBe("lower"); + + expect((await remove("A.TAR")).status).toBe(204); + expect(await names()).toEqual([]); + + await writeFile(join(leaseFolder, "A.tar"), "first"); + await writeFile(join(leaseFolder, "a.tar"), "second"); + if ((await folderEntries(leaseFolder)).length === 2) { + expect(await text(await get("a.tar"))).toBe("second"); + expect(await text(await get("A.TAR"))).toBe("first"); + expect(await names()).toEqual(["A.tar", "a.tar"]); + + expect((await remove("a.tar")).status).toBe(204); + expect(await names()).toEqual([]); + expect(await folderEntries(leaseFolder)).toEqual([]); + expect((await get("a.tar")).status).toBe(404); + expect((await get("A.tar")).status).toBe(404); + } + }); + + it("a lease holder GETs a file and gets it whole or the range it asked for", async () => { + const { get, put } = await filesRig(); + const thousand = Uint8Array.from({ length: 1000 }, (_, index) => index % 251); + expect((await put("thousand.bin", thousand)).status).toBe(201); + expect((await put("empty.bin", "")).status).toBe(201); + const ranged = (name: string, range: string) => get(name, { ...agentAuth, range }); + const bytesOf = async (response: Response) => new Uint8Array(await response.arrayBuffer()); + + const whole = await get("thousand.bin"); + expect(whole.status).toBe(200); + expect(Object.fromEntries(whole.headers)).toMatchObject({ + "accept-ranges": "bytes", + "cache-control": "no-store", + "content-length": "1000", + "content-type": "application/octet-stream", + "simlock-content-length": "1000", + }); + expect(await bytesOf(whole)).toEqual(thousand); + + const partial = [ + ["bytes=0-99", 0, 99], + ["bytes=900-", 900, 999], + ["bytes=-100", 900, 999], + ["bytes=990-5000", 990, 999], + ["bytes=-5000", 0, 999], + ] as const; + for (const [range, start, end] of partial) { + const response = await ranged("thousand.bin", range); + expect(response.status).toBe(206); + expect(response.headers.get("content-range")).toBe(`bytes ${start}-${end}/1000`); + expect(response.headers.get("content-length")).toBe(String(end - start + 1)); + expect(response.headers.get("simlock-content-length")).toBe(String(end - start + 1)); + expect(await bytesOf(response)).toEqual(thousand.subarray(start, end + 1)); + } + + for (const [name, range] of [ + ["thousand.bin", "bytes=1000-"], + ["thousand.bin", "bytes=-0"], + ["empty.bin", "bytes=-5"], + ] as const) { + const response = await ranged(name, range); + expect(response.status).toBe(416); + expect(response.headers.get("content-range")).toBe( + `bytes */${name === "empty.bin" ? 0 : 1000}`, + ); + } + + for (const range of ["bytes=0-1,5-9", "bytes=9-5", "items=0-9", "bytes=abc", "bytes=-"]) { + const response = await ranged("thousand.bin", range); + expect(response.status).toBe(200); + expect(await bytesOf(response)).toEqual(thousand); + } + + const unknown = await get("missing.bin"); + expect(unknown.status).toBe(404); + expect(await unknown.json()).toEqual({ + error: { + code: "UNKNOWN_FILE", + message: "No file missing.bin in this lease", + name: "missing.bin", + }, + }); + }); + + it("a caller that does not hold the lease gets 404 UNKNOWN_LEASE on every file route, and an operator is served", async () => { + const { app, put, list, get } = await filesRig(); + expect((await put("mine.bin", "mine")).status).toBe(201); + const routes = (leaseId: string, headers: Record) => [ + () => + app.request(`/v1/leases/${leaseId}/files/mine.bin`, { body: "x", headers, method: "PUT" }), + () => app.request(`/v1/leases/${leaseId}/files/mine.bin`, { headers }), + () => app.request(`/v1/leases/${leaseId}/files`, { headers }), + () => app.request(`/v1/leases/${leaseId}/files/mine.bin`, { headers, method: "DELETE" }), + ]; + + for (const [leaseId, headers] of [ + ["lse_1", otherAgentAuth], + ["lse_unknown", agentAuth], + ] as const) { + for (const response of await Promise.all(routes(leaseId, headers).map((call) => call()))) { + expect(response.status).toBe(404); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe( + "UNKNOWN_LEASE", + ); + } + } + expect(await (await list()).json()).toEqual({ files: [{ name: "mine.bin", size: 4 }] }); + expect(new TextDecoder().decode(await (await get("mine.bin")).arrayBuffer())).toBe("mine"); + + const statuses: number[] = []; + for (const call of routes("lse_1", operatorAuth)) statuses.push((await call()).status); + expect(statuses).toEqual([201, 200, 200, 204]); + }); +}); diff --git a/src/http/app-leases-lease-routes.test.ts b/src/http/app-leases-lease-routes.test.ts new file mode 100644 index 000000000..31b687520 --- /dev/null +++ b/src/http/app-leases-lease-routes.test.ts @@ -0,0 +1,746 @@ +import { describe, expect, it } from "vitest"; +import { UnknownLeaseError } from "../core/index.js"; +import { DispatchError } from "../daemon/dispatcher.js"; +import { + agentAuth, + type App, + buildHarness, + createLeaseRequest, + operatorAuth, + postLeaseRequest, + readSseFrames, + settle, +} from "./test-app.js"; +import { makeDevice, makeGrant, makeLease, waitForDispatch } from "./test-fakes.js"; + +describe("lease routes", () => { + it("GET /v1/leases/:id returns the acquisition-shaped lease object", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + expect(response.status).toBe(200); + const body = (await response.json()) as { lease: Record }; + expect(body.lease).toMatchObject({ + dataPlane: null, + deviceId: "dev_1", + device: "iPhone 17 Pro", + id: "lse_1", + platform: "ios", + udid: "ABCD-1234", + }); + expect(body.lease).not.toHaveProperty("worker"); + expect(body.lease).not.toHaveProperty("workerId"); + }); + + it("GET /v1/leases/:id on a gateway builds the lease from its own worker's device, not another worker's with the same device id", async () => { + const device = (workerId: string, udid: string) => ({ + ...makeDevice({ driverDeviceId: udid, id: "dev_1" }), + workerId, + }); + const { app, dispatcher } = buildHarness({ + registry: { + snapshot: { devices: [device("wrk_a", "UDID-A"), device("wrk_b", "UDID-B")] }, + }, + }); + dispatcher.handlers["lease.get"] = () => ({ + lease: { + ...makeLease({ deviceId: "dev_1", id: "wrk_b.lse_1", ownerId: "tok_agent" }), + worker: { id: "wrk_b", label: "mac-b" }, + workerId: "wrk_b", + }, + }); + + const response = await app.request("/v1/leases/wrk_b.lse_1", { headers: agentAuth }); + + expect(response.status).toBe(200); + expect(((await response.json()) as { lease: unknown }).lease).toMatchObject({ + deviceId: "dev_1", + id: "wrk_b.lse_1", + udid: "UDID-B", + worker: { id: "wrk_b", label: "mac-b" }, + workerId: "wrk_b", + }); + }); + + it("GET /v1/leases/:id keeps a simulator's mode on the lease object", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1", mode: "slim" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + + expect(((await response.json()) as { lease: unknown }).lease).toMatchObject({ mode: "slim" }); + }); + + it("GET /v1/leases/:id answers a lease on a physical device with no mode, on a worker", async () => { + const { app, registry } = buildHarness(); + registry.devices = [ + { + createdAt: 0, + driverData: undefined, + driverDeviceId: "00008110-001A2B3C4D5E", + enrolledApps: [], + id: "dev_phys", + physical: true, + spec: { + class: "phone", + model: "iPhone 15 Pro", + osVersion: "18.4", + physical: true, + platform: "ios", + }, + state: "leased", + }, + ]; + registry.leases = [makeLease({ deviceId: "dev_phys", id: "lse_p", ownerId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_p", { headers: agentAuth }); + + expect(response.status).toBe(200); + const { lease } = (await response.json()) as { lease: Record }; + expect(lease).toMatchObject({ + device: "iPhone 15 Pro", + deviceId: "dev_phys", + os: "18.4", + udid: "00008110-001A2B3C4D5E", + }); + expect(lease).not.toHaveProperty("mode"); + }); + + it("GET /v1/leases/:id answers a lease on a physical device with no mode, on a gateway", async () => { + const { app, dispatcher } = buildHarness({ + registry: { + snapshot: { + devices: [ + { + driverDeviceId: "00008110-001A2B3C4D5E", + id: "dev_phys", + spec: { model: "iPhone 15 Pro", osVersion: "18.4", platform: "ios" }, + workerId: "wrk_a", + }, + ], + }, + }, + }); + dispatcher.handlers["lease.get"] = () => ({ + lease: { + ...makeLease({ deviceId: "dev_phys", id: "wrk_a.lse_p", ownerId: "tok_agent" }), + worker: { id: "wrk_a" }, + workerId: "wrk_a", + }, + }); + + const response = await app.request("/v1/leases/wrk_a.lse_p", { headers: agentAuth }); + + expect(response.status).toBe(200); + const { lease } = (await response.json()) as { lease: Record }; + expect(lease).toMatchObject({ deviceId: "dev_phys", udid: "00008110-001A2B3C4D5E" }); + expect(lease).not.toHaveProperty("mode"); + }); + + it("names an Android emulator's adb serial as address, beside its AVD name as udid, on the lease object the POST, the request poll and GET /v1/leases/:id answer", async () => { + const emulator = { + address: "emulator-5554", + driverDeviceId: "simlock_3f9a2c1e", + id: "dev_and", + spec: { model: "Pixel 8", osVersion: "35", physical: false, platform: "android" }, + } as const; + const { app, dispatcher, registry } = buildHarness(); + const posted = postLeaseRequest(app, { device: "Pixel 8", platform: "android" }); + const call = await waitForDispatch(dispatcher, "lease.request"); + call.resolve(makeGrant({ device: emulator, lease: { deviceId: "dev_and", id: "lse_and" } })); + const created = (await (await posted).json()) as { request: { id: string; lease: unknown } }; + registry.devices = [makeDevice(emulator)]; + registry.leases = [makeLease({ deviceId: "dev_and", id: "lse_and", ownerId: "tok_agent" })]; + + const polled = await app.request(`/v1/lease-requests/${created.request.id}`, { + headers: agentAuth, + }); + const read = await app.request("/v1/leases/lse_and", { headers: agentAuth }); + + const named = { address: "emulator-5554", udid: "simlock_3f9a2c1e" }; + expect({ + get: ((await read.json()) as { lease: unknown }).lease, + poll: ((await polled.json()) as { request: { lease: unknown } }).request.lease, + post: created.request.lease, + }).toMatchObject({ get: named, poll: named, post: named }); + }); + + it("POST /v1/leases/:id/renew drains buffered device-health notices", async () => { + const { app, dispatcher, eventBus, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + dispatcher.handlers["lease.renew"] = (input) => ({ + deviceId: "dev_1", + grantedAt: 1_000, + id: (input as { leaseId: string }).leaseId, + ownerId: "tok_agent", + requesterId: "tok_agent", + ttlDeadline: 2_000 + ((input as { ttlMs?: number }).ttlMs ?? 900_000), + }); + + eventBus.emit( + "device.crash-detected", + { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, + "test", + ); + eventBus.emit( + "device.recovered", + { attempts: 1, deviceId: "dev_1", duration: 500, leaseId: "lse_1", platform: "ios" }, + "test", + ); + + const response = await app.request("/v1/leases/lse_1/renew", { + body: JSON.stringify({ ttlMs: 120_000 }), + headers: { ...agentAuth, "content-type": "application/json" }, + method: "POST", + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { leaseId: string; notices: unknown[] }; + expect(body.leaseId).toBe("lse_1"); + expect(body.notices).toEqual([ + { event: "device_unhealthy" }, + { attempts: 1, event: "device_recovered" }, + ]); + }); + + it.each([ + ["renew", "/v1/leases/lse_1/renew", '{"ttlMs":60000}', "lease.renew"], + ["exec", "/v1/leases/lse_1/exec", '{"args":["list"],"tool":"simctl"}', "device.exec"], + [ + "install", + "/v1/components/install", + '{"platform":"android","version":"35"}', + "component.install", + ], + ])( + "a caller sends the %s body without a JSON content type and gets 415 UNSUPPORTED_MEDIA_TYPE; with application/json it reaches the dispatcher", + async (_route, path, body, operation) => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.renew"] = () => ({ + deviceId: "dev_1", + grantedAt: 1_000, + id: "lse_1", + ownerId: "tok_agent", + requesterId: "tok_agent", + ttlDeadline: 2_000, + }); + dispatcher.handlers["device.exec"] = () => ({ exitCode: 0 }); + dispatcher.handlers["component.install"] = () => ({ + component: "35", + outcome: "installed", + platform: "android", + version: "35", + }); + const send = (contentType: string | undefined) => + app.request(path, { + body, + headers: { + ...agentAuth, + ...(contentType === undefined ? {} : { "content-type": contentType }), + }, + method: "POST", + }); + + for (const contentType of [undefined, "text/plain"]) { + const refused = await send(contentType); + expect(refused.status).toBe(415); + expect(await refused.json()).toEqual({ + error: { + code: "UNSUPPORTED_MEDIA_TYPE", + message: "Content-Type must be application/json", + }, + }); + } + expect(dispatcher.calls.filter((call) => call.operation === operation)).toEqual([]); + + const accepted = await send("application/json"); + expect(accepted.status).toBe(200); + expect(dispatcher.calls.filter((call) => call.operation === operation)).toHaveLength(1); + }, + ); + + it("a caller sends a bad wait or renew body and gets 400 with today's message; a renew with no body, an empty body under any content type or an extra key renews", async () => { + const { app, dispatcher } = buildHarness(); + const { id } = await createLeaseRequest(app, dispatcher); + const renewals: unknown[] = []; + dispatcher.handlers["lease.renew"] = (input) => { + renewals.push(input); + return { + deviceId: "dev_1", + grantedAt: 1_000, + id: "lse_1", + ownerId: "tok_agent", + requesterId: "tok_agent", + ttlDeadline: 2_000, + }; + }; + const renew = ( + body: string | undefined, + contentType: string | undefined = "application/json", + ) => + app.request("/v1/leases/lse_1/renew", { + ...(body === undefined ? {} : { body }), + headers: { + ...agentAuth, + ...(contentType === undefined ? {} : { "content-type": contentType }), + }, + method: "POST", + }); + const refusedWait = "wait must be a non-negative number of seconds"; + + const refusals: [string, Response | Promise, string][] = [ + [ + "wait=-1", + app.request(`/v1/lease-requests/${id}?wait=-1`, { headers: agentAuth }), + refusedWait, + ], + [ + "wait=abc", + app.request(`/v1/lease-requests/${id}?wait=abc`, { headers: agentAuth }), + refusedWait, + ], + [ + "wait=-1 on an unknown request", + app.request("/v1/lease-requests/unknown?wait=-1", { headers: agentAuth }), + refusedWait, + ], + ["ttlMs a string", renew('{"ttlMs":"60s"}'), "ttlMs must be a number"], + ["ttlMs negative", renew('{"ttlMs":-1}'), "ttlMs must be a positive number"], + ["ttlMs zero", renew('{"ttlMs":0}'), "ttlMs must be a positive number"], + ["a number", renew("5"), "Body must be a JSON object"], + ["an array", renew("[]"), "Body must be a JSON object"], + ["malformed JSON", renew("{"), "Invalid JSON body"], + ]; + for (const [name, pending, message] of refusals) { + const response = await pending; + expect(response.status, name).toBe(400); + expect(await response.json()).toEqual({ error: { code: "BAD_REQUEST", message } }); + } + expect(renewals).toEqual([]); + + for (const response of [ + await renew(undefined, undefined), + await renew("", "application/json"), + await renew("", "text/plain"), + await renew("", "application/x-www-form-urlencoded"), + await renew('{"ttlMs":60000,"note":"x"}'), + ]) { + expect(response.status).toBe(200); + } + expect(renewals).toEqual([ + { leaseId: "lse_1" }, + { leaseId: "lse_1" }, + { leaseId: "lse_1" }, + { leaseId: "lse_1" }, + { leaseId: "lse_1", ttlMs: 60_000 }, + ]); + }); + + it("404s a renew for an unknown lease", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.renew"] = () => { + throw new UnknownLeaseError("lse-missing"); + }; + + const response = await app.request("/v1/leases/lse-missing/renew", { + headers: agentAuth, + method: "POST", + }); + expect(response.status).toBe(404); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe( + "UNKNOWN_LEASE", + ); + }); + + it("403s a renew for another requester's still-live lease -- dispatched directly through lease.renew's own ownsLease hook, not lease.list's 404-for-everything filter (S6)", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.renew"] = () => { + throw new DispatchError("FORBIDDEN", "Not authorized for lease.renew"); + }; + + const response = await app.request("/v1/leases/lse_other/renew", { + headers: agentAuth, + method: "POST", + }); + expect(response.status).toBe(403); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + }); + + it("streams live lease notices over SSE, ending on lease_lost -- fed by the owner-routed fact bus, not a direct eventBus subscription", async () => { + const { app, eventBus, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + + const streamResponse = await app.request("/v1/leases/lse_1/events", { headers: agentAuth }); + const framesPromise = readSseFrames(streamResponse, 2); + await Promise.resolve(); + eventBus.emit( + "device.crash-detected", + { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, + "test", + ); + eventBus.emit( + "lease.expired", + { deviceId: "dev_1", leaseId: "lse_1", ownerId: "tok_agent" }, + "test", + ); + const frames = await framesPromise; + + expect(frames.map((frame) => frame.event)).toEqual(["device_unhealthy", "lease_lost"]); + }); + + describe("POST /v1/leases/:id/exec", () => { + function postExec( + app: App, + body: Record = { args: ["list"], tool: "simctl" }, + ) { + return app.request("/v1/leases/lse_1/exec", { + body: JSON.stringify(body), + headers: { ...agentAuth, "content-type": "application/json" }, + method: "POST", + }); + } + + it("streams a chunk per output event and ends with the command's exit code", async () => { + const { app, dispatcher } = buildHarness(); + + const responsePromise = postExec(app); + const call = await waitForDispatch(dispatcher, "device.exec"); + expect(call.input).toEqual({ args: ["list"], leaseId: "lse_1", tool: "simctl" }); + + // why: The stream opens when the process starts, not when it first speaks (ADR 0005 §19e). + // This chunk lands in the window between the two, which is the only thing the route's + // handoff buffer is for: it must still arrive, and still arrive first. + call.session.onStarted?.(); + call.session.onOutput?.("stdout", "one"); + const response = await responsePromise; + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/event-stream"); + + const framesPromise = readSseFrames(response, 3); + await Promise.resolve(); + call.session.onOutput?.("stderr", "two"); + call.resolve({ exitCode: 5 }); + const frames = await framesPromise; + + expect(frames.map((frame) => frame.event)).toEqual(["output", "output", "exit"]); + expect(frames.map((frame) => frame.data)).toEqual([ + { chunk: "one", stream: "stdout" }, + { chunk: "two", stream: "stderr" }, + { exitCode: 5 }, + ]); + }); + + it("answers a failure that lands before any output with its own status, not a stream", async () => { + // why: An SSE response cannot take back its status code, so ownership and the refusal list + // have to be answered ahead of the stream -- 403 here, exactly as `renew`/`release` do. + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["device.exec"] = () => { + throw new DispatchError("FORBIDDEN", "Not authorized for device.exec"); + }; + + const response = await postExec(app); + expect(response.status).toBe(403); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + }); + + it("reports a failure that lands after output began as a terminal error event", async () => { + const { app, dispatcher } = buildHarness(); + + const responsePromise = postExec(app); + const call = await waitForDispatch(dispatcher, "device.exec"); + call.session.onStarted?.(); + call.session.onOutput?.("stdout", "working"); + const response = await responsePromise; + + const framesPromise = readSseFrames(response, 2); + await Promise.resolve(); + call.reject(new DispatchError("EXEC_TIMEOUT", "exceeded exec.timeoutMs (1000ms)")); + const frames = await framesPromise; + + expect(frames.map((frame) => frame.event)).toEqual(["output", "error"]); + expect(frames.at(-1)?.data).toMatchObject({ error: { code: "EXEC_TIMEOUT" } }); + }); + + it("forwards requesterId unchanged for both an operator and an agent token, leaving the FORBIDDEN decision to device.exec's own authorize hook", async () => { + // why: Requester identity is never client-declared over HTTP: the token is the identity, + // except for an operator token proxying for someone (ADR 0005 §19b/§27). Round 4, F4: this route used to decide on its own + // that an agent token may not supply `requesterId` at all, which left that rule living + // only in the HTTP transport (ADR 0003 §2 puts a contract operation's answer in the + // dispatcher). The rule moved into `device.exec`'s own `authorize` hook -- exercised + // directly in `dispatcher-exec.test.ts` -- so this route now forwards `requesterId` exactly + // the same way for either token and merely surfaces whatever the dispatcher answers. + const { app, dispatcher } = buildHarness(); + + const asOperator = app.request("/v1/leases/lse_1/exec", { + body: JSON.stringify({ args: ["list"], requesterId: "agent-7", tool: "simctl" }), + headers: { ...operatorAuth, "content-type": "application/json" }, + method: "POST", + }); + const operatorCall = await waitForDispatch(dispatcher, "device.exec"); + expect(operatorCall.input).toEqual({ + args: ["list"], + leaseId: "lse_1", + requesterId: "agent-7", + tool: "simctl", + }); + operatorCall.session.onStarted?.(); + operatorCall.resolve({ exitCode: 0 }); + await asOperator; + + // why: Forwarded exactly the same way for an agent token -- the route makes no distinction + // any more. `FakeDispatcher` has no `authorize` hook of its own to reject this with, so + // the rejection is simulated here (`DispatchError("FORBIDDEN", ...)`, precisely what the + // real dispatcher's `authorize` hook now throws for this case) to prove this route still + // surfaces it as a 403 rather than requiring a route-level check to produce one. + const asAgent = app.request("/v1/leases/lse_1/exec", { + body: JSON.stringify({ args: ["list"], requesterId: "agent-7", tool: "simctl" }), + headers: { ...agentAuth, "content-type": "application/json" }, + method: "POST", + }); + const agentCall = await waitForDispatch(dispatcher, "device.exec", 1); + expect(agentCall.input).toEqual({ + args: ["list"], + leaseId: "lse_1", + requesterId: "agent-7", + tool: "simctl", + }); + agentCall.reject(new DispatchError("FORBIDDEN", "Not authorized for device.exec")); + const asAgentResponse = await asAgent; + expect(asAgentResponse.status).toBe(403); + expect(((await asAgentResponse.json()) as { error: { code: string } }).error.code).toBe( + "FORBIDDEN", + ); + + // why: A `null` for either optional is the documented example's own spelling, and normalizes + // to an omitted field rather than a 400 -- including for an agent, which is not + // "supplying a requesterId". + const withNulls = app.request("/v1/leases/lse_1/exec", { + body: JSON.stringify({ args: ["list"], requesterId: null, stdin: null, tool: "simctl" }), + headers: { ...agentAuth, "content-type": "application/json" }, + method: "POST", + }); + const nullCall = await waitForDispatch(dispatcher, "device.exec", 2); + expect(nullCall.input).toEqual({ args: ["list"], leaseId: "lse_1", tool: "simctl" }); + nullCall.session.onStarted?.(); + nullCall.resolve({ exitCode: 0 }); + await withNulls; + }); + + it("dispatches an unwrapped tool rather than rejecting it, and 400s a malformed body", async () => { + // why: Which wrappers exist is the drivers' answer: `bash` is a tool this host does not wrap + // (`UNKNOWN_PASSTHROUGH_TOOL`, 422, from the driver catalog -- see the dispatcher suite), + // not a malformed request. `400` here is about the body's shape only. + const { app, dispatcher } = buildHarness(); + + const unwrapped = postExec(app, { args: [], tool: "bash" }); + const call = await waitForDispatch(dispatcher, "device.exec"); + expect(call.input).toEqual({ args: [], leaseId: "lse_1", tool: "bash" }); + call.reject( + new DispatchError("UNKNOWN_PASSTHROUGH_TOOL", "No driver provides a bash passthrough"), + ); + const refused = await unwrapped; + expect(refused.status).toBe(422); + expect(((await refused.json()) as { error: { code: string } }).error.code).toBe( + "UNKNOWN_PASSTHROUGH_TOOL", + ); + + for (const body of [ + { args: [] }, + { args: "list", tool: "simctl" }, + { args: [], tool: "simctl", unexpected: true }, + { args: [], requesterId: "a\u001b[2Jb", tool: "simctl" }, + { args: [], requesterId: "x".repeat(257), tool: "simctl" }, + { args: [], stdin: "A".repeat(8_388_609), tool: "simctl" }, + ]) { + const response = await postExec(app, body); + expect(response.status).toBe(400); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe( + "BAD_REQUEST", + ); + } + expect(dispatcher.calls.filter((call) => call.operation === "device.exec")).toHaveLength(1); + }); + + it("opens the stream when the process starts, before it has written anything", async () => { + // why: ADR 0005 §19e: a command that prints nothing for nine minutes still gets its `200` and + // its keepalives, and an `EXEC_TIMEOUT` therefore always arrives as the stream's terminal + // event rather than as a status the client can no longer be given. + // + // C3 (round 3 review): this only proves that half of the route's *own* logic given a + // dispatcher that calls `onStarted` when told to (`call.session.onStarted?.()` below is + // this test driving that by hand) -- it says nothing about when a real dispatcher actually + // calls it, and cannot notice a dispatcher that stops calling it for a genuinely silent, + // long-running command. That is the gateway's own job (`FleetForwarding.exec` in + // `src/gateway/fleet-forwarding.ts`); the invariant this comment describes is guarded end + // to end there, not here -- see `fleet-coordinator.test.ts`'s "relays the worker's own + // started push for a silent command that never answers (ADR §19a/§19b/§19e)". + const { app, clock, dispatcher } = buildHarness(); + + const responsePromise = postExec(app); + const call = await waitForDispatch(dispatcher, "device.exec"); + call.session.onStarted?.(); + const response = await responsePromise; + expect(response.status).toBe(200); + + const framesPromise = readSseFrames(response, 1); + await Promise.resolve(); + clock.advance(15_000); + call.reject(new DispatchError("EXEC_TIMEOUT", "exceeded exec.timeoutMs (600000ms)")); + const frames = await framesPromise; + expect(frames).toEqual([ + { data: { error: { code: "EXEC_TIMEOUT", message: expect.any(String) } }, event: "error" }, + ]); + }); + + it("stalls the command when the client stops reading, instead of buffering for it", async () => { + // why: ADR 0005 §19e end to end. The route hands each chunk's SSE write back through + // `onOutput`, the process runner awaits it, so a client that opened the stream and does + // not read stops the command at its own pipe -- the alternative being an unbounded + // in-process queue for exactly the client least able to consume it. + const { app, dispatcher } = buildHarness(); + + const responsePromise = postExec(app); + const call = await waitForDispatch(dispatcher, "device.exec"); + call.session.onStarted?.(); + const response = await responsePromise; + const reader = response.body?.getReader(); + await Promise.resolve(); + + const settled: number[] = []; + let pending: number | undefined; + for (let index = 0; index < 64 && pending === undefined; index += 1) { + const delivery = Promise.resolve(call.session.onOutput?.("stdout", `chunk-${index}`)).then( + () => settled.push(index), + ); + // why: A delivery that has not settled after the microtask queue drains is the stream + // refusing more, which is the state this test exists to reach. + await new Promise((resolve) => setTimeout(resolve, 0)); + if (!settled.includes(index)) { + pending = index; + void delivery; + } + } + expect(pending, "the unread stream never applied backpressure").toBeDefined(); + + // why: Reading again lets the queued writes through, and the stalled delivery completes -- + // the command resumes rather than having been dropped. Each read is bounded: once the + // queue is empty a further read would block on the very producer this test has stalled. + for (let attempt = 0; attempt < 8 && !settled.includes(pending as number); attempt += 1) { + await Promise.race([reader?.read(), new Promise((resolve) => setTimeout(resolve, 20))]); + await new Promise((resolve) => setTimeout(resolve, 0)); + } + expect(settled).toContain(pending); + + await reader?.cancel(); + call.resolve({ exitCode: 0 }); + await responsePromise; + }); + + it("survives a client that disconnects while the command keeps writing", async () => { + // why: The command runs on deliberately, so chunks keep arriving at a route with nowhere to + // put them. They go nowhere and the request still settles normally -- what "nowhere" + // means precisely, and that nothing accumulates, is `output-relay.test.ts`. + const { app, dispatcher } = buildHarness(); + + const responsePromise = postExec(app); + const call = await waitForDispatch(dispatcher, "device.exec"); + call.session.onStarted?.(); + const response = await responsePromise; + const reader = response.body?.getReader(); + await Promise.resolve(); + await reader?.cancel(); + await Promise.resolve(); + + expect(call.session.onOutput).toBeDefined(); + const chunk = "x".repeat(1_024); + for (let index = 0; index < 1_000; index += 1) call.session.onOutput?.("stdout", chunk); + + // why: The relay is dropped, not merely detached: a chunk after the disconnect is discarded + // outright, which is observable as the delivery no longer being something to wait for. + // (Without the `relay.drop()` in the route's unsubscribe, this returns the pending write + // of a stream nobody will ever read.) + const onOutput = call.session.onOutput; + if (onOutput === undefined) throw new Error("the exec dispatch has no onOutput"); + expect(onOutput("stdout", chunk)).toBeUndefined(); + + call.resolve({ exitCode: 0 }); + await settle(); + await expect(reader?.read()).resolves.toMatchObject({ done: true }); + }); + }); + + it("DELETE /v1/leases/:id releases and answers 202 with the device's post-release state", async () => { + const { app, dispatcher, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1", state: "leased" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + dispatcher.handlers["lease.release"] = (input) => { + const leaseId = (input as { leaseId: string }).leaseId; + registry.leases = registry.leases.filter((lease) => lease.id !== leaseId); + // why: Not `reclaiming`: that is also the route's default for a device it cannot find, so only + // another state proves the answer is read from the registry. + registry.devices = registry.devices.map((device) => + device.id === "dev_1" ? { ...device, state: "quarantined" } : device, + ); + return { leaseId }; + }; + + const response = await app.request("/v1/leases/lse_1", { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + device: { id: "dev_1", state: "quarantined" }, + released: true, + }); + }); + + it("an operator may release another requester's lease (lease.list's own admin bypass)", async () => { + const { app, dispatcher, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; + dispatcher.handlers["lease.release"] = (input) => ({ + leaseId: (input as { leaseId: string }).leaseId, + }); + + const response = await app.request("/v1/leases/lse_1", { + headers: operatorAuth, + method: "DELETE", + }); + expect(response.status).toBe(202); + }); + + it("404s a release for an unknown lease", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.release"] = () => { + throw new UnknownLeaseError("lse-missing"); + }; + + const response = await app.request("/v1/leases/lse-missing", { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(404); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe( + "UNKNOWN_LEASE", + ); + }); + + it("403s a release of another requester's still-live lease -- dispatched directly through lease.release's own ownsLease hook, not lease.list's 404-for-everything filter (S6)", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["lease.release"] = () => { + throw new DispatchError("FORBIDDEN", "Not authorized for lease.release"); + }; + + const response = await app.request("/v1/leases/lse_other", { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(403); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + }); +}); diff --git a/src/http/app-leases.test.ts b/src/http/app-leases.test.ts index c6ee81696..3fe8ade19 100644 --- a/src/http/app-leases.test.ts +++ b/src/http/app-leases.test.ts @@ -1,12 +1,6 @@ -import { createHash, randomFillSync } from "node:crypto"; -import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterEach, describe, expect, it } from "vitest"; - +import { describe, expect, it } from "vitest"; import { EventBus } from "../bus/index.js"; -import { UnknownLeaseError, SerializedDecision } from "../core/index.js"; +import { SerializedDecision } from "../core/index.js"; import { loadTestRegistry, openTestState } from "../core/testing.js"; import { NoCapacityError, @@ -16,11 +10,9 @@ import { } from "../leasing/index.js"; import { DispatchError } from "../daemon/dispatcher.js"; import { describeLeaseRequestFailure } from "../daemon/error-code.js"; -import { createLeaseFiles, type LeaseFilesLimits } from "../lease-files/index.js"; -import { FakeClock, NodeFilesystem } from "../ports/index.js"; +import { FakeClock } from "../ports/index.js"; import { agentAuth, - type App, buildHarness, createLeaseRequest, defaultBody, @@ -30,14 +22,7 @@ import { readSseFrames, settle, } from "./test-app.js"; -import { - FakeDispatcher, - makeDevice, - makeGrant, - makeLease, - sequenceIdGenerator, - waitForDispatch, -} from "./test-fakes.js"; +import { FakeDispatcher, makeGrant, sequenceIdGenerator, waitForDispatch } from "./test-fakes.js"; describe("POST /v1/lease-requests", () => { it("creates a request resource, 201 with a Location header", async () => { @@ -747,1321 +732,3 @@ describe("DELETE /v1/lease-requests/:id", () => { expect(dispatcher.calls.map((call) => call.operation)).not.toContain("lease.cancel"); }); }); - -describe("lease routes", () => { - it("GET /v1/leases/:id returns the acquisition-shaped lease object", async () => { - const { app, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - - const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); - expect(response.status).toBe(200); - const body = (await response.json()) as { lease: Record }; - expect(body.lease).toMatchObject({ - dataPlane: null, - deviceId: "dev_1", - device: "iPhone 17 Pro", - id: "lse_1", - platform: "ios", - udid: "ABCD-1234", - }); - expect(body.lease).not.toHaveProperty("worker"); - expect(body.lease).not.toHaveProperty("workerId"); - }); - - it("GET /v1/leases/:id on a gateway builds the lease from its own worker's device, not another worker's with the same device id", async () => { - const device = (workerId: string, udid: string) => ({ - ...makeDevice({ driverDeviceId: udid, id: "dev_1" }), - workerId, - }); - const { app, dispatcher } = buildHarness({ - registry: { - snapshot: { devices: [device("wrk_a", "UDID-A"), device("wrk_b", "UDID-B")] }, - }, - }); - dispatcher.handlers["lease.get"] = () => ({ - lease: { - ...makeLease({ deviceId: "dev_1", id: "wrk_b.lse_1", ownerId: "tok_agent" }), - worker: { id: "wrk_b", label: "mac-b" }, - workerId: "wrk_b", - }, - }); - - const response = await app.request("/v1/leases/wrk_b.lse_1", { headers: agentAuth }); - - expect(response.status).toBe(200); - expect(((await response.json()) as { lease: unknown }).lease).toMatchObject({ - deviceId: "dev_1", - id: "wrk_b.lse_1", - udid: "UDID-B", - worker: { id: "wrk_b", label: "mac-b" }, - workerId: "wrk_b", - }); - }); - - it("GET /v1/leases/:id keeps a simulator's mode on the lease object", async () => { - const { app, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1", mode: "slim" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - - const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); - - expect(((await response.json()) as { lease: unknown }).lease).toMatchObject({ mode: "slim" }); - }); - - it("GET /v1/leases/:id answers a lease on a physical device with no mode, on a worker", async () => { - const { app, registry } = buildHarness(); - registry.devices = [ - { - createdAt: 0, - driverData: undefined, - driverDeviceId: "00008110-001A2B3C4D5E", - enrolledApps: [], - id: "dev_phys", - physical: true, - spec: { - class: "phone", - model: "iPhone 15 Pro", - osVersion: "18.4", - physical: true, - platform: "ios", - }, - state: "leased", - }, - ]; - registry.leases = [makeLease({ deviceId: "dev_phys", id: "lse_p", ownerId: "tok_agent" })]; - - const response = await app.request("/v1/leases/lse_p", { headers: agentAuth }); - - expect(response.status).toBe(200); - const { lease } = (await response.json()) as { lease: Record }; - expect(lease).toMatchObject({ - device: "iPhone 15 Pro", - deviceId: "dev_phys", - os: "18.4", - udid: "00008110-001A2B3C4D5E", - }); - expect(lease).not.toHaveProperty("mode"); - }); - - it("GET /v1/leases/:id answers a lease on a physical device with no mode, on a gateway", async () => { - const { app, dispatcher } = buildHarness({ - registry: { - snapshot: { - devices: [ - { - driverDeviceId: "00008110-001A2B3C4D5E", - id: "dev_phys", - spec: { model: "iPhone 15 Pro", osVersion: "18.4", platform: "ios" }, - workerId: "wrk_a", - }, - ], - }, - }, - }); - dispatcher.handlers["lease.get"] = () => ({ - lease: { - ...makeLease({ deviceId: "dev_phys", id: "wrk_a.lse_p", ownerId: "tok_agent" }), - worker: { id: "wrk_a" }, - workerId: "wrk_a", - }, - }); - - const response = await app.request("/v1/leases/wrk_a.lse_p", { headers: agentAuth }); - - expect(response.status).toBe(200); - const { lease } = (await response.json()) as { lease: Record }; - expect(lease).toMatchObject({ deviceId: "dev_phys", udid: "00008110-001A2B3C4D5E" }); - expect(lease).not.toHaveProperty("mode"); - }); - - it("names an Android emulator's adb serial as address, beside its AVD name as udid, on the lease object the POST, the request poll and GET /v1/leases/:id answer", async () => { - const emulator = { - address: "emulator-5554", - driverDeviceId: "simlock_3f9a2c1e", - id: "dev_and", - spec: { model: "Pixel 8", osVersion: "35", physical: false, platform: "android" }, - } as const; - const { app, dispatcher, registry } = buildHarness(); - const posted = postLeaseRequest(app, { device: "Pixel 8", platform: "android" }); - const call = await waitForDispatch(dispatcher, "lease.request"); - call.resolve(makeGrant({ device: emulator, lease: { deviceId: "dev_and", id: "lse_and" } })); - const created = (await (await posted).json()) as { request: { id: string; lease: unknown } }; - registry.devices = [makeDevice(emulator)]; - registry.leases = [makeLease({ deviceId: "dev_and", id: "lse_and", ownerId: "tok_agent" })]; - - const polled = await app.request(`/v1/lease-requests/${created.request.id}`, { - headers: agentAuth, - }); - const read = await app.request("/v1/leases/lse_and", { headers: agentAuth }); - - const named = { address: "emulator-5554", udid: "simlock_3f9a2c1e" }; - expect({ - get: ((await read.json()) as { lease: unknown }).lease, - poll: ((await polled.json()) as { request: { lease: unknown } }).request.lease, - post: created.request.lease, - }).toMatchObject({ get: named, poll: named, post: named }); - }); - - it("POST /v1/leases/:id/renew drains buffered device-health notices", async () => { - const { app, dispatcher, eventBus, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - dispatcher.handlers["lease.renew"] = (input) => ({ - deviceId: "dev_1", - grantedAt: 1_000, - id: (input as { leaseId: string }).leaseId, - ownerId: "tok_agent", - requesterId: "tok_agent", - ttlDeadline: 2_000 + ((input as { ttlMs?: number }).ttlMs ?? 900_000), - }); - - eventBus.emit( - "device.crash-detected", - { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, - "test", - ); - eventBus.emit( - "device.recovered", - { attempts: 1, deviceId: "dev_1", duration: 500, leaseId: "lse_1", platform: "ios" }, - "test", - ); - - const response = await app.request("/v1/leases/lse_1/renew", { - body: JSON.stringify({ ttlMs: 120_000 }), - headers: { ...agentAuth, "content-type": "application/json" }, - method: "POST", - }); - expect(response.status).toBe(200); - const body = (await response.json()) as { leaseId: string; notices: unknown[] }; - expect(body.leaseId).toBe("lse_1"); - expect(body.notices).toEqual([ - { event: "device_unhealthy" }, - { attempts: 1, event: "device_recovered" }, - ]); - }); - - it.each([ - ["renew", "/v1/leases/lse_1/renew", '{"ttlMs":60000}', "lease.renew"], - ["exec", "/v1/leases/lse_1/exec", '{"args":["list"],"tool":"simctl"}', "device.exec"], - [ - "install", - "/v1/components/install", - '{"platform":"android","version":"35"}', - "component.install", - ], - ])( - "a caller sends the %s body without a JSON content type and gets 415 UNSUPPORTED_MEDIA_TYPE; with application/json it reaches the dispatcher", - async (_route, path, body, operation) => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["lease.renew"] = () => ({ - deviceId: "dev_1", - grantedAt: 1_000, - id: "lse_1", - ownerId: "tok_agent", - requesterId: "tok_agent", - ttlDeadline: 2_000, - }); - dispatcher.handlers["device.exec"] = () => ({ exitCode: 0 }); - dispatcher.handlers["component.install"] = () => ({ - component: "35", - outcome: "installed", - platform: "android", - version: "35", - }); - const send = (contentType: string | undefined) => - app.request(path, { - body, - headers: { - ...agentAuth, - ...(contentType === undefined ? {} : { "content-type": contentType }), - }, - method: "POST", - }); - - for (const contentType of [undefined, "text/plain"]) { - const refused = await send(contentType); - expect(refused.status).toBe(415); - expect(await refused.json()).toEqual({ - error: { - code: "UNSUPPORTED_MEDIA_TYPE", - message: "Content-Type must be application/json", - }, - }); - } - expect(dispatcher.calls.filter((call) => call.operation === operation)).toEqual([]); - - const accepted = await send("application/json"); - expect(accepted.status).toBe(200); - expect(dispatcher.calls.filter((call) => call.operation === operation)).toHaveLength(1); - }, - ); - - it("a caller sends a bad wait or renew body and gets 400 with today's message; a renew with no body, an empty body under any content type or an extra key renews", async () => { - const { app, dispatcher } = buildHarness(); - const { id } = await createLeaseRequest(app, dispatcher); - const renewals: unknown[] = []; - dispatcher.handlers["lease.renew"] = (input) => { - renewals.push(input); - return { - deviceId: "dev_1", - grantedAt: 1_000, - id: "lse_1", - ownerId: "tok_agent", - requesterId: "tok_agent", - ttlDeadline: 2_000, - }; - }; - const renew = ( - body: string | undefined, - contentType: string | undefined = "application/json", - ) => - app.request("/v1/leases/lse_1/renew", { - ...(body === undefined ? {} : { body }), - headers: { - ...agentAuth, - ...(contentType === undefined ? {} : { "content-type": contentType }), - }, - method: "POST", - }); - const refusedWait = "wait must be a non-negative number of seconds"; - - const refusals: [string, Response | Promise, string][] = [ - [ - "wait=-1", - app.request(`/v1/lease-requests/${id}?wait=-1`, { headers: agentAuth }), - refusedWait, - ], - [ - "wait=abc", - app.request(`/v1/lease-requests/${id}?wait=abc`, { headers: agentAuth }), - refusedWait, - ], - [ - "wait=-1 on an unknown request", - app.request("/v1/lease-requests/unknown?wait=-1", { headers: agentAuth }), - refusedWait, - ], - ["ttlMs a string", renew('{"ttlMs":"60s"}'), "ttlMs must be a number"], - ["ttlMs negative", renew('{"ttlMs":-1}'), "ttlMs must be a positive number"], - ["ttlMs zero", renew('{"ttlMs":0}'), "ttlMs must be a positive number"], - ["a number", renew("5"), "Body must be a JSON object"], - ["an array", renew("[]"), "Body must be a JSON object"], - ["malformed JSON", renew("{"), "Invalid JSON body"], - ]; - for (const [name, pending, message] of refusals) { - const response = await pending; - expect(response.status, name).toBe(400); - expect(await response.json()).toEqual({ error: { code: "BAD_REQUEST", message } }); - } - expect(renewals).toEqual([]); - - for (const response of [ - await renew(undefined, undefined), - await renew("", "application/json"), - await renew("", "text/plain"), - await renew("", "application/x-www-form-urlencoded"), - await renew('{"ttlMs":60000,"note":"x"}'), - ]) { - expect(response.status).toBe(200); - } - expect(renewals).toEqual([ - { leaseId: "lse_1" }, - { leaseId: "lse_1" }, - { leaseId: "lse_1" }, - { leaseId: "lse_1" }, - { leaseId: "lse_1", ttlMs: 60_000 }, - ]); - }); - - it("404s a renew for an unknown lease", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["lease.renew"] = () => { - throw new UnknownLeaseError("lse-missing"); - }; - - const response = await app.request("/v1/leases/lse-missing/renew", { - headers: agentAuth, - method: "POST", - }); - expect(response.status).toBe(404); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe( - "UNKNOWN_LEASE", - ); - }); - - it("403s a renew for another requester's still-live lease -- dispatched directly through lease.renew's own ownsLease hook, not lease.list's 404-for-everything filter (S6)", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["lease.renew"] = () => { - throw new DispatchError("FORBIDDEN", "Not authorized for lease.renew"); - }; - - const response = await app.request("/v1/leases/lse_other/renew", { - headers: agentAuth, - method: "POST", - }); - expect(response.status).toBe(403); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - }); - - it("streams live lease notices over SSE, ending on lease_lost -- fed by the owner-routed fact bus, not a direct eventBus subscription", async () => { - const { app, eventBus, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - - const streamResponse = await app.request("/v1/leases/lse_1/events", { headers: agentAuth }); - const framesPromise = readSseFrames(streamResponse, 2); - await Promise.resolve(); - eventBus.emit( - "device.crash-detected", - { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, - "test", - ); - eventBus.emit( - "lease.expired", - { deviceId: "dev_1", leaseId: "lse_1", ownerId: "tok_agent" }, - "test", - ); - const frames = await framesPromise; - - expect(frames.map((frame) => frame.event)).toEqual(["device_unhealthy", "lease_lost"]); - }); - - describe("POST /v1/leases/:id/exec", () => { - function postExec( - app: App, - body: Record = { args: ["list"], tool: "simctl" }, - ) { - return app.request("/v1/leases/lse_1/exec", { - body: JSON.stringify(body), - headers: { ...agentAuth, "content-type": "application/json" }, - method: "POST", - }); - } - - it("streams a chunk per output event and ends with the command's exit code", async () => { - const { app, dispatcher } = buildHarness(); - - const responsePromise = postExec(app); - const call = await waitForDispatch(dispatcher, "device.exec"); - expect(call.input).toEqual({ args: ["list"], leaseId: "lse_1", tool: "simctl" }); - - // why: The stream opens when the process starts, not when it first speaks (ADR 0005 §19e). - // This chunk lands in the window between the two, which is the only thing the route's - // handoff buffer is for: it must still arrive, and still arrive first. - call.session.onStarted?.(); - call.session.onOutput?.("stdout", "one"); - const response = await responsePromise; - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("text/event-stream"); - - const framesPromise = readSseFrames(response, 3); - await Promise.resolve(); - call.session.onOutput?.("stderr", "two"); - call.resolve({ exitCode: 5 }); - const frames = await framesPromise; - - expect(frames.map((frame) => frame.event)).toEqual(["output", "output", "exit"]); - expect(frames.map((frame) => frame.data)).toEqual([ - { chunk: "one", stream: "stdout" }, - { chunk: "two", stream: "stderr" }, - { exitCode: 5 }, - ]); - }); - - it("answers a failure that lands before any output with its own status, not a stream", async () => { - // why: An SSE response cannot take back its status code, so ownership and the refusal list - // have to be answered ahead of the stream -- 403 here, exactly as `renew`/`release` do. - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["device.exec"] = () => { - throw new DispatchError("FORBIDDEN", "Not authorized for device.exec"); - }; - - const response = await postExec(app); - expect(response.status).toBe(403); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - }); - - it("reports a failure that lands after output began as a terminal error event", async () => { - const { app, dispatcher } = buildHarness(); - - const responsePromise = postExec(app); - const call = await waitForDispatch(dispatcher, "device.exec"); - call.session.onStarted?.(); - call.session.onOutput?.("stdout", "working"); - const response = await responsePromise; - - const framesPromise = readSseFrames(response, 2); - await Promise.resolve(); - call.reject(new DispatchError("EXEC_TIMEOUT", "exceeded exec.timeoutMs (1000ms)")); - const frames = await framesPromise; - - expect(frames.map((frame) => frame.event)).toEqual(["output", "error"]); - expect(frames.at(-1)?.data).toMatchObject({ error: { code: "EXEC_TIMEOUT" } }); - }); - - it("forwards requesterId unchanged for both an operator and an agent token, leaving the FORBIDDEN decision to device.exec's own authorize hook", async () => { - // why: Requester identity is never client-declared over HTTP: the token is the identity, - // except for an operator token proxying for someone (ADR 0005 §19b/§27). Round 4, F4: this route used to decide on its own - // that an agent token may not supply `requesterId` at all, which left that rule living - // only in the HTTP transport (ADR 0003 §2 puts a contract operation's answer in the - // dispatcher). The rule moved into `device.exec`'s own `authorize` hook -- exercised - // directly in `dispatcher-exec.test.ts` -- so this route now forwards `requesterId` exactly - // the same way for either token and merely surfaces whatever the dispatcher answers. - const { app, dispatcher } = buildHarness(); - - const asOperator = app.request("/v1/leases/lse_1/exec", { - body: JSON.stringify({ args: ["list"], requesterId: "agent-7", tool: "simctl" }), - headers: { ...operatorAuth, "content-type": "application/json" }, - method: "POST", - }); - const operatorCall = await waitForDispatch(dispatcher, "device.exec"); - expect(operatorCall.input).toEqual({ - args: ["list"], - leaseId: "lse_1", - requesterId: "agent-7", - tool: "simctl", - }); - operatorCall.session.onStarted?.(); - operatorCall.resolve({ exitCode: 0 }); - await asOperator; - - // why: Forwarded exactly the same way for an agent token -- the route makes no distinction - // any more. `FakeDispatcher` has no `authorize` hook of its own to reject this with, so - // the rejection is simulated here (`DispatchError("FORBIDDEN", ...)`, precisely what the - // real dispatcher's `authorize` hook now throws for this case) to prove this route still - // surfaces it as a 403 rather than requiring a route-level check to produce one. - const asAgent = app.request("/v1/leases/lse_1/exec", { - body: JSON.stringify({ args: ["list"], requesterId: "agent-7", tool: "simctl" }), - headers: { ...agentAuth, "content-type": "application/json" }, - method: "POST", - }); - const agentCall = await waitForDispatch(dispatcher, "device.exec", 1); - expect(agentCall.input).toEqual({ - args: ["list"], - leaseId: "lse_1", - requesterId: "agent-7", - tool: "simctl", - }); - agentCall.reject(new DispatchError("FORBIDDEN", "Not authorized for device.exec")); - const asAgentResponse = await asAgent; - expect(asAgentResponse.status).toBe(403); - expect(((await asAgentResponse.json()) as { error: { code: string } }).error.code).toBe( - "FORBIDDEN", - ); - - // why: A `null` for either optional is the documented example's own spelling, and normalizes - // to an omitted field rather than a 400 -- including for an agent, which is not - // "supplying a requesterId". - const withNulls = app.request("/v1/leases/lse_1/exec", { - body: JSON.stringify({ args: ["list"], requesterId: null, stdin: null, tool: "simctl" }), - headers: { ...agentAuth, "content-type": "application/json" }, - method: "POST", - }); - const nullCall = await waitForDispatch(dispatcher, "device.exec", 2); - expect(nullCall.input).toEqual({ args: ["list"], leaseId: "lse_1", tool: "simctl" }); - nullCall.session.onStarted?.(); - nullCall.resolve({ exitCode: 0 }); - await withNulls; - }); - - it("dispatches an unwrapped tool rather than rejecting it, and 400s a malformed body", async () => { - // why: Which wrappers exist is the drivers' answer: `bash` is a tool this host does not wrap - // (`UNKNOWN_PASSTHROUGH_TOOL`, 422, from the driver catalog -- see the dispatcher suite), - // not a malformed request. `400` here is about the body's shape only. - const { app, dispatcher } = buildHarness(); - - const unwrapped = postExec(app, { args: [], tool: "bash" }); - const call = await waitForDispatch(dispatcher, "device.exec"); - expect(call.input).toEqual({ args: [], leaseId: "lse_1", tool: "bash" }); - call.reject( - new DispatchError("UNKNOWN_PASSTHROUGH_TOOL", "No driver provides a bash passthrough"), - ); - const refused = await unwrapped; - expect(refused.status).toBe(422); - expect(((await refused.json()) as { error: { code: string } }).error.code).toBe( - "UNKNOWN_PASSTHROUGH_TOOL", - ); - - for (const body of [ - { args: [] }, - { args: "list", tool: "simctl" }, - { args: [], tool: "simctl", unexpected: true }, - { args: [], requesterId: "a\u001b[2Jb", tool: "simctl" }, - { args: [], requesterId: "x".repeat(257), tool: "simctl" }, - { args: [], stdin: "A".repeat(8_388_609), tool: "simctl" }, - ]) { - const response = await postExec(app, body); - expect(response.status).toBe(400); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe( - "BAD_REQUEST", - ); - } - expect(dispatcher.calls.filter((call) => call.operation === "device.exec")).toHaveLength(1); - }); - - it("opens the stream when the process starts, before it has written anything", async () => { - // why: ADR 0005 §19e: a command that prints nothing for nine minutes still gets its `200` and - // its keepalives, and an `EXEC_TIMEOUT` therefore always arrives as the stream's terminal - // event rather than as a status the client can no longer be given. - // - // C3 (round 3 review): this only proves that half of the route's *own* logic given a - // dispatcher that calls `onStarted` when told to (`call.session.onStarted?.()` below is - // this test driving that by hand) -- it says nothing about when a real dispatcher actually - // calls it, and cannot notice a dispatcher that stops calling it for a genuinely silent, - // long-running command. That is the gateway's own job (`FleetForwarding.exec` in - // `src/gateway/fleet-forwarding.ts`); the invariant this comment describes is guarded end - // to end there, not here -- see `fleet-coordinator.test.ts`'s "relays the worker's own - // started push for a silent command that never answers (ADR §19a/§19b/§19e)". - const { app, clock, dispatcher } = buildHarness(); - - const responsePromise = postExec(app); - const call = await waitForDispatch(dispatcher, "device.exec"); - call.session.onStarted?.(); - const response = await responsePromise; - expect(response.status).toBe(200); - - const framesPromise = readSseFrames(response, 1); - await Promise.resolve(); - clock.advance(15_000); - call.reject(new DispatchError("EXEC_TIMEOUT", "exceeded exec.timeoutMs (600000ms)")); - const frames = await framesPromise; - expect(frames).toEqual([ - { data: { error: { code: "EXEC_TIMEOUT", message: expect.any(String) } }, event: "error" }, - ]); - }); - - it("stalls the command when the client stops reading, instead of buffering for it", async () => { - // why: ADR 0005 §19e end to end. The route hands each chunk's SSE write back through - // `onOutput`, the process runner awaits it, so a client that opened the stream and does - // not read stops the command at its own pipe -- the alternative being an unbounded - // in-process queue for exactly the client least able to consume it. - const { app, dispatcher } = buildHarness(); - - const responsePromise = postExec(app); - const call = await waitForDispatch(dispatcher, "device.exec"); - call.session.onStarted?.(); - const response = await responsePromise; - const reader = response.body?.getReader(); - await Promise.resolve(); - - const settled: number[] = []; - let pending: number | undefined; - for (let index = 0; index < 64 && pending === undefined; index += 1) { - const delivery = Promise.resolve(call.session.onOutput?.("stdout", `chunk-${index}`)).then( - () => settled.push(index), - ); - // why: A delivery that has not settled after the microtask queue drains is the stream - // refusing more, which is the state this test exists to reach. - await new Promise((resolve) => setTimeout(resolve, 0)); - if (!settled.includes(index)) { - pending = index; - void delivery; - } - } - expect(pending, "the unread stream never applied backpressure").toBeDefined(); - - // why: Reading again lets the queued writes through, and the stalled delivery completes -- - // the command resumes rather than having been dropped. Each read is bounded: once the - // queue is empty a further read would block on the very producer this test has stalled. - for (let attempt = 0; attempt < 8 && !settled.includes(pending as number); attempt += 1) { - await Promise.race([reader?.read(), new Promise((resolve) => setTimeout(resolve, 20))]); - await new Promise((resolve) => setTimeout(resolve, 0)); - } - expect(settled).toContain(pending); - - await reader?.cancel(); - call.resolve({ exitCode: 0 }); - await responsePromise; - }); - - it("survives a client that disconnects while the command keeps writing", async () => { - // why: The command runs on deliberately, so chunks keep arriving at a route with nowhere to - // put them. They go nowhere and the request still settles normally -- what "nowhere" - // means precisely, and that nothing accumulates, is `output-relay.test.ts`. - const { app, dispatcher } = buildHarness(); - - const responsePromise = postExec(app); - const call = await waitForDispatch(dispatcher, "device.exec"); - call.session.onStarted?.(); - const response = await responsePromise; - const reader = response.body?.getReader(); - await Promise.resolve(); - await reader?.cancel(); - await Promise.resolve(); - - expect(call.session.onOutput).toBeDefined(); - const chunk = "x".repeat(1_024); - for (let index = 0; index < 1_000; index += 1) call.session.onOutput?.("stdout", chunk); - - // why: The relay is dropped, not merely detached: a chunk after the disconnect is discarded - // outright, which is observable as the delivery no longer being something to wait for. - // (Without the `relay.drop()` in the route's unsubscribe, this returns the pending write - // of a stream nobody will ever read.) - const onOutput = call.session.onOutput; - if (onOutput === undefined) throw new Error("the exec dispatch has no onOutput"); - expect(onOutput("stdout", chunk)).toBeUndefined(); - - call.resolve({ exitCode: 0 }); - await settle(); - await expect(reader?.read()).resolves.toMatchObject({ done: true }); - }); - }); - - it("DELETE /v1/leases/:id releases and answers 202 with the device's post-release state", async () => { - const { app, dispatcher, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1", state: "leased" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - dispatcher.handlers["lease.release"] = (input) => { - const leaseId = (input as { leaseId: string }).leaseId; - registry.leases = registry.leases.filter((lease) => lease.id !== leaseId); - // why: Not `reclaiming`: that is also the route's default for a device it cannot find, so only - // another state proves the answer is read from the registry. - registry.devices = registry.devices.map((device) => - device.id === "dev_1" ? { ...device, state: "quarantined" } : device, - ); - return { leaseId }; - }; - - const response = await app.request("/v1/leases/lse_1", { - headers: agentAuth, - method: "DELETE", - }); - expect(response.status).toBe(202); - expect(await response.json()).toEqual({ - device: { id: "dev_1", state: "quarantined" }, - released: true, - }); - }); - - it("an operator may release another requester's lease (lease.list's own admin bypass)", async () => { - const { app, dispatcher, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1" })]; - registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent" })]; - dispatcher.handlers["lease.release"] = (input) => ({ - leaseId: (input as { leaseId: string }).leaseId, - }); - - const response = await app.request("/v1/leases/lse_1", { - headers: operatorAuth, - method: "DELETE", - }); - expect(response.status).toBe(202); - }); - - it("404s a release for an unknown lease", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["lease.release"] = () => { - throw new UnknownLeaseError("lse-missing"); - }; - - const response = await app.request("/v1/leases/lse-missing", { - headers: agentAuth, - method: "DELETE", - }); - expect(response.status).toBe(404); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe( - "UNKNOWN_LEASE", - ); - }); - - it("403s a release of another requester's still-live lease -- dispatched directly through lease.release's own ownsLease hook, not lease.list's 404-for-everything filter (S6)", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["lease.release"] = () => { - throw new DispatchError("FORBIDDEN", "Not authorized for lease.release"); - }; - - const response = await app.request("/v1/leases/lse_other", { - headers: agentAuth, - method: "DELETE", - }); - expect(response.status).toBe(403); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - }); -}); - -const filesRoots: string[] = []; - -afterEach(async () => { - await Promise.all(filesRoots.splice(0).map((root) => rm(root, { force: true, recursive: true }))); -}); - -// why: a worker's HTTP app over a real lease-files store in a temporary folder, with lease `lse_1` held by `tok_agent`. -async function filesRig( - options: { - readonly limits?: LeaseFilesLimits; - readonly filesystem?: NodeFilesystem; - readonly diskPressureThreshold?: number; - } = {}, -) { - const dataDirectory = await mkdtemp(join(tmpdir(), "simlock-lease-files-")); - filesRoots.push(dataDirectory); - const rig = buildHarness({ - leaseFiles: ({ clock, logger }) => - createLeaseFiles({ - clock, - dataDirectory, - diskPressureThreshold: options.diskPressureThreshold ?? 0, - filesystem: options.filesystem ?? new NodeFilesystem(), - findGrant: (leaseId) => - leaseId === "lse_1" ? { grantedAt: 1_000, id: leaseId, platform: "ios" } : undefined, - idGenerator: sequenceIdGenerator("up"), - logger, - ...(options.limits === undefined ? {} : { limits: options.limits }), - }), - }); - rig.registry.leases = [makeLease({ id: "lse_1", ownerId: "tok_agent" })]; - const put = (name: string, body: BodyInit, headers: Record = agentAuth) => - rig.app.request(`/v1/leases/lse_1/files/${name}`, { body, headers, method: "PUT" }); - const get = (name: string, headers: Record = agentAuth) => - rig.app.request(`/v1/leases/lse_1/files/${name}`, { headers }); - const remove = (name: string, headers: Record = agentAuth) => - rig.app.request(`/v1/leases/lse_1/files/${name}`, { headers, method: "DELETE" }); - const list = (headers: Record = agentAuth) => - rig.app.request("/v1/leases/lse_1/files", { headers }); - const leaseFolder = join( - dataDirectory, - "lease-files", - `${Buffer.from("lse_1").toString("hex")}-1000`, - ); - return { ...rig, dataDirectory, get, leaseFolder, list, put, remove }; -} - -// why: what a lease folder holds, temporary upload files included, so "nothing stored" means nothing on disk. -async function folderEntries(folder: string): Promise { - return readdir(folder).catch(() => []); -} - -// why: a request body the test feeds by hand: `push` resolves once the upload has taken the bytes and asks for more, so a test acts only after the store has seen them. -function liveBody() { - let controller!: ReadableStreamDefaultController; - let waiting: (() => void)[] = []; - let pulled!: () => void; - const firstPull = new Promise((resolve) => { - pulled = resolve; - }); - const stream = new ReadableStream( - { - start(c) { - controller = c; - }, - pull() { - pulled(); - for (const resume of waiting.splice(0)) resume(); - }, - }, - { highWaterMark: 0 }, - ); - return { - fail: (error: Error) => controller.error(error), - finish: () => controller.close(), - firstPull, - push: (bytes: Uint8Array) => - new Promise((resolve) => { - controller.enqueue(bytes); - waiting = [...waiting, resolve]; - }), - stream, - }; -} - -// why: a request the code under test must answer; when it never does, the assertion on `undefined` fails by name where an await would time the whole test out. -function answered( - response: Promise | Response, - waitMs = 2_000, -): Promise { - return Promise.race([ - Promise.resolve(response), - new Promise((resolve) => setTimeout(() => resolve(undefined), waitMs)), - ]); -} - -// why: counts the files opened and not yet closed, so a test sees a response that left one open. -class CountingFilesystem extends NodeFilesystem { - open = 0; - override async openFile(path: string) { - const file = await super.openFile(path); - this.open += 1; - let closed = false; - const release = () => { - if (!closed) this.open -= 1; - closed = true; - }; - return { - size: file.size, - read: (range?: { readonly start: number; readonly end: number }) => { - const stream = file.read(range); - const reader = stream.getReader(); - return new ReadableStream({ - async pull(controller) { - const piece = await reader.read(); - if (piece.done) { - release(); - controller.close(); - } else controller.enqueue(piece.value); - }, - async cancel(reason) { - release(); - await reader.cancel(reason); - }, - }); - }, - close: async () => { - release(); - await file.close(); - }, - }; - } -} - -class LowDiskFilesystem extends NodeFilesystem { - override diskFree(): Promise { - return Promise.resolve(5 * 1024 ** 3); - } -} - -// why: comparing megabytes with `toEqual` walks them one by one and takes seconds; their digests say the same. -function sha256(bytes: Uint8Array): string { - return createHash("sha256").update(bytes).digest("hex"); -} - -describe("a lease's files on a worker", () => { - it("a lease holder PUTs a file and gets 201 with its name and size; a GET returns the same bytes", async () => { - const { get, put } = await filesRig(); - const patterned = (length: number): Uint8Array => { - const bytes = new Uint8Array(length); - randomFillSync(bytes.subarray(0, Math.min(length, 1 << 20))); - for (let at = 1 << 20; at < length; at += 1 << 20) bytes.copyWithin(at, 0, 1 << 20); - return bytes; - }; - const cases: [string, Uint8Array][] = [ - ["fixture.tar", patterned(3 * 1024 * 1024)], - ["fixture.tar", patterned(1000)], - ["empty.bin", new Uint8Array(0)], - ["big.bin", patterned(70_000_000)], - ]; - - for (const [name, bytes] of cases) { - const response = await put(name, bytes); - expect(response.status).toBe(201); - expect(await response.json()).toEqual({ name, size: bytes.length }); - const download = await get(name); - expect(download.status).toBe(200); - expect(sha256(new Uint8Array(await download.arrayBuffer()))).toBe(sha256(bytes)); - } - }); - - it("a lease holder PUTs a bad file name and gets 400 BAD_REQUEST with the name rule", async () => { - const { get, put, app } = await filesRig(); - const accepted = [ - "fixture.tar", - "app-release.apk", - "Documents.tar.deflate", - "A", - "a".repeat(128), - ]; - const refused = [ - ["a".repeat(129), "a".repeat(129)], - [".hidden", ".hidden"], - ["-rf", "-rf"], - ["my%20file.tar", "my file.tar"], - ["a%2Fb.tar", "a/b.tar"], - ["za%C5%BC%C3%B3%C5%82%C4%87.txt", "zażółć.txt"], - ] as const; - - for (const name of accepted) expect((await put(name, "x")).status).toBe(201); - for (const [encoded, decoded] of refused) { - for (const response of [await put(encoded, "x"), await get(encoded)]) { - expect(response.status).toBe(400); - expect(await response.json()).toEqual({ - error: { - code: "BAD_REQUEST", - message: `File ${decoded} must be 1 to 128 letters, digits, '.', '_' or '-', starting with a letter or digit`, - }, - }); - } - } - expect((await app.request("/v1/leases/lse_1/files/..", { headers: agentAuth })).status).toBe( - 404, - ); - }); - - it("a lease holder PUTs past a limit and gets the limit's code, with nothing stored and the previous file kept", async () => { - const limits = { fileBytes: 10, leaseBytes: 100, silenceMs: 330_000 }; - const big = new Uint8Array(11); - const rig = await filesRig({ limits }); - const refusal = async (response: Response) => - (await response.json()) as { error: { code: string; message: string } }; - const text = async (response: Response) => - new TextDecoder().decode(await response.arrayBuffer()); - - expect((await rig.put("kept.bin", "1234567890")).status).toBe(201); - const tooBig = await rig.put("kept.bin", big); - expect(tooBig.status).toBe(413); - expect((await refusal(tooBig)).error).toEqual({ - code: "FILE_TOO_LARGE", - message: "File kept.bin is over the 2 GiB limit for one file", - name: "kept.bin", - }); - expect(await text(await rig.get("kept.bin"))).toBe("1234567890"); - expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); - - let pulled = false; - const unread = () => - new ReadableStream( - { - pull(c) { - pulled = true; - c.close(); - }, - }, - { highWaterMark: 0 }, - ); - const declared = await rig.app.request("/v1/leases/lse_1/files/declared.bin", { - body: unread(), - duplex: "half", - headers: { ...agentAuth, "content-length": "11" }, - method: "PUT", - } as RequestInit); - expect(declared.status).toBe(413); - expect((await refusal(declared)).error.code).toBe("FILE_TOO_LARGE"); - expect(pulled).toBe(false); - expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); - - const roomy = { fileBytes: 1000, leaseBytes: 100, silenceMs: 330_000 }; - const full = await filesRig({ limits: roomy }); - await mkdir(full.leaseFolder, { recursive: true }); - await writeFile(join(full.leaseFolder, "my file.png"), new Uint8Array(60)); - expect((await full.put("fits.bin", new Uint8Array(40))).status).toBe(201); - await rm(join(full.leaseFolder, "fits.bin")); - for (const headers of [agentAuth, { ...agentAuth, "content-length": "50" }]) { - const over = await full.put("over.bin", new Uint8Array(50), headers); - expect(over.status).toBe(413); - expect((await refusal(over)).error).toEqual({ - code: "LEASE_FILES_FULL", - leaseId: "lse_1", - message: "Lease lse_1 holds 60 bytes of files; this upload would pass the 4 GiB limit", - usedBytes: 60, - }); - expect(await folderEntries(full.leaseFolder)).toEqual(["my file.png"]); - } - pulled = false; - const declaredFull = await full.app.request("/v1/leases/lse_1/files/over.bin", { - body: unread(), - duplex: "half", - headers: { ...agentAuth, "content-length": "50" }, - method: "PUT", - } as RequestInit); - expect(declaredFull.status).toBe(413); - expect((await refusal(declaredFull)).error.code).toBe("LEASE_FILES_FULL"); - expect(pulled).toBe(false); - - const together = await filesRig({ limits: roomy }); - const first = liveBody(); - const running = together.app.request("/v1/leases/lse_1/files/a.bin", { - body: first.stream, - duplex: "half", - headers: agentAuth, - method: "PUT", - } as RequestInit); - await first.push(new Uint8Array(60)); - const second = await together.put("b.bin", new Uint8Array(60)); - expect(second.status).toBe(413); - expect((await refusal(second)).error.code).toBe("LEASE_FILES_FULL"); - first.finish(); - expect((await running).status).toBe(201); - expect(await folderEntries(together.leaseFolder)).toEqual(["a.bin"]); - - const chunked = await filesRig({ limits: roomy }); - const chunks = liveBody(); - const chunkedRequest = chunked.app.request("/v1/leases/lse_1/files/chunks.bin", { - body: chunks.stream, - duplex: "half", - headers: agentAuth, - method: "PUT", - } as RequestInit); - for (let piece = 0; piece < 3; piece += 1) { - await Promise.race([chunks.push(new Uint8Array(30)), chunkedRequest]); - } - chunks.finish(); - expect((await chunkedRequest).status).toBe(201); - expect(await folderEntries(chunked.leaseFolder)).toEqual(["chunks.bin"]); - - const replacing = await filesRig({ limits: roomy }); - expect((await replacing.put("x.bin", new Uint8Array(80))).status).toBe(201); - expect((await replacing.put("x.bin", new Uint8Array(80))).status).toBe(201); - - const lowDisk = await filesRig({ - diskPressureThreshold: 10 * 1024 ** 3, - filesystem: new LowDiskFilesystem(), - }); - const noDisk = await lowDisk.put("fixture.tar", "x"); - expect(noDisk.status).toBe(422); - expect((await refusal(noDisk)).error).toEqual({ - availableBytes: 5 * 1024 ** 3, - code: "INSUFFICIENT_DISK_SPACE", - message: - "Not enough free disk space to store fixture.tar: the worker keeps 10.0 GiB free, only 5.0 GiB available", - platform: "ios", - requiredBytes: 10 * 1024 ** 3, - }); - expect(await folderEntries(lowDisk.leaseFolder)).toEqual([]); - }); - - it("a lease holder whose PUT is cut or goes silent keeps the previous file or none; a second PUT to a name being uploaded gets 409 FILE_BUSY", async () => { - const rig = await filesRig(); - const uploading = (name: string, body: ReadableStream) => - rig.app.request(`/v1/leases/lse_1/files/${name}`, { - body, - duplex: "half", - headers: agentAuth, - method: "PUT", - } as RequestInit); - const text = async (response: Response) => - new TextDecoder().decode(await response.arrayBuffer()); - const bytes = new TextEncoder().encode("new"); - - const cutFresh = liveBody(); - const cutting = uploading("fresh.bin", cutFresh.stream); - await cutFresh.push(bytes); - cutFresh.fail(new Error("socket hang up")); - expect(await answered(cutting)).toBeDefined(); - expect(await folderEntries(rig.leaseFolder)).toEqual([]); - - expect((await rig.put("kept.bin", "old")).status).toBe(201); - const cutKept = liveBody(); - const cuttingKept = uploading("kept.bin", cutKept.stream); - await cutKept.push(bytes); - cutKept.fail(new Error("socket hang up")); - expect(await answered(cuttingKept)).toBeDefined(); - expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); - expect(await text(await rig.get("kept.bin"))).toBe("old"); - - const silent = liveBody(); - const silentRequest = uploading("kept.bin", silent.stream); - await silent.push(bytes); - rig.clock.advance(329_999); - expect(await answered(silentRequest, 50)).toBeUndefined(); - rig.clock.advance(1); - const stalled = await answered(silentRequest); - expect(stalled?.status).toBe(408); - expect(await stalled?.json()).toEqual({ - error: { - code: "UPLOAD_STALLED", - message: "No bytes of kept.bin arrived for 330 s", - name: "kept.bin", - }, - }); - expect(await folderEntries(rig.leaseFolder)).toEqual(["kept.bin"]); - expect(await text(await rig.get("kept.bin"))).toBe("old"); - - const mute = liveBody(); - const muteRequest = uploading("mute.bin", mute.stream); - await mute.firstPull; - rig.clock.advance(329_999); - expect(await answered(muteRequest, 50)).toBeUndefined(); - rig.clock.advance(1); - expect((await answered(muteRequest))?.status).toBe(408); - - const moving = liveBody(); - const movingRequest = uploading("moving.bin", moving.stream); - for (const chunk of ["a", "b", "c"]) { - await moving.push(new TextEncoder().encode(chunk)); - rig.clock.advance(200_000); - } - moving.finish(); - expect((await movingRequest).status).toBe(201); - expect(await text(await rig.get("moving.bin"))).toBe("abc"); - - for (const [firstName, secondName] of [ - ["x.bin", "x.bin"], - ["A.tar", "a.tar"], - ] as const) { - const running = liveBody(); - const runningRequest = uploading(firstName, running.stream); - await running.push(bytes); - const refused = await rig.put(secondName, "other"); - expect(refused.status).toBe(409); - expect(await refused.json()).toEqual({ - error: { - code: "FILE_BUSY", - message: `File ${secondName} is being uploaded by another request`, - name: secondName, - }, - }); - running.finish(); - expect((await runningRequest).status).toBe(201); - } - }); - - it("a lease holder lists the lease's files and deletes one", async () => { - const { leaseFolder, list, put, remove } = await filesRig(); - const files = async () => ((await (await list()).json()) as { files: unknown[] }).files; - - expect((await list()).status).toBe(200); - expect(await files()).toEqual([]); - - expect((await put("b.txt", "bb")).status).toBe(201); - expect((await put("a.txt", "a")).status).toBe(201); - await mkdir(join(leaseFolder, "sub")); - await writeFile(join(leaseFolder, "sub", "inner.txt"), "x"); - await writeFile(join(leaseFolder, "my file.png"), "x"); - expect(await files()).toEqual([ - { name: "a.txt", size: 1 }, - { name: "b.txt", size: 2 }, - ]); - - const removed = await remove("a.txt"); - expect(removed.status).toBe(204); - expect(await files()).toEqual([{ name: "b.txt", size: 2 }]); - - const unknown = await remove("a.txt"); - expect(unknown.status).toBe(404); - expect(await unknown.json()).toEqual({ - error: { code: "UNKNOWN_FILE", message: "No file a.txt in this lease", name: "a.txt" }, - }); - }); - - it("a lease holder's HEAD answers a GET's headers with no body and leaves no file open", async () => { - const filesystem = new CountingFilesystem(); - const { app, put } = await filesRig({ filesystem }); - expect((await put("head.bin", "twelve bytes")).status).toBe(201); - const head = await app.request("/v1/leases/lse_1/files/head.bin", { - headers: agentAuth, - method: "HEAD", - }); - expect(head.status).toBe(200); - expect(head.headers.get("content-length")).toBe("12"); - expect(await head.text()).toBe(""); - expect(filesystem.open).toBe(0); - }); - - it("a lease holder's file names match ignoring case", async () => { - const { get, leaseFolder, list, put, remove } = await filesRig(); - const text = async (response: Response) => - new TextDecoder().decode(await response.arrayBuffer()); - const names = async () => - ((await (await list()).json()) as { files: { name: string }[] }).files.map( - (file) => file.name, - ); - - expect((await put("A.tar", "upper")).status).toBe(201); - expect(await text(await get("a.tar"))).toBe("upper"); - - expect((await put("a.tar", "lower")).status).toBe(201); - expect(await folderEntries(leaseFolder)).toEqual(["a.tar"]); - expect(await names()).toEqual(["a.tar"]); - expect(await text(await get("A.TAR"))).toBe("lower"); - - expect((await remove("A.TAR")).status).toBe(204); - expect(await names()).toEqual([]); - - await writeFile(join(leaseFolder, "A.tar"), "first"); - await writeFile(join(leaseFolder, "a.tar"), "second"); - if ((await folderEntries(leaseFolder)).length === 2) { - expect(await text(await get("a.tar"))).toBe("second"); - expect(await text(await get("A.TAR"))).toBe("first"); - expect(await names()).toEqual(["A.tar", "a.tar"]); - - expect((await remove("a.tar")).status).toBe(204); - expect(await names()).toEqual([]); - expect(await folderEntries(leaseFolder)).toEqual([]); - expect((await get("a.tar")).status).toBe(404); - expect((await get("A.tar")).status).toBe(404); - } - }); - - it("a lease holder GETs a file and gets it whole or the range it asked for", async () => { - const { get, put } = await filesRig(); - const thousand = Uint8Array.from({ length: 1000 }, (_, index) => index % 251); - expect((await put("thousand.bin", thousand)).status).toBe(201); - expect((await put("empty.bin", "")).status).toBe(201); - const ranged = (name: string, range: string) => get(name, { ...agentAuth, range }); - const bytesOf = async (response: Response) => new Uint8Array(await response.arrayBuffer()); - - const whole = await get("thousand.bin"); - expect(whole.status).toBe(200); - expect(Object.fromEntries(whole.headers)).toMatchObject({ - "accept-ranges": "bytes", - "cache-control": "no-store", - "content-length": "1000", - "content-type": "application/octet-stream", - "simlock-content-length": "1000", - }); - expect(await bytesOf(whole)).toEqual(thousand); - - const partial = [ - ["bytes=0-99", 0, 99], - ["bytes=900-", 900, 999], - ["bytes=-100", 900, 999], - ["bytes=990-5000", 990, 999], - ["bytes=-5000", 0, 999], - ] as const; - for (const [range, start, end] of partial) { - const response = await ranged("thousand.bin", range); - expect(response.status).toBe(206); - expect(response.headers.get("content-range")).toBe(`bytes ${start}-${end}/1000`); - expect(response.headers.get("content-length")).toBe(String(end - start + 1)); - expect(response.headers.get("simlock-content-length")).toBe(String(end - start + 1)); - expect(await bytesOf(response)).toEqual(thousand.subarray(start, end + 1)); - } - - for (const [name, range] of [ - ["thousand.bin", "bytes=1000-"], - ["thousand.bin", "bytes=-0"], - ["empty.bin", "bytes=-5"], - ] as const) { - const response = await ranged(name, range); - expect(response.status).toBe(416); - expect(response.headers.get("content-range")).toBe( - `bytes */${name === "empty.bin" ? 0 : 1000}`, - ); - } - - for (const range of ["bytes=0-1,5-9", "bytes=9-5", "items=0-9", "bytes=abc", "bytes=-"]) { - const response = await ranged("thousand.bin", range); - expect(response.status).toBe(200); - expect(await bytesOf(response)).toEqual(thousand); - } - - const unknown = await get("missing.bin"); - expect(unknown.status).toBe(404); - expect(await unknown.json()).toEqual({ - error: { - code: "UNKNOWN_FILE", - message: "No file missing.bin in this lease", - name: "missing.bin", - }, - }); - }); - - it("a caller that does not hold the lease gets 404 UNKNOWN_LEASE on every file route, and an operator is served", async () => { - const { app, put, list, get } = await filesRig(); - expect((await put("mine.bin", "mine")).status).toBe(201); - const routes = (leaseId: string, headers: Record) => [ - () => - app.request(`/v1/leases/${leaseId}/files/mine.bin`, { body: "x", headers, method: "PUT" }), - () => app.request(`/v1/leases/${leaseId}/files/mine.bin`, { headers }), - () => app.request(`/v1/leases/${leaseId}/files`, { headers }), - () => app.request(`/v1/leases/${leaseId}/files/mine.bin`, { headers, method: "DELETE" }), - ]; - - for (const [leaseId, headers] of [ - ["lse_1", otherAgentAuth], - ["lse_unknown", agentAuth], - ] as const) { - for (const response of await Promise.all(routes(leaseId, headers).map((call) => call()))) { - expect(response.status).toBe(404); - expect(((await response.json()) as { error: { code: string } }).error.code).toBe( - "UNKNOWN_LEASE", - ); - } - } - expect(await (await list()).json()).toEqual({ files: [{ name: "mine.bin", size: 4 }] }); - expect(new TextDecoder().decode(await (await get("mine.bin")).arrayBuffer())).toBe("mine"); - - const statuses: number[] = []; - for (const call of routes("lse_1", operatorAuth)) statuses.push((await call()).status); - expect(statuses).toEqual([201, 200, 200, 204]); - }); -}); From 902a450a8e7ab0ec32d56dc98f4422e859c8707c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:30:12 +0100 Subject: [PATCH 56/83] test(http): split app.test.ts by topic --- src/http/app-operator-listings.test.ts | 266 ++++++++ src/http/app-stream-and-caching.test.ts | 146 +++++ src/http/app-updates.test.ts | 230 +++++++ src/http/app-worker-routes.test.ts | 204 ++++++ src/http/app.test.ts | 821 +----------------------- 5 files changed, 847 insertions(+), 820 deletions(-) create mode 100644 src/http/app-operator-listings.test.ts create mode 100644 src/http/app-stream-and-caching.test.ts create mode 100644 src/http/app-updates.test.ts create mode 100644 src/http/app-worker-routes.test.ts diff --git a/src/http/app-operator-listings.test.ts b/src/http/app-operator-listings.test.ts new file mode 100644 index 000000000..8ad209d54 --- /dev/null +++ b/src/http/app-operator-listings.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from "vitest"; +import { runDispatch } from "../daemon/dispatch.js"; +import { DispatchError } from "../daemon/dispatcher.js"; +import { agentAuth, buildHarness, operatorAuth, readSseFrames, settle } from "./test-app.js"; +import { makeDevice, makeLease } from "./test-fakes.js"; + +describe("operator-only listing routes", () => { + it.each([ + ["an agent", agentAuth, { principal: "tok_agent", role: "agent" }], + ["an operator", operatorAuth, { principal: "tok_operator", role: "admin" }], + ] as const)( + "GET /v1/leases dispatches lease.list under %s's own session and answers the handler's leases", + async (_caller, headers, session) => { + // Which leases a caller may see is `lease.list`'s rule, owned and tested by the dispatcher; + // the route's part is to dispatch under the caller's session and return what it answers. + const { app, dispatcher } = buildHarness(); + const leases = [makeLease({ id: "lse_1", ownerId: "tok_somebody" })]; + dispatcher.handlers["lease.list"] = () => ({ leases }); + + const response = await app.request("/v1/leases", { headers }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ leases }); + expect(dispatcher.calls).toEqual([ + expect.objectContaining({ + input: {}, + operation: "lease.list", + session: expect.objectContaining(session), + }), + ]); + }, + ); + + it("GET /v1/lease-requests answers 403 for an agent token", async () => { + const { app, dispatcher } = buildHarness(); + const waiting = { + createdAt: 900, + id: "req_1", + queuePosition: 1, + requesterId: "agent-b", + spec: { model: "iPhone 16", platform: "ios" }, + stage: "queued", + }; + // `list.get`'s own role check, through the contract's real pipeline: the route adds none. + dispatcher.handlers["list.get"] = (input, session) => + runDispatch("list.get", input, session, { + handlers: { + "list.get": (listInput) => { + expect(listInput).toEqual({ kind: "requests" }); + return [waiting]; + }, + }, + }); + + const asAgent = await app.request("/v1/lease-requests", { headers: agentAuth }); + const asOperator = await app.request("/v1/lease-requests", { headers: operatorAuth }); + + expect(asAgent.status).toBe(403); + expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + expect(asOperator.status).toBe(200); + expect(await asOperator.json()).toEqual({ requests: [waiting] }); + }); + + it("GET /v1/devices dispatches list.get(kind: devices)", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["list.get"] = (input) => { + expect(input).toEqual({ kind: "devices" }); + return [makeDevice({ id: "dev_1" }), makeDevice({ id: "dev_2" })]; + }; + + const response = await app.request("/v1/devices", { headers: operatorAuth }); + const body = (await response.json()) as { devices: Array<{ id: string }> }; + expect(body.devices.map((device) => device.id).sort()).toEqual(["dev_1", "dev_2"]); + }); + + it("GET /v1/events replays via events.replay, translating ?since in seconds or days to an absolute sinceTs", async () => { + const { app, clock, dispatcher } = buildHarness(); + const seen: Array = []; + dispatcher.handlers["events.replay"] = (input) => { + const sinceTs = (input as { sinceTs?: number }).sinceTs; + seen.push(sinceTs); + if (sinceTs === undefined) return [{ event: "daemon.started" }, { event: "daemon.stopping" }]; + return [{ event: "daemon.stopping" }]; + }; + + const all = await app.request("/v1/events", { headers: operatorAuth }); + expect(((await all.json()) as { events: unknown[] }).events).toHaveLength(2); + + for (const [since, windowMs] of [ + ["5s", 5_000], + ["2d", 2 * 24 * 60 * 60 * 1000], + ] as const) { + const recent = await app.request(`/v1/events?since=${since}`, { headers: operatorAuth }); + const recentBody = (await recent.json()) as { events: Array<{ event: string }> }; + expect(recentBody.events).toEqual([{ event: "daemon.stopping" }]); + expect(seen.at(-1)).toBe(clock.now() - windowMs); + } + }); + + it("GET /v1/events with before and limit dispatches events.replay with beforeId and limit, and answers HISTORY_NOT_KEPT with 422 and oldestTs in the body and HISTORY_UNAVAILABLE with 503", async () => { + const { app, dispatcher } = buildHarness(); + const asked: unknown[] = []; + dispatcher.handlers["events.replay"] = (input) => { + asked.push(input); + throw new DispatchError("HISTORY_NOT_KEPT", "The history does not hold that event.", { + oldestTs: 1_234, + }); + }; + const notKept = await app.request("/v1/events?before=evt_abc&limit=50&worker=w1", { + headers: operatorAuth, + }); + dispatcher.handlers["events.replay"] = () => { + throw new DispatchError("HISTORY_UNAVAILABLE", "The history could not be read."); + }; + const unavailable = await app.request("/v1/events?before=evt_abc", { headers: operatorAuth }); + + expect(asked).toEqual([{ beforeId: "evt_abc", limit: 50, workerId: "w1" }]); + expect(notKept.status).toBe(422); + expect(await notKept.json()).toMatchObject({ + error: { code: "HISTORY_NOT_KEPT", oldestTs: 1_234 }, + }); + expect(unavailable.status).toBe(503); + expect(await unavailable.json()).toMatchObject({ error: { code: "HISTORY_UNAVAILABLE" } }); + }); + + it.each([ + ["before=evt_abc&limit=0", "limit must be a whole number from 1 to 500"], + ["before=evt_abc&limit=501", "limit must be a whole number from 1 to 500"], + ["before=evt_abc&limit=x", "limit must be a whole number from 1 to 500"], + ["before=not-an-id", "before must be an event id"], + ["since=abc", "Invalid duration: abc"], + ["before=evt_abc&since=1h", "before and since cannot be used together"], + ["limit=50&since=1h", "limit and since cannot be used together"], + ["worker=", "worker must be a worker id of 1 to 128 characters"], + [`worker=${"w".repeat(129)}`, "worker must be a worker id of 1 to 128 characters"], + ])("GET /v1/events?%s answers 400 BAD_REQUEST before dispatching", async (query, message) => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["events.replay"] = () => { + throw new Error("should not be called"); + }; + + const response = await app.request(`/v1/events?${query}`, { headers: operatorAuth }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "BAD_REQUEST", message } }); + }); + + describe("GET /v1/stats", () => { + const NOW = Date.parse("2026-10-16T12:00:00Z"); + const HOUR = 3_600_000; + + function statsHarness() { + const harness = buildHarness(); + harness.clock.advance(NOW - harness.clock.now()); + const asked: { from: number; to: number; includeLedger?: boolean }[] = []; + harness.dispatcher.handlers["usage.get"] = (input) => { + asked.push(input as never); + return { window: input }; + }; + const get = (query: string) => + harness.app.request(`/v1/stats${query}`, { headers: operatorAuth }); + return { ...harness, asked, get }; + } + + it("dispatches usage.get for since back from now, from to to, from to now, and the last 24 hours with no parameter", async () => { + const { asked, get } = statsHarness(); + + const answered = await get("?since=1h"); + await get("?from=2026-10-15T00:00Z&to=2026-10-16T00:00Z"); + await get("?from=2026-10-15T00:00Z"); + await get(""); + + expect(answered.status).toBe(200); + expect(await answered.json()).toEqual({ window: { from: NOW - HOUR, to: NOW } }); + expect(asked.map(({ from, to }) => ({ from, to }))).toEqual([ + { from: NOW - HOUR, to: NOW }, + { from: Date.parse("2026-10-15T00:00Z"), to: Date.parse("2026-10-16T00:00Z") }, + { from: Date.parse("2026-10-15T00:00Z"), to: NOW }, + { from: NOW - 24 * HOUR, to: NOW }, + ]); + }); + + it("answers 400 with the Inputs table's message for since with from, to without from, from not before to, a bad duration, the times 1, 2026, 10/15/2026, no zone and an unencoded +, and ledger other than 1, each after the same query without the fault answered 200, and answers 200 for %2B02:00", async () => { + const { asked, get } = statsHarness(); + const time = "from=2026-10-15T00:00Z"; + const timeMessage = (value: string) => + `Invalid time: ${value} (use ISO 8601 with Z or an offset)`; + const cases: readonly (readonly [string, string, string])[] = [ + [`?since=1h&${time}`, "?since=1h", "stats takes since or from, not both"], + ["?to=2026-10-16T00:00Z", `?${time}&to=2026-10-16T00:00Z`, "to needs from"], + [ + "?from=2026-10-16T00:00Z&to=2026-10-15T00:00Z", + "?from=2026-10-15T00:00Z&to=2026-10-16T00:00Z", + "from must be earlier than to, or than now when to is left out", + ], + ["?since=fortnight", "?since=1h", "Invalid duration: fortnight"], + ["?from=1", `?${time}`, timeMessage("1")], + ["?from=2026", `?${time}`, timeMessage("2026")], + ["?from=10/15/2026", `?${time}`, timeMessage("10/15/2026")], + ["?from=2026-10-15T00:00", `?${time}`, timeMessage("2026-10-15T00:00")], + ["?from=2026-10-15T00:00+02:00", `?${time}`, timeMessage("2026-10-15T00:00 02:00")], + ["?since=1h&ledger=yes", "?since=1h&ledger=1", "ledger must be 1"], + ]; + + for (const [bad, good, message] of cases) { + expect((await get(good)).status, good).toBe(200); + const dispatched = asked.length; + const refused = await get(bad); + expect(refused.status, bad).toBe(400); + expect(await refused.json(), bad).toMatchObject({ + error: { code: "BAD_REQUEST", message }, + }); + expect(asked, bad).toHaveLength(dispatched); + } + const offset = await get("?from=2026-10-15T00:00%2B02:00"); + expect(offset.status).toBe(200); + expect(asked.at(-1)).toEqual({ from: Date.parse("2026-10-14T22:00:00Z"), to: NOW }); + }); + + it("passes includeLedger: true only for ledger=1", async () => { + const { asked, get } = statsHarness(); + + await get("?since=1h"); + await get("?since=1h&ledger=1"); + + expect(asked.map((input) => input.includeLedger === true)).toEqual([false, true]); + }); + }); + + it("GET /v1/events/stream dispatches events.subscribe then follows the raw event bus live", async () => { + const { app, dispatcher, eventBus } = buildHarness(); + dispatcher.handlers["events.subscribe"] = (_input, session) => ({ + subscribed: true, + subscriptionId: session.manageEventSubscription(true), + }); + const streamResponse = await app.request("/v1/events/stream", { headers: operatorAuth }); + // `dispatch("events.subscribe", ...)` is awaited before the SSE stream opens, so it has + // already run by the time `app.request` resolves. + expect(dispatcher.calls.some((c) => c.operation === "events.subscribe")).toBe(true); + const framesPromise = readSseFrames(streamResponse, 1); + await Promise.resolve(); + eventBus.emit("daemon.stopping", { reason: "live" }, "test"); + const frames = await framesPromise; + expect(frames[0]?.event).toBe("daemon.stopping"); + }); + + it("GET /v1/events/stream writes a comment or retry line as soon as it opens, before any keepalive is due", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["events.subscribe"] = (_input, session) => ({ + subscribed: true, + subscriptionId: session.manageEventSubscription(true), + }); + const response = await app.request("/v1/events/stream", { headers: operatorAuth }); + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("response has no body"); + let written = "nothing written"; + void reader.read().then(({ value, done }) => { + written = done ? "stream ended" : new TextDecoder().decode(value); + }); + for (let turn = 0; turn < 50; turn += 1) await settle(); + const writtenBeforeCancel = written; + await reader.cancel(); + + expect(writtenBeforeCancel).toMatch(/^(:[^\n]*|retry: ?\d+)\n\n$/); + }); +}); diff --git a/src/http/app-stream-and-caching.test.ts b/src/http/app-stream-and-caching.test.ts new file mode 100644 index 000000000..7f58fe414 --- /dev/null +++ b/src/http/app-stream-and-caching.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { streamModule } from "../stream/index.js"; +import { createStreamMount } from "../stream/testing.js"; +import { agentAuth, buildHarness, operatorAuth, readSseFrames } from "./test-app.js"; + +// why: the stream index exports only the definition, so a registry reads no module code; the real mount is reached through `testing.ts` to prove the listed route against the real handler. +describe("the stream mount's listed route", () => { + it("is the route its handler answers: the listed path gets a status the definition lists, and /v1/stream/x gets its listed 404", async () => { + const route = streamModule.http?.routes[0]; + if (route === undefined) throw new Error("the stream definition lists no route"); + const link = { + serve: () => + Promise.resolve({ + body: new ReadableStream(), + close: () => Promise.resolve(), + meta: { codec: "h264" as const, fps: 30, height: 720, width: 330 }, + }), + }; + const { app } = buildHarness({ + mounts: [{ prefix: "stream", ...createStreamMount({ link }) }], + }); + const listed = Object.keys(route.statuses).map(Number); + const answer = (path: string, method = "GET") => + app.request(path, { headers: agentAuth, method }); + + const listedPath = route.path.replace("{workerId}", "w1").replace("{deviceId}", "dev_1"); + const found = await answer(listedPath); + const other = await answer("/v1/stream/x"); + const wrongMethod = await answer(listedPath, "POST"); + + expect(found.status).toBe(200); + expect(found.headers.get("content-type")).toBe("application/vnd.simlock.stream+h264"); + expect([other.status, wrongMethod.status]).toEqual([404, 405]); + expect(listed).toEqual( + expect.arrayContaining([found.status, other.status, wrongMethod.status]), + ); + expect(await other.json()).toMatchObject({ error: { code: "UNKNOWN_DEVICE" } }); + await found.body?.cancel(); + }); +}); + +describe("ETag on a JSON read under /v1", () => { + it("304 and no body for an unchanged read, 200 for a stale tag, a new ETag once the data changes, no ETag on a 404 or a mount", async () => { + const tokens: { id: string }[] = [{ id: "tok_1" }]; + const { app, dispatcher, eventBus } = buildHarness({ + mounts: [ + { + prefix: "json-mount", + handle: () => Response.json({ ok: true }), + }, + ] as never, + }); + dispatcher.handlers["token.list"] = () => ({ tokens: [...tokens] }); + dispatcher.handlers["events.subscribe"] = (_input, session) => ({ + subscribed: true, + subscriptionId: session.manageEventSubscription(true), + }); + const read = (path: string, headers: Record = {}) => + app.request(path, { headers: { ...operatorAuth, ...headers } }); + + const first = await read("/v1/tokens"); + const tag = first.headers.get("etag"); + expect(first.status).toBe(200); + expect(tag).toMatch(/^"[^"]+"$/); + + const unchanged = await read("/v1/tokens", { "If-None-Match": tag ?? "" }); + expect(unchanged.status).toBe(304); + expect(await unchanged.text()).toBe(""); + expect(unchanged.headers.get("etag")).toBe(tag); + + const stale = await read("/v1/tokens", { "If-None-Match": '"stale"' }); + expect(stale.status).toBe(200); + expect(stale.headers.get("etag")).toBe(tag); + + tokens.push({ id: "tok_2" }); + const changed = await read("/v1/tokens", { "If-None-Match": tag ?? "" }); + expect(changed.status).toBe(200); + expect(changed.headers.get("etag")).not.toBe(tag); + + const missing = await read("/v1/leases/lse_none"); + expect(missing.status).toBe(404); + expect(missing.headers.get("etag")).toBeNull(); + + const mount = await read("/v1/json-mount/x"); + expect(mount.status).toBe(200); + expect(mount.headers.get("content-type")).toContain("application/json"); + expect(mount.headers.get("etag")).toBeNull(); + + const streamOrStall = await Promise.race([ + read("/v1/events/stream"), + new Promise<"stalled">((resolve) => setTimeout(() => resolve("stalled"), 500)), + ]); + if (streamOrStall === "stalled") expect.fail("the event stream was buffered, not answered"); + const stream = streamOrStall; + expect(stream.headers.get("etag")).toBeNull(); + const framesPromise = readSseFrames(stream, 1); + await Promise.resolve(); + eventBus.emit("daemon.stopping", { reason: "live" }, "test"); + expect((await framesPromise)[0]?.event).toBe("daemon.stopping"); + }); +}); + +describe("Cache-Control on a JSON answer under /v1", () => { + it("a JSON read, its 304, a JSON error and /v1/healthz each carry Cache-Control: no-store, so a CDN never stores an answer, and a mount's answer keeps the Cache-Control it set", async () => { + const { app, dispatcher } = buildHarness({ + mounts: [ + { prefix: "json-mount", handle: () => Response.json({ ok: true }) }, + { + prefix: "cached-mount", + handle: () => Response.json({ ok: true }, { headers: { "Cache-Control": "max-age=60" } }), + }, + ] as never, + }); + dispatcher.handlers["token.list"] = () => ({ tokens: [{ id: "tok_1" }] }); + const read = (path: string, headers: Record = {}) => + app.request(path, { headers: { ...operatorAuth, ...headers } }); + + const first = await read("/v1/tokens"); + const unchanged = await read("/v1/tokens", { + "If-None-Match": first.headers.get("etag") ?? "", + }); + const missing = await read("/v1/leases/lse_none"); + const health = await app.request("/v1/healthz"); + const plainMount = await read("/v1/json-mount"); + const cachedMount = await read("/v1/cached-mount"); + + expect([first.status, unchanged.status, missing.status, health.status]).toEqual([ + 200, 304, 404, 200, + ]); + expect({ + plain: plainMount.headers.get("cache-control"), + cached: cachedMount.headers.get("cache-control"), + }).toEqual({ plain: null, cached: "max-age=60" }); + expect({ + read: first.headers.get("cache-control"), + notModified: unchanged.headers.get("cache-control"), + error: missing.headers.get("cache-control"), + healthz: health.headers.get("cache-control"), + }).toEqual({ + read: "no-store", + notModified: "no-store", + error: "no-store", + healthz: "no-store", + }); + }); +}); diff --git a/src/http/app-updates.test.ts b/src/http/app-updates.test.ts new file mode 100644 index 000000000..3bae4e521 --- /dev/null +++ b/src/http/app-updates.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; +import { runDispatch } from "../daemon/dispatch.js"; +import { DispatchError } from "../daemon/dispatcher.js"; +import { + agentAuth, + type App, + buildHarness, + createLeaseRequest, + defaultBody, + operatorAuth, + postLeaseRequest, + settle, +} from "./test-app.js"; +import { makeDevice, makeLease, waitForDispatch } from "./test-fakes.js"; + +describe("hardening from review", () => { + it("400s an oversized Idempotency-Key as BAD_REQUEST, where a 200-character key is accepted", async () => { + const { app, dispatcher } = buildHarness(); + const accepted = await Promise.race([ + postLeaseRequest(app, defaultBody, { ...agentAuth, "Idempotency-Key": "x".repeat(200) }), + waitForDispatch(dispatcher, "lease.request").then(() => "dispatched" as const), + ]); + if (accepted instanceof Response) expect(accepted.status).toBe(201); + else expect(accepted).toBe("dispatched"); + + const response = await postLeaseRequest(app, defaultBody, { + ...agentAuth, + "Idempotency-Key": "x".repeat(201), + }); + expect(response.status).toBe(400); + const { error } = (await response.json()) as { error: { code: string; message: string } }; + expect(error.code).toBe("BAD_REQUEST"); + expect(error.message).toContain("idempotencyKey"); + }); + + it("clamps an oversized ?wait to the 60s ceiling", async () => { + const { app, clock, dispatcher } = buildHarness(); + const { id } = await createLeaseRequest(app, dispatcher); + + const waitPromise = Promise.resolve( + app.request(`/v1/lease-requests/${id}?wait=999999`, { headers: agentAuth }), + ); + let answered = false; + void waitPromise.then(() => { + answered = true; + }); + await settle(); + clock.advance(59_999); + for (let turn = 0; turn < 50; turn += 1) await settle(); + expect(answered).toBe(false); + clock.advance(1); + const response = await waitPromise; + expect(((await response.json()) as { request: { state: string } }).request.state).toBe( + "queued", + ); + }); + + it("reports the lease's own stored ttlMs, with nothing gateway-side to remember", async () => { + // ADR 0004: the width lives on the lease record, so a payload served by a gateway that + // never saw the request (after a daemon restart, say) still reports the real width + // instead of a mode default standing in for one. + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [ + makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent", ttlMs: 123_000 }), + ]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + const body = (await response.json()) as { lease: { ttlMs: number } }; + expect(body.lease.ttlMs).toBe(123_000); + }); +}); + +/** + * ADR 0005 §23. The routes themselves are thin -- one dispatch each -- so what is worth + * asserting is which daemon has them at all, that they reach the right operation with the id + * from the path, and that a refusal's typed `details` reach the body. What a worker's own + * dispatcher answers for all four operations is proven in `src/daemon/dispatcher.test.ts`; + * `e2e/single-host-fleet.test.ts` drives `GET /v1/workers` and the drain route end to end. + */ +describe("POST /v1/update", () => { + const answer = (overrides: Record = {}) => ({ + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + ...overrides, + }); + const post = (app: App, body?: unknown, headers: Record = operatorAuth) => + Promise.resolve( + app.request("/v1/update", { + headers: { + ...headers, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + method: "POST", + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), + ); + + it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body", async () => { + const { app, dispatcher } = buildHarness(); + // why: `daemon.update`'s own role check runs through the contract's real pipeline; the route adds none. + let reply: Record = answer({ started: true, resolved: "0.4.0" }); + dispatcher.handlers["daemon.update"] = (input, session) => + runDispatch("daemon.update", input, session, { + handlers: { "daemon.update": () => reply }, + }); + + const started = await post(app, { version: "0.4.0", force: true }); + expect({ status: started.status, body: await started.json() }).toEqual({ + status: 202, + body: reply, + }); + expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ + { operation: "daemon.update", input: { version: "0.4.0", force: true } }, + ]); + + for (const [name, following] of [ + [ + "follows a running update", + answer({ + update: { stage: "installing", from: "0.3.0", to: "0.4.0", forced: false, startedAt: 1 }, + }), + ], + ["nothing to do", answer({ upToDate: true, resolved: "0.3.0" })], + ] as const) { + reply = following; + const response = await post(app); + expect({ name, status: response.status }).toEqual({ name, status: 200 }); + expect(await response.json()).toEqual(following); + } + expect(dispatcher.calls.at(-1)?.input).toEqual({}); + + const emptyJson = await app.request("/v1/update", { + headers: { ...operatorAuth, "content-type": "application/json" }, + method: "POST", + body: "", + }); + expect(emptyJson.status).toBe(200); + expect(dispatcher.calls.at(-1)?.input).toEqual({}); + + const asAgent = await post(app, {}, agentAuth); + expect(asAgent.status).toBe(403); + expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + + const callsBefore = dispatcher.calls.length; + for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { + const refused = await post(app, body); + expect({ body, status: refused.status }).toEqual({ body, status: 400 }); + } + expect(dispatcher.calls).toHaveLength(callsBefore); + }); +}); + +describe("POST /v1/workers/{id}/update", () => { + const answer = (overrides: Record = {}) => ({ + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + workerId: "wrk_1", + ...overrides, + }); + + it("answers 202 when the call began the update and 200 when it followed one, dispatches worker.update with the id from the path and the body, refuses an agent and a bad body", async () => { + const { app, dispatcher } = buildHarness(); + let reply: Record = answer({ started: true, resolved: "0.4.0" }); + dispatcher.handlers["worker.update"] = (input, session) => + runDispatch("worker.update", input, session, { + handlers: { "worker.update": () => reply }, + }); + const post = (body?: unknown, headers: Record = operatorAuth) => + Promise.resolve( + app.request("/v1/workers/wrk_1/update", { + headers: { + ...headers, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + method: "POST", + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), + ); + + const started = await post({ version: "0.4.0", force: true }); + expect(started.status).toBe(202); + expect(await started.json()).toEqual(reply); + expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ + { operation: "worker.update", input: { force: true, version: "0.4.0", workerId: "wrk_1" } }, + ]); + + reply = answer({ upToDate: true, resolved: "0.3.0" }); + const following = await post(); + expect(following.status).toBe(200); + expect(dispatcher.calls.at(-1)?.input).toEqual({ workerId: "wrk_1" }); + + const asAgent = await post({}, agentAuth); + expect(asAgent.status).toBe(403); + + const callsBefore = dispatcher.calls.length; + for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { + const refused = await post(body); + expect({ body, status: refused.status }).toEqual({ body, status: 400 }); + } + expect(dispatcher.calls).toHaveLength(callsBefore); + + for (const [code, status] of [ + ["WORKER_NOT_CONNECTED", 409], + ["UPDATE_UNSUPPORTED_INSTALL", 409], + ["UPDATE_SHARED_INSTALL", 409], + ["UPDATE_RESOLVE_FAILED", 502], + ["UNKNOWN_WORKER", 404], + ] as const) { + dispatcher.handlers["worker.update"] = () => { + throw new DispatchError(code, "refused"); + }; + const refusal = await post(); + expect({ code, status: refusal.status }).toEqual({ code, status }); + expect(((await refusal.json()) as { error: { code: string } }).error.code).toBe(code); + } + }); +}); diff --git a/src/http/app-worker-routes.test.ts b/src/http/app-worker-routes.test.ts new file mode 100644 index 000000000..5681fec75 --- /dev/null +++ b/src/http/app-worker-routes.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { runDispatch } from "../daemon/dispatch.js"; +import { DispatchError } from "../daemon/dispatcher.js"; +import { testConfig } from "../core/testing.js"; +import { + agentAuth, + buildHarness, + defaultBody, + operatorAuth, + postLeaseRequest, +} from "./test-app.js"; + +describe("worker routes", () => { + function gatewayHarness() { + return buildHarness(); + } + + it("are routes on a worker too (ADR 0012 §3), each dispatching its operation", async () => { + const { app, dispatcher } = buildHarness(); + dispatcher.handlers["worker.list"] = () => ({ workers: [] }); + for (const operation of ["worker.drain", "worker.undrain", "worker.remove"] as const) { + dispatcher.handlers[operation] = () => { + throw new DispatchError("UNSUPPORTED_IN_WORKER_MODE", `${operation} is refused`, { + operation, + }); + }; + } + + const list = await app.request("/v1/workers", { headers: operatorAuth }); + const drain = await app.request("/v1/workers/w/drain", { + headers: operatorAuth, + method: "POST", + }); + const undrain = await app.request("/v1/workers/w/drain", { + headers: operatorAuth, + method: "DELETE", + }); + const remove = await app.request("/v1/workers/w", { headers: operatorAuth, method: "DELETE" }); + + expect(list.status).toBe(200); + expect(await list.json()).toEqual({ workers: [] }); + // Each refusal names the operation its handler was reached through, so the body proves + // which route dispatched what. + const refusals = [drain, undrain, remove]; + expect(refusals.map((response) => response.status)).toEqual([501, 501, 501]); + expect(await Promise.all(refusals.map((response) => response.json()))).toEqual( + ["worker.drain", "worker.undrain", "worker.remove"].map((operation) => ({ + error: { code: "UNSUPPORTED_IN_WORKER_MODE", message: expect.any(String), operation }, + })), + ); + }); + + it("GET /v1/workers returns the views", async () => { + const { app, dispatcher } = gatewayHarness(); + dispatcher.handlers["worker.list"] = () => ({ + workers: [ + { + catalog: [], + connection: "connected", + devices: [], + drained: false, + id: "wrk_1", + lastSeenAt: 1, + leases: [], + }, + ], + }); + + const response = await app.request("/v1/workers", { headers: operatorAuth }); + + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ workers: [{ id: "wrk_1" }] }); + }); + + it("GET /v1/inbox dispatches inbox.get for the caller and answers its items", async () => { + const { app, dispatcher } = gatewayHarness(); + const item = { + condition: "disconnected", + group: "Broken", + key: "wrk_1/disconnected", + worker: { + connection: "disconnected", + drained: false, + heldLeases: 0, + id: "wrk_1", + lastSeenAt: 1, + leasedDevices: 0, + }, + }; + dispatcher.handlers["inbox.get"] = () => ({ items: [item] }); + + const response = await app.request("/v1/inbox", { headers: operatorAuth }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ items: [item] }); + expect(dispatcher.calls.map((call) => [call.operation, call.session.role])).toEqual([ + ["inbox.get", "admin"], + ]); + }); + + it("GET /v1/config dispatches config.get for the caller and answers the config; an agent token is a 403, no token a 401", async () => { + const { app, dispatcher } = gatewayHarness(); + const config = testConfig({ update: { check: false } }, "gateway"); + // why: `config.get`'s own role check runs through the contract's real pipeline; the route adds none. + dispatcher.handlers["config.get"] = (input, session) => + runDispatch("config.get", input, session, { + handlers: { + "config.get": (configInput) => { + expect(configInput).toEqual({}); + return config; + }, + }, + }); + + const asOperator = await app.request("/v1/config", { headers: operatorAuth }); + const asAgent = await app.request("/v1/config", { headers: agentAuth }); + const anonymous = await app.request("/v1/config"); + + expect(asOperator.status).toBe(200); + expect(await asOperator.json()).toMatchObject({ mode: "gateway", update: { check: false } }); + expect(asAgent.status).toBe(403); + expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + expect(anonymous.status).toBe(401); + expect(dispatcher.calls.map((call) => [call.operation, call.session.role])).toEqual([ + ["config.get", "admin"], + ["config.get", "agent"], + ]); + }); + + it("maps drain, undrain and remove onto their operations, with the id from the path", async () => { + const { app, dispatcher } = gatewayHarness(); + dispatcher.handlers["worker.drain"] = (input: never) => ({ + drained: true, + workerId: (input as { workerId: string }).workerId, + }); + dispatcher.handlers["worker.undrain"] = (input: never) => ({ + drained: false, + workerId: (input as { workerId: string }).workerId, + }); + dispatcher.handlers["worker.remove"] = (input: never) => ({ + removed: true, + workerId: (input as { workerId: string }).workerId, + }); + + const drained = await app.request("/v1/workers/wrk_1/drain", { + headers: operatorAuth, + method: "POST", + }); + expect(await drained.json()).toEqual({ drained: true, workerId: "wrk_1" }); + const undrained = await app.request("/v1/workers/wrk_1/drain", { + headers: operatorAuth, + method: "DELETE", + }); + expect(await undrained.json()).toEqual({ drained: false, workerId: "wrk_1" }); + const removed = await app.request("/v1/workers/wrk_2", { + headers: operatorAuth, + method: "DELETE", + }); + expect(await removed.json()).toEqual({ removed: true, workerId: "wrk_2" }); + + expect(dispatcher.calls.map((call) => call.operation)).toEqual([ + "worker.drain", + "worker.undrain", + "worker.remove", + ]); + }); + + it("puts a refusal's typed details in the body, so a client need not parse prose", async () => { + const { app, dispatcher } = gatewayHarness(); + dispatcher.handlers["worker.remove"] = () => { + throw new DispatchError("WORKER_CONNECTED", "Worker wrk_1 is still connected", { + workerId: "wrk_1", + }); + }; + + const response = await app.request("/v1/workers/wrk_1", { + headers: operatorAuth, + method: "DELETE", + }); + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: { code: "WORKER_CONNECTED", message: expect.any(String), workerId: "wrk_1" }, + }); + }); + + it("answers 501 for an operation a gateway does not implement", async () => { + const { app, dispatcher } = gatewayHarness(); + dispatcher.handlers["lease.request"] = () => { + throw new DispatchError( + "UNSUPPORTED_IN_GATEWAY_MODE", + "lease.request is not available on a gateway yet", + { operation: "lease.request" }, + ); + }; + + const response = await postLeaseRequest(app, defaultBody); + + expect(response.status).toBe(501); + expect(await response.json()).toMatchObject({ + error: { code: "UNSUPPORTED_IN_GATEWAY_MODE", operation: "lease.request" }, + }); + }); +}); diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 3a110310a..8efda02d1 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -1,22 +1,14 @@ import { describe, expect, it } from "vitest"; - -import { identityGet, runDispatch } from "../daemon/dispatch.js"; +import { identityGet } from "../daemon/dispatch.js"; import { DispatchError } from "../daemon/dispatcher.js"; -import { testConfig } from "../core/testing.js"; -import { streamModule } from "../stream/index.js"; -// why: the stream index exports only the definition, so a registry reads no module code; the real mount is reached through `testing.ts` to prove the listed route against the real handler. -import { createStreamMount } from "../stream/testing.js"; import { agentAuth, type App, buildHarness, createLeaseRequest, - defaultBody, operatorAuth, otherAgentAuth, - postLeaseRequest, readSseFrames, - settle, } from "./test-app.js"; import { makeDevice, makeLease, waitForDispatch } from "./test-fakes.js"; @@ -705,814 +697,3 @@ describe("POST /v1/components/install", () => { expect(dispatcher.calls).toEqual([]); }); }); - -describe("operator-only listing routes", () => { - it.each([ - ["an agent", agentAuth, { principal: "tok_agent", role: "agent" }], - ["an operator", operatorAuth, { principal: "tok_operator", role: "admin" }], - ] as const)( - "GET /v1/leases dispatches lease.list under %s's own session and answers the handler's leases", - async (_caller, headers, session) => { - // Which leases a caller may see is `lease.list`'s rule, owned and tested by the dispatcher; - // the route's part is to dispatch under the caller's session and return what it answers. - const { app, dispatcher } = buildHarness(); - const leases = [makeLease({ id: "lse_1", ownerId: "tok_somebody" })]; - dispatcher.handlers["lease.list"] = () => ({ leases }); - - const response = await app.request("/v1/leases", { headers }); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ leases }); - expect(dispatcher.calls).toEqual([ - expect.objectContaining({ - input: {}, - operation: "lease.list", - session: expect.objectContaining(session), - }), - ]); - }, - ); - - it("GET /v1/lease-requests answers 403 for an agent token", async () => { - const { app, dispatcher } = buildHarness(); - const waiting = { - createdAt: 900, - id: "req_1", - queuePosition: 1, - requesterId: "agent-b", - spec: { model: "iPhone 16", platform: "ios" }, - stage: "queued", - }; - // `list.get`'s own role check, through the contract's real pipeline: the route adds none. - dispatcher.handlers["list.get"] = (input, session) => - runDispatch("list.get", input, session, { - handlers: { - "list.get": (listInput) => { - expect(listInput).toEqual({ kind: "requests" }); - return [waiting]; - }, - }, - }); - - const asAgent = await app.request("/v1/lease-requests", { headers: agentAuth }); - const asOperator = await app.request("/v1/lease-requests", { headers: operatorAuth }); - - expect(asAgent.status).toBe(403); - expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - expect(asOperator.status).toBe(200); - expect(await asOperator.json()).toEqual({ requests: [waiting] }); - }); - - it("GET /v1/devices dispatches list.get(kind: devices)", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["list.get"] = (input) => { - expect(input).toEqual({ kind: "devices" }); - return [makeDevice({ id: "dev_1" }), makeDevice({ id: "dev_2" })]; - }; - - const response = await app.request("/v1/devices", { headers: operatorAuth }); - const body = (await response.json()) as { devices: Array<{ id: string }> }; - expect(body.devices.map((device) => device.id).sort()).toEqual(["dev_1", "dev_2"]); - }); - - it("GET /v1/events replays via events.replay, translating ?since in seconds or days to an absolute sinceTs", async () => { - const { app, clock, dispatcher } = buildHarness(); - const seen: Array = []; - dispatcher.handlers["events.replay"] = (input) => { - const sinceTs = (input as { sinceTs?: number }).sinceTs; - seen.push(sinceTs); - if (sinceTs === undefined) return [{ event: "daemon.started" }, { event: "daemon.stopping" }]; - return [{ event: "daemon.stopping" }]; - }; - - const all = await app.request("/v1/events", { headers: operatorAuth }); - expect(((await all.json()) as { events: unknown[] }).events).toHaveLength(2); - - for (const [since, windowMs] of [ - ["5s", 5_000], - ["2d", 2 * 24 * 60 * 60 * 1000], - ] as const) { - const recent = await app.request(`/v1/events?since=${since}`, { headers: operatorAuth }); - const recentBody = (await recent.json()) as { events: Array<{ event: string }> }; - expect(recentBody.events).toEqual([{ event: "daemon.stopping" }]); - expect(seen.at(-1)).toBe(clock.now() - windowMs); - } - }); - - it("GET /v1/events with before and limit dispatches events.replay with beforeId and limit, and answers HISTORY_NOT_KEPT with 422 and oldestTs in the body and HISTORY_UNAVAILABLE with 503", async () => { - const { app, dispatcher } = buildHarness(); - const asked: unknown[] = []; - dispatcher.handlers["events.replay"] = (input) => { - asked.push(input); - throw new DispatchError("HISTORY_NOT_KEPT", "The history does not hold that event.", { - oldestTs: 1_234, - }); - }; - const notKept = await app.request("/v1/events?before=evt_abc&limit=50&worker=w1", { - headers: operatorAuth, - }); - dispatcher.handlers["events.replay"] = () => { - throw new DispatchError("HISTORY_UNAVAILABLE", "The history could not be read."); - }; - const unavailable = await app.request("/v1/events?before=evt_abc", { headers: operatorAuth }); - - expect(asked).toEqual([{ beforeId: "evt_abc", limit: 50, workerId: "w1" }]); - expect(notKept.status).toBe(422); - expect(await notKept.json()).toMatchObject({ - error: { code: "HISTORY_NOT_KEPT", oldestTs: 1_234 }, - }); - expect(unavailable.status).toBe(503); - expect(await unavailable.json()).toMatchObject({ error: { code: "HISTORY_UNAVAILABLE" } }); - }); - - it.each([ - ["before=evt_abc&limit=0", "limit must be a whole number from 1 to 500"], - ["before=evt_abc&limit=501", "limit must be a whole number from 1 to 500"], - ["before=evt_abc&limit=x", "limit must be a whole number from 1 to 500"], - ["before=not-an-id", "before must be an event id"], - ["since=abc", "Invalid duration: abc"], - ["before=evt_abc&since=1h", "before and since cannot be used together"], - ["limit=50&since=1h", "limit and since cannot be used together"], - ["worker=", "worker must be a worker id of 1 to 128 characters"], - [`worker=${"w".repeat(129)}`, "worker must be a worker id of 1 to 128 characters"], - ])("GET /v1/events?%s answers 400 BAD_REQUEST before dispatching", async (query, message) => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["events.replay"] = () => { - throw new Error("should not be called"); - }; - - const response = await app.request(`/v1/events?${query}`, { headers: operatorAuth }); - - expect(response.status).toBe(400); - expect(await response.json()).toMatchObject({ error: { code: "BAD_REQUEST", message } }); - }); - - describe("GET /v1/stats", () => { - const NOW = Date.parse("2026-10-16T12:00:00Z"); - const HOUR = 3_600_000; - - function statsHarness() { - const harness = buildHarness(); - harness.clock.advance(NOW - harness.clock.now()); - const asked: { from: number; to: number; includeLedger?: boolean }[] = []; - harness.dispatcher.handlers["usage.get"] = (input) => { - asked.push(input as never); - return { window: input }; - }; - const get = (query: string) => - harness.app.request(`/v1/stats${query}`, { headers: operatorAuth }); - return { ...harness, asked, get }; - } - - it("dispatches usage.get for since back from now, from to to, from to now, and the last 24 hours with no parameter", async () => { - const { asked, get } = statsHarness(); - - const answered = await get("?since=1h"); - await get("?from=2026-10-15T00:00Z&to=2026-10-16T00:00Z"); - await get("?from=2026-10-15T00:00Z"); - await get(""); - - expect(answered.status).toBe(200); - expect(await answered.json()).toEqual({ window: { from: NOW - HOUR, to: NOW } }); - expect(asked.map(({ from, to }) => ({ from, to }))).toEqual([ - { from: NOW - HOUR, to: NOW }, - { from: Date.parse("2026-10-15T00:00Z"), to: Date.parse("2026-10-16T00:00Z") }, - { from: Date.parse("2026-10-15T00:00Z"), to: NOW }, - { from: NOW - 24 * HOUR, to: NOW }, - ]); - }); - - it("answers 400 with the Inputs table's message for since with from, to without from, from not before to, a bad duration, the times 1, 2026, 10/15/2026, no zone and an unencoded +, and ledger other than 1, each after the same query without the fault answered 200, and answers 200 for %2B02:00", async () => { - const { asked, get } = statsHarness(); - const time = "from=2026-10-15T00:00Z"; - const timeMessage = (value: string) => - `Invalid time: ${value} (use ISO 8601 with Z or an offset)`; - const cases: readonly (readonly [string, string, string])[] = [ - [`?since=1h&${time}`, "?since=1h", "stats takes since or from, not both"], - ["?to=2026-10-16T00:00Z", `?${time}&to=2026-10-16T00:00Z`, "to needs from"], - [ - "?from=2026-10-16T00:00Z&to=2026-10-15T00:00Z", - "?from=2026-10-15T00:00Z&to=2026-10-16T00:00Z", - "from must be earlier than to, or than now when to is left out", - ], - ["?since=fortnight", "?since=1h", "Invalid duration: fortnight"], - ["?from=1", `?${time}`, timeMessage("1")], - ["?from=2026", `?${time}`, timeMessage("2026")], - ["?from=10/15/2026", `?${time}`, timeMessage("10/15/2026")], - ["?from=2026-10-15T00:00", `?${time}`, timeMessage("2026-10-15T00:00")], - ["?from=2026-10-15T00:00+02:00", `?${time}`, timeMessage("2026-10-15T00:00 02:00")], - ["?since=1h&ledger=yes", "?since=1h&ledger=1", "ledger must be 1"], - ]; - - for (const [bad, good, message] of cases) { - expect((await get(good)).status, good).toBe(200); - const dispatched = asked.length; - const refused = await get(bad); - expect(refused.status, bad).toBe(400); - expect(await refused.json(), bad).toMatchObject({ - error: { code: "BAD_REQUEST", message }, - }); - expect(asked, bad).toHaveLength(dispatched); - } - const offset = await get("?from=2026-10-15T00:00%2B02:00"); - expect(offset.status).toBe(200); - expect(asked.at(-1)).toEqual({ from: Date.parse("2026-10-14T22:00:00Z"), to: NOW }); - }); - - it("passes includeLedger: true only for ledger=1", async () => { - const { asked, get } = statsHarness(); - - await get("?since=1h"); - await get("?since=1h&ledger=1"); - - expect(asked.map((input) => input.includeLedger === true)).toEqual([false, true]); - }); - }); - - it("GET /v1/events/stream dispatches events.subscribe then follows the raw event bus live", async () => { - const { app, dispatcher, eventBus } = buildHarness(); - dispatcher.handlers["events.subscribe"] = (_input, session) => ({ - subscribed: true, - subscriptionId: session.manageEventSubscription(true), - }); - const streamResponse = await app.request("/v1/events/stream", { headers: operatorAuth }); - // `dispatch("events.subscribe", ...)` is awaited before the SSE stream opens, so it has - // already run by the time `app.request` resolves. - expect(dispatcher.calls.some((c) => c.operation === "events.subscribe")).toBe(true); - const framesPromise = readSseFrames(streamResponse, 1); - await Promise.resolve(); - eventBus.emit("daemon.stopping", { reason: "live" }, "test"); - const frames = await framesPromise; - expect(frames[0]?.event).toBe("daemon.stopping"); - }); - - it("GET /v1/events/stream writes a comment or retry line as soon as it opens, before any keepalive is due", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["events.subscribe"] = (_input, session) => ({ - subscribed: true, - subscriptionId: session.manageEventSubscription(true), - }); - const response = await app.request("/v1/events/stream", { headers: operatorAuth }); - const reader = response.body?.getReader(); - if (reader === undefined) throw new Error("response has no body"); - let written = "nothing written"; - void reader.read().then(({ value, done }) => { - written = done ? "stream ended" : new TextDecoder().decode(value); - }); - for (let turn = 0; turn < 50; turn += 1) await settle(); - const writtenBeforeCancel = written; - await reader.cancel(); - - expect(writtenBeforeCancel).toMatch(/^(:[^\n]*|retry: ?\d+)\n\n$/); - }); -}); - -describe("hardening from review", () => { - it("400s an oversized Idempotency-Key as BAD_REQUEST, where a 200-character key is accepted", async () => { - const { app, dispatcher } = buildHarness(); - const accepted = await Promise.race([ - postLeaseRequest(app, defaultBody, { ...agentAuth, "Idempotency-Key": "x".repeat(200) }), - waitForDispatch(dispatcher, "lease.request").then(() => "dispatched" as const), - ]); - if (accepted instanceof Response) expect(accepted.status).toBe(201); - else expect(accepted).toBe("dispatched"); - - const response = await postLeaseRequest(app, defaultBody, { - ...agentAuth, - "Idempotency-Key": "x".repeat(201), - }); - expect(response.status).toBe(400); - const { error } = (await response.json()) as { error: { code: string; message: string } }; - expect(error.code).toBe("BAD_REQUEST"); - expect(error.message).toContain("idempotencyKey"); - }); - - it("clamps an oversized ?wait to the 60s ceiling", async () => { - const { app, clock, dispatcher } = buildHarness(); - const { id } = await createLeaseRequest(app, dispatcher); - - const waitPromise = Promise.resolve( - app.request(`/v1/lease-requests/${id}?wait=999999`, { headers: agentAuth }), - ); - let answered = false; - void waitPromise.then(() => { - answered = true; - }); - await settle(); - clock.advance(59_999); - for (let turn = 0; turn < 50; turn += 1) await settle(); - expect(answered).toBe(false); - clock.advance(1); - const response = await waitPromise; - expect(((await response.json()) as { request: { state: string } }).request.state).toBe( - "queued", - ); - }); - - it("reports the lease's own stored ttlMs, with nothing gateway-side to remember", async () => { - // ADR 0004: the width lives on the lease record, so a payload served by a gateway that - // never saw the request (after a daemon restart, say) still reports the real width - // instead of a mode default standing in for one. - const { app, registry } = buildHarness(); - registry.devices = [makeDevice({ id: "dev_1" })]; - registry.leases = [ - makeLease({ deviceId: "dev_1", id: "lse_1", ownerId: "tok_agent", ttlMs: 123_000 }), - ]; - - const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); - const body = (await response.json()) as { lease: { ttlMs: number } }; - expect(body.lease.ttlMs).toBe(123_000); - }); -}); - -/** - * ADR 0005 §23. The routes themselves are thin -- one dispatch each -- so what is worth - * asserting is which daemon has them at all, that they reach the right operation with the id - * from the path, and that a refusal's typed `details` reach the body. What a worker's own - * dispatcher answers for all four operations is proven in `src/daemon/dispatcher.test.ts`; - * `e2e/single-host-fleet.test.ts` drives `GET /v1/workers` and the drain route end to end. - */ -describe("POST /v1/update", () => { - const answer = (overrides: Record = {}) => ({ - started: false, - upToDate: false, - resolved: null, - version: "0.3.0", - health: "running", - update: null, - lastUpdateFailure: null, - latestVersion: null, - ...overrides, - }); - const post = (app: App, body?: unknown, headers: Record = operatorAuth) => - Promise.resolve( - app.request("/v1/update", { - headers: { - ...headers, - ...(body === undefined ? {} : { "content-type": "application/json" }), - }, - method: "POST", - ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }), - ); - - it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body", async () => { - const { app, dispatcher } = buildHarness(); - // why: `daemon.update`'s own role check runs through the contract's real pipeline; the route adds none. - let reply: Record = answer({ started: true, resolved: "0.4.0" }); - dispatcher.handlers["daemon.update"] = (input, session) => - runDispatch("daemon.update", input, session, { - handlers: { "daemon.update": () => reply }, - }); - - const started = await post(app, { version: "0.4.0", force: true }); - expect({ status: started.status, body: await started.json() }).toEqual({ - status: 202, - body: reply, - }); - expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ - { operation: "daemon.update", input: { version: "0.4.0", force: true } }, - ]); - - for (const [name, following] of [ - [ - "follows a running update", - answer({ - update: { stage: "installing", from: "0.3.0", to: "0.4.0", forced: false, startedAt: 1 }, - }), - ], - ["nothing to do", answer({ upToDate: true, resolved: "0.3.0" })], - ] as const) { - reply = following; - const response = await post(app); - expect({ name, status: response.status }).toEqual({ name, status: 200 }); - expect(await response.json()).toEqual(following); - } - expect(dispatcher.calls.at(-1)?.input).toEqual({}); - - const emptyJson = await app.request("/v1/update", { - headers: { ...operatorAuth, "content-type": "application/json" }, - method: "POST", - body: "", - }); - expect(emptyJson.status).toBe(200); - expect(dispatcher.calls.at(-1)?.input).toEqual({}); - - const asAgent = await post(app, {}, agentAuth); - expect(asAgent.status).toBe(403); - expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - - const callsBefore = dispatcher.calls.length; - for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { - const refused = await post(app, body); - expect({ body, status: refused.status }).toEqual({ body, status: 400 }); - } - expect(dispatcher.calls).toHaveLength(callsBefore); - }); -}); - -describe("POST /v1/workers/{id}/update", () => { - const answer = (overrides: Record = {}) => ({ - started: false, - upToDate: false, - resolved: null, - version: "0.3.0", - health: "running", - update: null, - lastUpdateFailure: null, - latestVersion: null, - workerId: "wrk_1", - ...overrides, - }); - - it("answers 202 when the call began the update and 200 when it followed one, dispatches worker.update with the id from the path and the body, refuses an agent and a bad body", async () => { - const { app, dispatcher } = buildHarness(); - let reply: Record = answer({ started: true, resolved: "0.4.0" }); - dispatcher.handlers["worker.update"] = (input, session) => - runDispatch("worker.update", input, session, { - handlers: { "worker.update": () => reply }, - }); - const post = (body?: unknown, headers: Record = operatorAuth) => - Promise.resolve( - app.request("/v1/workers/wrk_1/update", { - headers: { - ...headers, - ...(body === undefined ? {} : { "content-type": "application/json" }), - }, - method: "POST", - ...(body === undefined ? {} : { body: JSON.stringify(body) }), - }), - ); - - const started = await post({ version: "0.4.0", force: true }); - expect(started.status).toBe(202); - expect(await started.json()).toEqual(reply); - expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ - { operation: "worker.update", input: { force: true, version: "0.4.0", workerId: "wrk_1" } }, - ]); - - reply = answer({ upToDate: true, resolved: "0.3.0" }); - const following = await post(); - expect(following.status).toBe(200); - expect(dispatcher.calls.at(-1)?.input).toEqual({ workerId: "wrk_1" }); - - const asAgent = await post({}, agentAuth); - expect(asAgent.status).toBe(403); - - const callsBefore = dispatcher.calls.length; - for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { - const refused = await post(body); - expect({ body, status: refused.status }).toEqual({ body, status: 400 }); - } - expect(dispatcher.calls).toHaveLength(callsBefore); - - for (const [code, status] of [ - ["WORKER_NOT_CONNECTED", 409], - ["UPDATE_UNSUPPORTED_INSTALL", 409], - ["UPDATE_SHARED_INSTALL", 409], - ["UPDATE_RESOLVE_FAILED", 502], - ["UNKNOWN_WORKER", 404], - ] as const) { - dispatcher.handlers["worker.update"] = () => { - throw new DispatchError(code, "refused"); - }; - const refusal = await post(); - expect({ code, status: refusal.status }).toEqual({ code, status }); - expect(((await refusal.json()) as { error: { code: string } }).error.code).toBe(code); - } - }); -}); - -describe("worker routes", () => { - function gatewayHarness() { - return buildHarness(); - } - - it("are routes on a worker too (ADR 0012 §3), each dispatching its operation", async () => { - const { app, dispatcher } = buildHarness(); - dispatcher.handlers["worker.list"] = () => ({ workers: [] }); - for (const operation of ["worker.drain", "worker.undrain", "worker.remove"] as const) { - dispatcher.handlers[operation] = () => { - throw new DispatchError("UNSUPPORTED_IN_WORKER_MODE", `${operation} is refused`, { - operation, - }); - }; - } - - const list = await app.request("/v1/workers", { headers: operatorAuth }); - const drain = await app.request("/v1/workers/w/drain", { - headers: operatorAuth, - method: "POST", - }); - const undrain = await app.request("/v1/workers/w/drain", { - headers: operatorAuth, - method: "DELETE", - }); - const remove = await app.request("/v1/workers/w", { headers: operatorAuth, method: "DELETE" }); - - expect(list.status).toBe(200); - expect(await list.json()).toEqual({ workers: [] }); - // Each refusal names the operation its handler was reached through, so the body proves - // which route dispatched what. - const refusals = [drain, undrain, remove]; - expect(refusals.map((response) => response.status)).toEqual([501, 501, 501]); - expect(await Promise.all(refusals.map((response) => response.json()))).toEqual( - ["worker.drain", "worker.undrain", "worker.remove"].map((operation) => ({ - error: { code: "UNSUPPORTED_IN_WORKER_MODE", message: expect.any(String), operation }, - })), - ); - }); - - it("GET /v1/workers returns the views", async () => { - const { app, dispatcher } = gatewayHarness(); - dispatcher.handlers["worker.list"] = () => ({ - workers: [ - { - catalog: [], - connection: "connected", - devices: [], - drained: false, - id: "wrk_1", - lastSeenAt: 1, - leases: [], - }, - ], - }); - - const response = await app.request("/v1/workers", { headers: operatorAuth }); - - expect(response.status).toBe(200); - expect(await response.json()).toMatchObject({ workers: [{ id: "wrk_1" }] }); - }); - - it("GET /v1/inbox dispatches inbox.get for the caller and answers its items", async () => { - const { app, dispatcher } = gatewayHarness(); - const item = { - condition: "disconnected", - group: "Broken", - key: "wrk_1/disconnected", - worker: { - connection: "disconnected", - drained: false, - heldLeases: 0, - id: "wrk_1", - lastSeenAt: 1, - leasedDevices: 0, - }, - }; - dispatcher.handlers["inbox.get"] = () => ({ items: [item] }); - - const response = await app.request("/v1/inbox", { headers: operatorAuth }); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ items: [item] }); - expect(dispatcher.calls.map((call) => [call.operation, call.session.role])).toEqual([ - ["inbox.get", "admin"], - ]); - }); - - it("GET /v1/config dispatches config.get for the caller and answers the config; an agent token is a 403, no token a 401", async () => { - const { app, dispatcher } = gatewayHarness(); - const config = testConfig({ update: { check: false } }, "gateway"); - // why: `config.get`'s own role check runs through the contract's real pipeline; the route adds none. - dispatcher.handlers["config.get"] = (input, session) => - runDispatch("config.get", input, session, { - handlers: { - "config.get": (configInput) => { - expect(configInput).toEqual({}); - return config; - }, - }, - }); - - const asOperator = await app.request("/v1/config", { headers: operatorAuth }); - const asAgent = await app.request("/v1/config", { headers: agentAuth }); - const anonymous = await app.request("/v1/config"); - - expect(asOperator.status).toBe(200); - expect(await asOperator.json()).toMatchObject({ mode: "gateway", update: { check: false } }); - expect(asAgent.status).toBe(403); - expect(((await asAgent.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - expect(anonymous.status).toBe(401); - expect(dispatcher.calls.map((call) => [call.operation, call.session.role])).toEqual([ - ["config.get", "admin"], - ["config.get", "agent"], - ]); - }); - - it("maps drain, undrain and remove onto their operations, with the id from the path", async () => { - const { app, dispatcher } = gatewayHarness(); - dispatcher.handlers["worker.drain"] = (input: never) => ({ - drained: true, - workerId: (input as { workerId: string }).workerId, - }); - dispatcher.handlers["worker.undrain"] = (input: never) => ({ - drained: false, - workerId: (input as { workerId: string }).workerId, - }); - dispatcher.handlers["worker.remove"] = (input: never) => ({ - removed: true, - workerId: (input as { workerId: string }).workerId, - }); - - const drained = await app.request("/v1/workers/wrk_1/drain", { - headers: operatorAuth, - method: "POST", - }); - expect(await drained.json()).toEqual({ drained: true, workerId: "wrk_1" }); - const undrained = await app.request("/v1/workers/wrk_1/drain", { - headers: operatorAuth, - method: "DELETE", - }); - expect(await undrained.json()).toEqual({ drained: false, workerId: "wrk_1" }); - const removed = await app.request("/v1/workers/wrk_2", { - headers: operatorAuth, - method: "DELETE", - }); - expect(await removed.json()).toEqual({ removed: true, workerId: "wrk_2" }); - - expect(dispatcher.calls.map((call) => call.operation)).toEqual([ - "worker.drain", - "worker.undrain", - "worker.remove", - ]); - }); - - it("puts a refusal's typed details in the body, so a client need not parse prose", async () => { - const { app, dispatcher } = gatewayHarness(); - dispatcher.handlers["worker.remove"] = () => { - throw new DispatchError("WORKER_CONNECTED", "Worker wrk_1 is still connected", { - workerId: "wrk_1", - }); - }; - - const response = await app.request("/v1/workers/wrk_1", { - headers: operatorAuth, - method: "DELETE", - }); - - expect(response.status).toBe(409); - expect(await response.json()).toEqual({ - error: { code: "WORKER_CONNECTED", message: expect.any(String), workerId: "wrk_1" }, - }); - }); - - it("answers 501 for an operation a gateway does not implement", async () => { - const { app, dispatcher } = gatewayHarness(); - dispatcher.handlers["lease.request"] = () => { - throw new DispatchError( - "UNSUPPORTED_IN_GATEWAY_MODE", - "lease.request is not available on a gateway yet", - { operation: "lease.request" }, - ); - }; - - const response = await postLeaseRequest(app, defaultBody); - - expect(response.status).toBe(501); - expect(await response.json()).toMatchObject({ - error: { code: "UNSUPPORTED_IN_GATEWAY_MODE", operation: "lease.request" }, - }); - }); -}); - -describe("the stream mount's listed route", () => { - it("is the route its handler answers: the listed path gets a status the definition lists, and /v1/stream/x gets its listed 404", async () => { - const route = streamModule.http?.routes[0]; - if (route === undefined) throw new Error("the stream definition lists no route"); - const link = { - serve: () => - Promise.resolve({ - body: new ReadableStream(), - close: () => Promise.resolve(), - meta: { codec: "h264" as const, fps: 30, height: 720, width: 330 }, - }), - }; - const { app } = buildHarness({ - mounts: [{ prefix: "stream", ...createStreamMount({ link }) }], - }); - const listed = Object.keys(route.statuses).map(Number); - const answer = (path: string, method = "GET") => - app.request(path, { headers: agentAuth, method }); - - const listedPath = route.path.replace("{workerId}", "w1").replace("{deviceId}", "dev_1"); - const found = await answer(listedPath); - const other = await answer("/v1/stream/x"); - const wrongMethod = await answer(listedPath, "POST"); - - expect(found.status).toBe(200); - expect(found.headers.get("content-type")).toBe("application/vnd.simlock.stream+h264"); - expect([other.status, wrongMethod.status]).toEqual([404, 405]); - expect(listed).toEqual( - expect.arrayContaining([found.status, other.status, wrongMethod.status]), - ); - expect(await other.json()).toMatchObject({ error: { code: "UNKNOWN_DEVICE" } }); - await found.body?.cancel(); - }); -}); - -describe("ETag on a JSON read under /v1", () => { - it("304 and no body for an unchanged read, 200 for a stale tag, a new ETag once the data changes, no ETag on a 404 or a mount", async () => { - const tokens: { id: string }[] = [{ id: "tok_1" }]; - const { app, dispatcher, eventBus } = buildHarness({ - mounts: [ - { - prefix: "json-mount", - handle: () => Response.json({ ok: true }), - }, - ] as never, - }); - dispatcher.handlers["token.list"] = () => ({ tokens: [...tokens] }); - dispatcher.handlers["events.subscribe"] = (_input, session) => ({ - subscribed: true, - subscriptionId: session.manageEventSubscription(true), - }); - const read = (path: string, headers: Record = {}) => - app.request(path, { headers: { ...operatorAuth, ...headers } }); - - const first = await read("/v1/tokens"); - const tag = first.headers.get("etag"); - expect(first.status).toBe(200); - expect(tag).toMatch(/^"[^"]+"$/); - - const unchanged = await read("/v1/tokens", { "If-None-Match": tag ?? "" }); - expect(unchanged.status).toBe(304); - expect(await unchanged.text()).toBe(""); - expect(unchanged.headers.get("etag")).toBe(tag); - - const stale = await read("/v1/tokens", { "If-None-Match": '"stale"' }); - expect(stale.status).toBe(200); - expect(stale.headers.get("etag")).toBe(tag); - - tokens.push({ id: "tok_2" }); - const changed = await read("/v1/tokens", { "If-None-Match": tag ?? "" }); - expect(changed.status).toBe(200); - expect(changed.headers.get("etag")).not.toBe(tag); - - const missing = await read("/v1/leases/lse_none"); - expect(missing.status).toBe(404); - expect(missing.headers.get("etag")).toBeNull(); - - const mount = await read("/v1/json-mount/x"); - expect(mount.status).toBe(200); - expect(mount.headers.get("content-type")).toContain("application/json"); - expect(mount.headers.get("etag")).toBeNull(); - - const streamOrStall = await Promise.race([ - read("/v1/events/stream"), - new Promise<"stalled">((resolve) => setTimeout(() => resolve("stalled"), 500)), - ]); - if (streamOrStall === "stalled") expect.fail("the event stream was buffered, not answered"); - const stream = streamOrStall; - expect(stream.headers.get("etag")).toBeNull(); - const framesPromise = readSseFrames(stream, 1); - await Promise.resolve(); - eventBus.emit("daemon.stopping", { reason: "live" }, "test"); - expect((await framesPromise)[0]?.event).toBe("daemon.stopping"); - }); -}); - -describe("Cache-Control on a JSON answer under /v1", () => { - it("a JSON read, its 304, a JSON error and /v1/healthz each carry Cache-Control: no-store, so a CDN never stores an answer, and a mount's answer keeps the Cache-Control it set", async () => { - const { app, dispatcher } = buildHarness({ - mounts: [ - { prefix: "json-mount", handle: () => Response.json({ ok: true }) }, - { - prefix: "cached-mount", - handle: () => Response.json({ ok: true }, { headers: { "Cache-Control": "max-age=60" } }), - }, - ] as never, - }); - dispatcher.handlers["token.list"] = () => ({ tokens: [{ id: "tok_1" }] }); - const read = (path: string, headers: Record = {}) => - app.request(path, { headers: { ...operatorAuth, ...headers } }); - - const first = await read("/v1/tokens"); - const unchanged = await read("/v1/tokens", { - "If-None-Match": first.headers.get("etag") ?? "", - }); - const missing = await read("/v1/leases/lse_none"); - const health = await app.request("/v1/healthz"); - const plainMount = await read("/v1/json-mount"); - const cachedMount = await read("/v1/cached-mount"); - - expect([first.status, unchanged.status, missing.status, health.status]).toEqual([ - 200, 304, 404, 200, - ]); - expect({ - plain: plainMount.headers.get("cache-control"), - cached: cachedMount.headers.get("cache-control"), - }).toEqual({ plain: null, cached: "max-age=60" }); - expect({ - read: first.headers.get("cache-control"), - notModified: unchanged.headers.get("cache-control"), - error: missing.headers.get("cache-control"), - healthz: health.headers.get("cache-control"), - }).toEqual({ - read: "no-store", - notModified: "no-store", - error: "no-store", - healthz: "no-store", - }); - }); -}); From 9df7067414c376b12308a193ecfcdda2d74a2a6d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:31:49 +0100 Subject: [PATCH 57/83] test(contract): split operations.test.ts by topic Protocol-bump: none (test-only split) --- .../operations-lease-device-daemon.test.ts | 495 +++++ ...rations-round-trips-catalog-config.test.ts | 656 +++++++ ...perations-round-trips-lease-status.test.ts | 518 +++++ src/contract/operations.test.ts | 1669 +---------------- 4 files changed, 1671 insertions(+), 1667 deletions(-) create mode 100644 src/contract/operations-lease-device-daemon.test.ts create mode 100644 src/contract/operations-round-trips-catalog-config.test.ts create mode 100644 src/contract/operations-round-trips-lease-status.test.ts diff --git a/src/contract/operations-lease-device-daemon.test.ts b/src/contract/operations-lease-device-daemon.test.ts new file mode 100644 index 000000000..6979ba06b --- /dev/null +++ b/src/contract/operations-lease-device-daemon.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; +import { OPERATIONS, requestedClass, requestedDevice, PUSH_SCHEMAS } from "./index.js"; + +describe("lease.request names a model, a class, or nothing (ADR 0015 §1)", () => { + const input = OPERATIONS["lease.request"].input; + + it("refuses a request with both a class and a model, with a message naming the two fields", () => { + const result = input.safeParse({ class: "phone", model: "iPhone 16", platform: "ios" }); + + expect(result.success).toBe(false); + const message = result.success ? "" : result.error.issues.map((issue) => issue.message).join(); + expect(message).toContain("`model`"); + expect(message).toContain("`class`"); + }); + + it("accepts a request with a class only, a model only, or neither", () => { + expect(input.safeParse({ class: "tablet", platform: "ios" }).success).toBe(true); + expect(input.safeParse({ model: "iPhone 16", platform: "ios" }).success).toBe(true); + expect(input.safeParse({ platform: "ios" }).success).toBe(true); + }); + + it("refuses a class that is not one of the seven", () => { + expect(input.safeParse({ class: "phablet", platform: "ios" }).success).toBe(false); + }); + + it("requestedDevice copies the class and leaves out the model a request did not name", () => { + expect(requestedDevice(input.parse({ class: "watch", platform: "ios" }))).toStrictEqual({ + class: "watch", + platform: "ios", + }); + expect(requestedDevice(input.parse({ platform: "android" }))).toStrictEqual({ + platform: "android", + }); + expect(requestedDevice(input.parse({ model: "Pixel 8", platform: "android" }))).toStrictEqual({ + model: "Pixel 8", + platform: "android", + }); + }); + + it("requestedClass answers the class named, phone for neither field, and none for an exact model", () => { + expect(requestedClass({ class: "tv" })).toBe("tv"); + expect(requestedClass({})).toBe("phone"); + expect(requestedClass({ model: "iPhone 16" })).toBeUndefined(); + }); +}); + +describe("lease.request physical (ADR 0022)", () => { + const input = OPERATIONS["lease.request"].input; + + it.each([ + ["mode", { mode: "slim" }], + ["imageTag", { imageTag: "x" }], + ["allowDownload", { allowDownload: true }], + ])("a physical request with %s is BAD_REQUEST naming the field", (field, extra) => { + expect(input.safeParse({ physical: true, platform: "ios" }).success).toBe(true); + + const result = input.safeParse({ ...extra, physical: true, platform: "ios" }); + + expect(result.success).toBe(false); + const message = result.success ? "" : result.error.issues.map((issue) => issue.message).join(); + expect(message).toContain(`\`${field}\``); + }); + + it.each([ + ["mode", { mode: "slim" }], + ["imageTag", { imageTag: "x" }], + ["allowDownload", { allowDownload: true }], + ])("a physical request with %s reports the issue at that field's path", (field, extra) => { + const result = input.safeParse({ ...extra, physical: true, platform: "ios" }); + + expect(result.success ? [] : result.error.issues.map((issue) => issue.path)).toStrictEqual([ + [field], + ]); + }); + + it("a physical request with allowDownload false passes", () => { + const result = input.safeParse({ allowDownload: false, physical: true, platform: "ios" }); + + expect(result.success).toBe(true); + expect(result.success ? (result.data as { physical?: boolean }).physical : undefined).toBe( + true, + ); + }); + + it("requestedDevice carries physical true only when the input set it", () => { + expect(requestedDevice(input.parse({ physical: true, platform: "ios" }))).toStrictEqual({ + physical: true, + platform: "ios", + }); + expect(requestedDevice(input.parse({ physical: false, platform: "ios" }))).toStrictEqual({ + platform: "ios", + }); + expect(requestedDevice(input.parse({ platform: "ios" }))).toStrictEqual({ platform: "ios" }); + }); +}); + +describe("status.get physicalQueueDepth (ADR 0022)", () => { + const host = { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }; + const entry = { + atRamBudget: false, + limit: 1, + maxRunning: 1, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }; + const capacity = { + android: entry, + global: { maxRunning: 1, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: entry, + }; + const output = OPERATIONS["status.get"].output; + + it("a status carries the physical line's depth beside the virtual one", () => { + const parsed = output.parse({ + capacity, + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + devices: [], + host, + leases: [], + physicalQueueDepth: 1, + queueDepth: 2, + }); + + expect(parsed.queueDepth).toBe(2); + expect((parsed as { physicalQueueDepth?: number }).physicalQueueDepth).toBe(1); + }); + + it("a worker's entry in a gateway's status carries it too", () => { + const parsed = output.parse({ + capacity, + daemon: { + health: "running", + mode: "gateway", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + devices: [], + host, + leases: [], + queueDepth: 0, + workers: [ + { + capacity, + catalog: [], + connection: "connected", + devices: [], + drained: false, + health: "running", + host, + id: "wrk_1", + label: "mac-mini-1", + lastSeenAt: 10, + leases: [], + physicalQueueDepth: 1, + queueDepth: 0, + version: "0.3.0", + }, + ], + }); + + expect( + (parsed.workers?.[0] as { physicalQueueDepth?: number } | undefined)?.physicalQueueDepth, + ).toBe(1); + }); +}); + +describe("device.add and device.remove ids (ADR 0022 §2)", () => { + const long = "x".repeat(201); + + it.each([ + ["201 characters", long], + ["a space", "00008110 001A2B3C4D5E"], + ["a tab", "00008110\t001A2B3C4D5E"], + ["a control character", "00008110\u0007001A2B3C4D5E"], + ["a leading dash", "-00008110-001A2B3C4D5E"], + ["nothing", ""], + ])("refuses an id with %s", (_case, id) => { + expect(OPERATIONS["device.add"].input.safeParse({ id, platform: "ios" }).success).toBe(false); + expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(false); + }); + + it("takes an id of exactly 200 characters, and refuses a field it does not declare", () => { + const id = "x".repeat(200); + + expect(OPERATIONS["device.add"].input.safeParse({ id, platform: "ios" }).success).toBe(true); + expect( + OPERATIONS["device.add"].input.safeParse({ id: "A", platform: "ios", worker: "w1" }).success, + ).toBe(false); + expect(OPERATIONS["device.remove"].input.safeParse({ id: "A", yes: true }).success).toBe(false); + }); + + it.each([ + ["a unit separator", "A\u001fB"], + ["a delete", "A\u007fB"], + ["a C1 control", "A\u0080B"], + ["the last C1 control", "A\u009fB"], + ["a no-break space", "A\u00a0B"], + ["a newline", "A\nB"], + ])("refuses an id with %s", (_case, id) => { + expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(false); + }); + + it.each([ + ["a letter beyond ASCII", "caf\u00e9"], + ["a dash inside", "A-B"], + ["a tilde and an inverted exclamation mark", "A~\u00a1B"], + ])("takes an id with %s", (_case, id) => { + expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(true); + }); + + it("says why an id is refused", () => { + const messages = (id: string) => + OPERATIONS["device.remove"].input + .safeParse({ id }) + .error?.issues.map((issue) => issue.message); + + expect(messages("A B")).toEqual(["must not contain whitespace or control characters"]); + expect(messages("-A")).toEqual(["must not start with -"]); + }); + + it("declares device.add and device.remove with their names, outputs and strict inputs", () => { + const output = { + appCount: 3, + class: "tablet", + driverDeviceId: "U", + id: "dev_1", + model: "iPad", + osVersion: "18", + platform: "ios", + reenrolled: true, + }; + + expect(OPERATIONS["device.add"].name).toBe("device.add"); + expect(OPERATIONS["device.remove"].name).toBe("device.remove"); + expect(OPERATIONS["device.add"].output.parse(output)).toEqual(output); + expect(OPERATIONS["device.add"].output.safeParse({ ...output, class: "watch" }).success).toBe( + false, + ); + expect(OPERATIONS["device.add"].output.safeParse({ ...output, appCount: -1 }).success).toBe( + false, + ); + expect(OPERATIONS["device.remove"].output.parse({ driverDeviceId: "U", id: "U" })).toEqual({ + driverDeviceId: "U", + id: "U", + }); + expect(OPERATIONS["device.add"].input.safeParse({ id: "A" }).success).toBe(false); + expect(OPERATIONS["device.add"].input.safeParse({ id: "A", platform: "windows" }).success).toBe( + false, + ); + expect(OPERATIONS["device.remove"].input.safeParse({}).success).toBe(false); + }); +}); + +describe("lease.request requires", () => { + const base = { model: "iPhone 17", platform: "ios" } as const; + const parse = (extra: Record) => + OPERATIONS["lease.request"].input.safeParse({ ...base, ...extra }); + const names = (count: number) => + Array.from({ length: count }, (_, index) => `cap-${String(index)}`); + + it.each([ + ["one name", ["agent-device"]], + ["an empty list", []], + ["8 names", names(8)], + ])("accepts %s", (_label, requires) => { + expect(parse({ requires }).success).toBe(true); + }); + + it("accepts a request without requires, and one with requires: [] parses to the same device", () => { + expect(parse({}).success).toBe(true); + const input = parse({ requires: [] }); + expect(input.success && requestedDevice(input.data)).toEqual({ + model: "iPhone 17", + platform: "ios", + }); + }); + + it.each([ + ["an upper-case name", ["Agent-Device"]], + ["a duplicate name", ["agent-device", "agent-device"]], + ["9 names", names(9)], + ])("refuses requires with %s with BAD_REQUEST-shaped issues on requires", (_label, requires) => { + const parsed = parse({ requires }); + + expect(parsed.success).toBe(false); + expect( + !parsed.success && parsed.error.issues.every((issue) => issue.path[0] === "requires"), + ).toBe(true); + }); + + it("refuses requires together with idempotencyKey, naming both, where each alone is accepted", () => { + expect(parse({ idempotencyKey: "key-1" }).success).toBe(true); + expect(parse({ requires: ["agent-device"] }).success).toBe(true); + + const both = parse({ idempotencyKey: "key-1", requires: ["agent-device"] }); + + expect(both.success).toBe(false); + expect(!both.success && both.error.issues.map((issue) => issue.message)).toContain( + "requires cannot be combined with idempotencyKey", + ); + }); +}); + +describe("relay.request on a byte channel", () => { + const request = { + hasBody: false, + headers: [], + method: "GET", + relayId: "r-1", + target: "agent-device", + tenant: "gw.g.a", + url: "http://gateway.test/x", + }; + const parse = (input: Record) => + OPERATIONS["relay.request"].input.safeParse(input).success; + + it("accepts a channel of 1 to 32 characters of [A-Za-z0-9-] and refuses an empty, a 33-character and a bad-character one, and a request without one", () => { + expect(parse({ ...request, channel: "a" })).toBe(true); + expect(parse({ ...request, channel: "Ab-9".repeat(8) })).toBe(true); + expect(parse({ ...request, channel: "" })).toBe(false); + expect(parse({ ...request, channel: "a".repeat(33) })).toBe(false); + expect(parse({ ...request, channel: "a b" })).toBe(false); + expect(parse({ ...request, channel: "a_b" })).toBe(false); + expect(parse(request)).toBe(false); + }); + + it("accepts twoWay with GET and no body, and refuses it with another method, with a body, or as false, where the same request without twoWay is accepted", () => { + const channel = { ...request, channel: "a" }; + expect(parse({ ...channel, twoWay: true })).toBe(true); + expect(parse({ ...channel, method: "POST" })).toBe(true); + expect(parse({ ...channel, method: "POST", twoWay: true })).toBe(false); + expect(parse({ ...channel, hasBody: true, method: "PUT" })).toBe(true); + expect(parse({ ...channel, hasBody: true, method: "PUT", twoWay: true })).toBe(false); + expect(parse({ ...channel, twoWay: false })).toBe(false); + }); + + it("OPERATIONS has no relay.body and PUSH_SCHEMAS has no relay-chunk", () => { + expect(Object.keys(OPERATIONS)).not.toContain("relay.body"); + expect(Object.keys(PUSH_SCHEMAS)).not.toContain("relay-chunk"); + expect(Object.keys(PUSH_SCHEMAS)).toContain("relay-head"); + }); +}); + +describe("usage.get input", () => { + const parse = (input: unknown) => OPERATIONS["usage.get"].input.safeParse(input); + const issues = (input: unknown) => { + const result = parse(input); + return result.success ? [] : result.error.issues.map((issue) => issue.message); + }; + const DAY = 24 * 60 * 60 * 1000; + + it("takes epoch milliseconds with from before to, up to ninety days apart, and names the bound it refuses", () => { + expect(parse({ from: 0, to: 1 }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_000 + 90 * DAY }).success).toBe(true); + expect(parse({ from: 1_000, to: 1_001 + 90 * DAY }).success).toBe(false); + expect(parse({ from: 5, to: 5 }).success).toBe(false); + expect(parse({ from: 6, to: 5 }).success).toBe(false); + expect(issues({ from: 6, to: 5 })).toEqual(["from must be before to"]); + expect(issues({ from: 0, to: 91 * DAY })).toEqual([ + "the window must not be longer than 90 days", + ]); + }); + + it("refuses a bound that is not an integer, is negative, is above 8.64e15 or is a string, and takes the last date a Date holds", () => { + expect(parse({ from: 8.64e15 - 1, to: 8.64e15 }).success).toBe(true); + expect(parse({ from: 0.5, to: 10 }).success).toBe(false); + expect(parse({ from: 0, to: 10.5 }).success).toBe(false); + expect(parse({ from: -1, to: 10 }).success).toBe(false); + expect(parse({ from: 8.64e15, to: 8.64e15 + 2 }).success).toBe(false); + expect(parse({ from: "0", to: 10 }).success).toBe(false); + expect(parse({ from: 0 }).success).toBe(false); + }); +}); + +describe("events.replay input", () => { + const parse = (input: unknown) => OPERATIONS["events.replay"].input.safeParse(input); + + it("accepts beforeId and limit in bounds, and refuses limit 0, 501 and 1.5, a beforeId of evt_ or abc, and beforeId or limit with sinceTs; accepts a workerId of 1 to 128 characters with any of them, and refuses an empty one or 129", () => { + expect(parse({ beforeId: "evt_abc", limit: 1 }).success).toBe(true); + expect(parse({ limit: 500 }).success).toBe(true); + expect(parse({ sinceTs: 5 }).success).toBe(true); + for (const workerId of ["w", "w".repeat(128)]) { + expect(parse({ workerId }).success).toBe(true); + expect(parse({ limit: 5, workerId }).success).toBe(true); + expect(parse({ sinceTs: 5, workerId }).success).toBe(true); + } + for (const workerId of ["", "w".repeat(129)]) expect(parse({ workerId }).success).toBe(false); + + for (const limit of [0, 501, 1.5]) expect(parse({ limit }).success).toBe(false); + for (const beforeId of ["evt_", "abc"]) expect(parse({ beforeId }).success).toBe(false); + expect(parse({ beforeId: "evt_abc", sinceTs: 5 }).success).toBe(false); + expect(parse({ limit: 5, sinceTs: 5 }).success).toBe(false); + }); +}); + +describe("daemon.update", () => { + const SCHEMA = "https://json-schema.org/draft/2020-12/schema"; + const exactVersion = { + type: "string", + maxLength: 64, + pattern: "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z.-]+)?$", + }; + const version = { type: "string", maxLength: 64 }; + const stateBase = { + from: version, + to: version, + forced: { type: "boolean" }, + startedAt: { type: "integer", minimum: 0, maximum: 9007199254740991 }, + }; + const state = (stage: string, extra: Record = {}) => ({ + type: "object", + properties: { stage: { type: "string", const: stage }, ...extra, ...stateBase }, + required: ["stage", ...Object.keys(extra), "from", "to", "forced", "startedAt"], + additionalProperties: false, + }); + + it("keeps the input and the answer the shapes every daemon version accepts, because it is a frozen call", () => { + expect(z.toJSONSchema(OPERATIONS["daemon.update"].input, { io: "input" })).toEqual({ + $schema: SCHEMA, + type: "object", + properties: { + version: exactVersion, + force: { type: "boolean" }, + query: { type: "boolean" }, + }, + }); + expect(z.toJSONSchema(OPERATIONS["daemon.update"].output)).toEqual({ + $schema: SCHEMA, + type: "object", + properties: { + started: { type: "boolean" }, + upToDate: { type: "boolean" }, + resolved: { anyOf: [exactVersion, { type: "null" }] }, + version, + health: { type: "string", enum: ["starting", "running", "failed"] }, + update: { + anyOf: [ + { + oneOf: [ + state("waiting", { + leases: { type: "integer", minimum: 0, maximum: 9007199254740991 }, + }), + state("installing"), + state("restarting"), + ], + }, + { type: "null" }, + ], + }, + lastUpdateFailure: { + anyOf: [ + { + type: "object", + properties: { + stage: { type: "string", enum: ["installing", "restarting"] }, + from: version, + to: version, + message: { type: "string", maxLength: 2000 }, + at: { type: "integer", minimum: 0, maximum: 9007199254740991 }, + }, + required: ["stage", "from", "to", "message", "at"], + additionalProperties: false, + }, + { type: "null" }, + ], + }, + latestVersion: { anyOf: [version, { type: "null" }] }, + }, + required: [ + "started", + "upToDate", + "resolved", + "version", + "health", + "update", + "lastUpdateFailure", + "latestVersion", + ], + additionalProperties: false, + }); + }); +}); diff --git a/src/contract/operations-round-trips-catalog-config.test.ts b/src/contract/operations-round-trips-catalog-config.test.ts new file mode 100644 index 000000000..fa14a8593 --- /dev/null +++ b/src/contract/operations-round-trips-catalog-config.test.ts @@ -0,0 +1,656 @@ +import { describe, expect, it } from "vitest"; +import { OPERATIONS, PUSH_SCHEMAS, platformCatalogSchema } from "./index.js"; + +describe("operation input/output round trips", () => { + it("status.get: round-trips a gateway's aggregate, workers and all (ADR 0005 §20)", () => { + const capacityEntry = { + atRamBudget: false, + running: 1, + maxRunning: 2, + reserved: 0, + overLimit: false, + limit: 2, + warm: 0, + used: 1, + }; + const capacity = { + ios: capacityEntry, + android: capacityEntry, + global: { running: 1, maxRunning: 4, reserved: 0, overLimit: false, warm: 0 }, + }; + const lease = { + id: "lease_1", + deviceId: "dev_1", + requesterId: "agent-1", + ownerId: "agent-1", + grantedAt: 1, + ttlMs: 2, + ttlDeadline: 3, + lastRenewedAt: 1, + idChosenByRequester: false, + workerId: "wrk_1", + }; + const parsed = OPERATIONS["status.get"].output.parse({ + devices: [ + { + physical: false, + id: "dev_1", + spec: { physical: false, platform: "ios", model: "iPhone 17", osVersion: "26.0" }, + mode: "full", + servesDefaultMode: true, + state: "leased", + workerId: "wrk_1", + }, + ], + leases: [lease], + capacity, + daemon: { + health: "running", + mode: "gateway", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { os: "Linux", osVersion: "6.8.0", arch: "x64", tools: [] }, + queueDepth: 0, + workers: [ + { + id: "wrk_1", + label: "mac-mini-1", + connection: "connected", + drained: false, + lastSeenAt: 10, + health: "running", + version: "0.3.0", + capacity, + host: { + os: "macOS", + osVersion: "15.5", + arch: "arm64", + tools: [{ platform: "android", name: "emulator", version: "35.4.9" }], + }, + queueDepth: 0, + leases: [lease], + devices: [], + catalog: [ + { + platform: "ios", + models: ["iPhone 17"], + runtimes: ["26.0"], + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["26.0"] }, + }, + ], + }, + { + id: "wrk_2", + connection: "incompatible", + drained: false, + lastSeenAt: 11, + protocol: { gateway: { min: 4, max: 4 }, worker: { min: 3, max: 3 } }, + leases: [], + devices: [], + catalog: [], + }, + ], + }); + expect(parsed.workers?.[1]?.protocol?.worker).toEqual({ min: 3, max: 3 }); + expect(parsed.workers?.[0]?.host?.tools[0]?.name).toBe("emulator"); + expect(parsed.devices?.[0]?.workerId).toBe("wrk_1"); + expect(parsed.leases?.[0]?.workerId).toBe("wrk_1"); + }); + + it("catalog.get: round-trips a gateway's per-entry worker annotations (ADR 0005 §21)", () => { + const parsed = OPERATIONS["catalog.get"].output.parse({ + platforms: [ + { + platform: "ios", + models: ["iPhone 17"], + runtimes: ["26.0"], + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "iPhone 17": ["26.0"] }, + modelWorkers: { "iPhone 17": ["wrk_1", "wrk_2"] }, + runtimeWorkers: { "26.0": ["wrk_1"] }, + }, + ], + }); + expect(parsed.platforms[0]?.modelWorkers).toEqual({ "iPhone 17": ["wrk_1", "wrk_2"] }); + }); + + it("catalog.get rejects a platform entry without modelRuntimes", () => { + const entry = { + platform: "ios", + models: ["iPhone 17"], + runtimes: ["26.0"], + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + }; + expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).toThrow( + /modelRuntimes/, + ); + expect(() => + OPERATIONS["catalog.get"].output.parse({ + platforms: [{ ...entry, modelRuntimes: { "iPhone 17": ["26.0"] } }], + }), + ).not.toThrow(); + }); + + it("catalog.get rejects an entry without modelClasses and one whose class is not one of the seven", () => { + const entry = { + platform: "ios", + models: ["iPhone 17"], + runtimes: ["26.0"], + classDefaults: {}, + modelAliases: {}, + modelRuntimes: { "iPhone 17": ["26.0"] }, + }; + const parse = (modelClasses?: unknown) => + platformCatalogSchema.parse(modelClasses === undefined ? entry : { ...entry, modelClasses }); + + expect(() => parse()).toThrow(/modelClasses/); + expect(() => parse({ "iPhone 17": "phablet" })).toThrow(/modelClasses/); + expect(() => parse({ ["x".repeat(257)]: "phone" })).toThrow(/modelClasses/); + for (const deviceClass of ["phone", "tablet", "watch", "tv", "vision", "auto", "desktop"]) { + expect(parse({ "iPhone 17": deviceClass }).modelClasses).toEqual({ + "iPhone 17": deviceClass, + }); + } + }); + + it("catalog.get rejects a platform entry without modelAliases, and takes images as optional", () => { + const entry = { + platform: "android", + models: ["Pixel 8"], + runtimes: ["35"], + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { "Pixel 8": ["35"] }, + }; + expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).toThrow( + /modelAliases/, + ); + const parsed = OPERATIONS["catalog.get"].output.parse({ + platforms: [ + { + ...entry, + modelAliases: { "Pixel 8": ["pixel_8"] }, + images: [{ runtime: "35", tag: "google_apis", abi: "arm64-v8a" }], + }, + ], + }); + expect(parsed.platforms[0]?.images).toEqual([ + { runtime: "35", tag: "google_apis", abi: "arm64-v8a" }, + ]); + }); + + it("catalog.get bounds every string and list in modelAliases, customModels, images and slimCategories", () => { + const base = { + platform: "android", + models: ["Pixel 8"], + runtimes: ["35"], + modelRuntimes: { "Pixel 8": ["35"] }, + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + }; + const image = { runtime: "35", tag: "google_apis", abi: "arm64-v8a" }; + const names = (count: number) => Array.from({ length: count }, (_, index) => `p${index}`); + const aliasedModels = (count: number) => + Object.fromEntries(names(count).map((name) => [name, ["a"]])); + const classedModels = (count: number) => + Object.fromEntries(names(count).map((name) => [name, "phone"])); + const bounds = [ + [ + { modelAliases: { ["x".repeat(256)]: ["a"] } }, + { modelAliases: { ["x".repeat(257)]: ["a"] } }, + ], + [ + { modelAliases: { "Pixel 8": ["x".repeat(256)] } }, + { modelAliases: { "Pixel 8": ["x".repeat(257)] } }, + ], + [{ modelAliases: { "Pixel 8": names(32) } }, { modelAliases: { "Pixel 8": names(33) } }], + [{ modelAliases: aliasedModels(4096) }, { modelAliases: aliasedModels(4097) }], + [ + { modelClasses: { ["x".repeat(256)]: "phone" } }, + { modelClasses: { ["x".repeat(257)]: "phone" } }, + ], + [{ modelClasses: classedModels(4096) }, { modelClasses: classedModels(4097) }], + [{ customModels: ["x".repeat(256)] }, { customModels: ["x".repeat(257)] }], + [{ customModels: names(4096) }, { customModels: names(4097) }], + [{ slimCategories: names(32) }, { slimCategories: names(33) }], + [{ slimCategories: ["a" + "x".repeat(31)] }, { slimCategories: ["a" + "x".repeat(32)] }], + [{ slimCategories: ["a-1"] }, { slimCategories: ["Store"] }], + [{ defaultMode: "slim" }, { defaultMode: "lean" }], + [ + { images: [{ ...image, runtime: "x".repeat(128) }] }, + { images: [{ ...image, runtime: "x".repeat(129) }] }, + ], + [ + { images: [{ ...image, tag: "x".repeat(128) }] }, + { images: [{ ...image, tag: "x".repeat(129) }] }, + ], + [ + { images: [{ ...image, abi: "x".repeat(128) }] }, + { images: [{ ...image, abi: "x".repeat(129) }] }, + ], + [ + { images: Array.from({ length: 1024 }, () => image) }, + { images: Array.from({ length: 1025 }, () => image) }, + ], + ] as const; + for (const [accepted, refused] of bounds) { + const parse = (extra: object) => + OPERATIONS["catalog.get"].output.parse({ platforms: [{ ...base, ...extra }] }); + expect(() => parse(accepted)).not.toThrow(); + expect(() => parse(refused)).toThrow(); + } + }); + + it("worker.remove reports whether there was a view to forget; drain never lies", () => { + // why: `remove` is idempotent: forgetting a worker the gateway has already forgotten is done, + // not an error -- so `false` is a legitimate outcome, the same shape `token.revoke` uses. + expect(OPERATIONS["worker.remove"].output.parse({ workerId: "wrk_1", removed: false })).toEqual( + { workerId: "wrk_1", removed: false }, + ); + // why: Drain and undrain each report the state they establish, and only that state: an + // `undrain` answering `drained: true` would be a contradiction rather than an outcome. + expect(OPERATIONS["worker.drain"].output.parse({ workerId: "wrk_1", drained: true })).toEqual({ + workerId: "wrk_1", + drained: true, + }); + expect(() => + OPERATIONS["worker.drain"].output.parse({ workerId: "wrk_1", drained: false }), + ).toThrow(); + expect(() => + OPERATIONS["worker.undrain"].output.parse({ workerId: "wrk_1", drained: true }), + ).toThrow(); + }); + + it("list.get: accepts each kind's array shape", () => { + expect(OPERATIONS["list.get"].output.parse([])).toEqual([]); + expect(OPERATIONS["list.get"].output.parse([{ name: "idle-shutdown" }])).toEqual([ + { name: "idle-shutdown" }, + ]); + }); + + it("device.exec: parses a command, keeps stdin optional, and refuses anything else", () => { + // why: ADR 0005 §19a. `.strict()` is doing real work here: a client that sent `input` or `env` + // hoping either would reach the child must be told no, not have it silently dropped. + expect( + OPERATIONS["device.exec"].input.parse({ + args: ["shell", "getprop"], + leaseId: "lse_1", + stdin: "y\n", + tool: "adb", + }), + ).toEqual({ args: ["shell", "getprop"], leaseId: "lse_1", stdin: "y\n", tool: "adb" }); + + expect( + OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "simctl" }), + ).toEqual({ args: [], leaseId: "lse_1", tool: "simctl" }); + + // why: `tool` is an open string, exactly as `driver.passthrough`'s is: which wrappers exist is + // the drivers' answer, so a name none of them wraps parses fine here and comes back as + // `UNKNOWN_PASSTHROUGH_TOOL` from the driver catalog rather than as a malformed body. + expect( + OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "bash" }), + ).toEqual({ args: [], leaseId: "lse_1", tool: "bash" }); + expect(() => + OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "" }), + ).toThrow(); + expect(() => + OPERATIONS["device.exec"].input.parse({ + args: [], + leaseId: "lse_1", + tool: "simctl", + env: {}, + }), + ).toThrow(); + expect(() => OPERATIONS["device.exec"].input.parse({ args: [], tool: "simctl" })).toThrow(); + + expect(OPERATIONS["device.exec"].output.parse({ exitCode: 0 })).toEqual({ exitCode: 0 }); + }); + + it("device.rotate: accepts a leaseId and requesterId of 1 to 256 characters and each orientation, and refuses an empty or 257-character one, another orientation or an unknown key", () => { + const parse = (input: object) => OPERATIONS["device.rotate"].input.parse(input); + const base = { leaseId: "lse_1", orientation: "portrait" }; + + for (const orientation of [ + "portrait", + "portrait_upside_down", + "landscape_left", + "landscape_right", + ]) { + expect(parse({ ...base, orientation })).toEqual({ ...base, orientation }); + } + expect(parse({ ...base, leaseId: "x".repeat(256), requesterId: "y".repeat(256) })).toEqual({ + ...base, + leaseId: "x".repeat(256), + requesterId: "y".repeat(256), + }); + + for (const refused of [ + { ...base, leaseId: "" }, + { ...base, leaseId: "x".repeat(257) }, + { ...base, requesterId: "" }, + { ...base, requesterId: "y".repeat(257) }, + { ...base, requesterId: "a\u001b[2Jb" }, + { ...base, orientation: "upside_down" }, + { ...base, angle: 90 }, + ]) { + expect(() => parse(refused)).toThrow(); + } + }); + + it("baseline.remove: accepts a workerId of 1 to 128 characters, and refuses an empty or 129-character one, a non-string and an unknown key", () => { + const parse = (input: object) => OPERATIONS["baseline.remove"].input.parse(input); + const base = { name: "checkout" }; + + expect(parse(base)).toEqual(base); + for (const workerId of ["w", "x".repeat(128)]) { + expect(parse({ ...base, workerId })).toEqual({ ...base, workerId }); + } + for (const refused of [ + { ...base, workerId: "" }, + { ...base, workerId: "x".repeat(129) }, + { ...base, workerId: 7 }, + { ...base, worker: "wrk_a" }, + ]) { + expect(() => parse(refused)).toThrow(); + } + }); + + it("the output push carries a frame id, a stream, and a chunk", () => { + expect(PUSH_SCHEMAS.output.parse({ chunk: "hello", requestId: 7, stream: "stdout" })).toEqual({ + chunk: "hello", + requestId: 7, + stream: "stdout", + }); + expect(() => + PUSH_SCHEMAS.output.parse({ chunk: "hello", requestId: 7, stream: "stdin" }), + ).toThrow(); + expect(() => PUSH_SCHEMAS.output.parse({ chunk: "hello", stream: "stdout" })).toThrow(); + }); + + it("the progress push carries the downloading stage with its component, whether it waits, and an optional whole percent", () => { + const pushes = [ + { component: "26.4", stage: "downloading", waiting: true }, + { component: "26.4", percent: 41, stage: "downloading", waiting: false }, + ]; + for (const progress of pushes) { + expect(PUSH_SCHEMAS.progress.parse({ progress, requestId: 7 })).toEqual({ + progress, + requestId: 7, + }); + } + }); + + it.each([ + ["a percent above 100", { percent: 101 }, { percent: 100 }], + ["a percent below 0", { percent: -1 }, { percent: 0 }], + ["a percent that is not whole", { percent: 41.7 }, { percent: 41 }], + [ + "a component over 64 characters", + { component: "x".repeat(65) }, + { component: "x".repeat(64) }, + ], + ])("refuses a downloading progress push with %s", (_case, outside, inside) => { + const base = { component: "26.4", stage: "downloading", waiting: false }; + const parse = (fields: object) => + PUSH_SCHEMAS.progress.safeParse({ progress: { ...base, ...fields }, requestId: 7 }).success; + expect(parse(outside)).toBe(false); + expect(parse(inside)).toBe(true); + }); + + it("config.get: round-trips a representative config, carries a driver's own keys under ios and android as sent, and refuses a platform section whose core keys are missing or wrong", () => { + const config = { + capabilities: {}, + mode: "worker", + gateway: { + disconnectedRetentionMs: 86_400_000, + execTimeoutMs: 660_000, + leaseRequestTimeoutMs: 300_000, + routing: "warm-then-free", + alerts: { destinations: [] }, + admission: { command: ["bun", "policy.ts"], timeoutMs: 5_000 }, + }, + capacity: { strategy: "fixed", config: { maxRunning: 4 } }, + downloads: { policy: "on-request", acceptAndroidLicenses: false, timeoutMs: 1_000 }, + idle: { shutdownAfterMs: 1, deleteAfterMs: 2 }, + update: { check: true }, + warmPool: { + enabled: true, + maxConcurrentBoots: 1, + reserveRunning: { android: 0, ios: 0 }, + targets: [], + quarantine: { + maxRetries: 1, + retryBackoffMs: 1, + retryBackoffMultiplier: 1, + maxRetryBackoffMs: 1, + }, + }, + devices: { autoEnroll: false, deletedRetentionMs: 1 }, + lease: { + defaultTtlMs: 1, + maxTtlMs: 1, + identity: { ios: "fresh", android: "reusable" }, + requestRetentionMs: 1, + endedIdRetentionMs: 1, + maxRequestRecords: 1, + maxPerRequester: 1, + endWaitMs: 600_000, + }, + exec: { timeoutMs: 1 }, + diskPressure: { freeBytesThreshold: 1 }, + memoryPressure: { + clearAfterMs: 60_000, + enabled: true, + pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, + pauseWarming: { level: "warn", minAvailablePercent: 20 }, + }, + drivers: {}, + eventBuffer: { capacity: 1 }, + log: { level: "info", rotateBytes: 1 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, + health: { + enabled: true, + probeIntervalMs: 1, + stableObservations: 1, + maxRecoveryAttempts: 1, + recoveryBackoffMs: 1, + maxConcurrentRecoveries: 1, + }, + ios: { + defaultMode: "full", + defaultModels: {}, + slim: { bootTimeoutMs: 1 }, + simulator: { audio: false, headless: false }, + aKeyOfALaterDriver: { nested: [1, "two"] }, + }, + android: { + defaultModels: {}, + emulator: { headless: true, gpu: "host", audio: false, bootAnimation: false }, + }, + stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, + }; + const output = OPERATIONS["config.get"].output; + const parsed = output.parse(config); + expect([parsed.ios, parsed.android]).toEqual([config.ios, config.android]); + expect([ + output.safeParse({ ...config, ios: { ...config.ios, defaultMode: "fast" } }).success, + output.safeParse({ ...config, android: { emulator: config.android.emulator } }).success, + ]).toEqual([false, false]); + for (const maxPerRequester of [2, { ios: 2 }, { ios: 1, android: 1 }]) { + const parsed = OPERATIONS["config.get"].output.parse({ + ...config, + lease: { ...config.lease, maxPerRequester }, + }); + expect(parsed.lease.maxPerRequester).toEqual(maxPerRequester); + } + }); + + it("config.get: keeps every warm target key and rejects a target with a platform or a mode the config loader would not accept", () => { + const targets = [ + { + count: 2, + mode: "slim", + model: "iPhone 17", + osVersion: ">=18", + platform: "ios", + }, + { count: 1, mode: "full", model: "Pixel 9", platform: "android" }, + ]; + const warmPool = { targets, maxConcurrentBoots: 3, baselineLearnAfter: 5 }; + const output = OPERATIONS["config.get"].output; + const base = { + capabilities: {}, + mode: "worker", + gateway: { + disconnectedRetentionMs: 1, + execTimeoutMs: 1, + leaseRequestTimeoutMs: 1, + routing: "warm-then-free", + alerts: { destinations: [] }, + admission: { command: ["bun", "policy.ts"], timeoutMs: 5_000 }, + }, + capacity: { strategy: "fixed", config: { maxRunning: 4 } }, + downloads: { policy: "on-request", acceptAndroidLicenses: false, timeoutMs: 1 }, + idle: { shutdownAfterMs: 1, deleteAfterMs: 2 }, + update: { check: true }, + devices: { autoEnroll: false, deletedRetentionMs: 1 }, + lease: { + defaultTtlMs: 1, + maxTtlMs: 1, + identity: { ios: "fresh", android: "reusable" }, + requestRetentionMs: 1, + endedIdRetentionMs: 1, + maxRequestRecords: 1, + maxPerRequester: 1, + endWaitMs: 600_000, + }, + exec: { timeoutMs: 1 }, + drivers: {}, + diskPressure: { freeBytesThreshold: 1 }, + memoryPressure: { + clearAfterMs: 60_000, + enabled: true, + pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, + pauseWarming: { level: "warn", minAvailablePercent: 20 }, + }, + eventBuffer: { capacity: 1 }, + log: { level: "info", rotateBytes: 1 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, + health: { + enabled: true, + probeIntervalMs: 1, + stableObservations: 1, + maxRecoveryAttempts: 1, + recoveryBackoffMs: 1, + maxConcurrentRecoveries: 1, + }, + ios: { defaultMode: "full", defaultModels: {} }, + android: { defaultModels: {} }, + stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, + }; + const config = (pool: object) => ({ + ...base, + warmPool: { + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + quarantine: { + maxRetries: 1, + retryBackoffMs: 1, + retryBackoffMultiplier: 1, + maxRetryBackoffMs: 1, + }, + ...pool, + }, + }); + + expect(output.parse(config(warmPool)).warmPool).toMatchObject(warmPool); + expect(() => + output.parse(config({ ...warmPool, targets: [{ ...targets[0], platform: "tvos" }] })), + ).toThrow(); + expect(() => + output.parse(config({ ...warmPool, targets: [{ ...targets[0], mode: "lean" }] })), + ).toThrow(); + expect(() => output.parse(config({ maxConcurrentBoots: 1 }))).toThrow(); + expect(() => output.parse(config({ targets: [] }))).toThrow(); + }); + + it("config.get: bounds the drivers section's driver names, keys and string values, and takes strings, numbers and booleans", () => { + const drivers = OPERATIONS["config.get"].output.shape.drivers; + const long = (n: number) => "x".repeat(n); + + expect( + drivers.safeParse({ [long(64)]: { [long(64)]: long(4096), n: 1, b: false } }).success, + ).toBe(true); + for (const refused of [ + { [long(65)]: { a: "x" } }, + { a: { [long(65)]: "x" } }, + { a: { k: long(4097) } }, + { a: { k: null } }, + { a: { k: ["x"] } }, + ]) { + expect(drivers.safeParse(refused).success).toBe(false); + } + }); + + it("status.get: rejects a RAM budget whose use is negative or not finite", () => { + const shape = OPERATIONS["status.get"].output.shape.capacity.unwrap().shape.ramBudget; + const budget = { limitBytes: 8, overLimit: false, usedBytes: 1 }; + + expect(shape.parse(budget)).toEqual(budget); + for (const usedBytes of [-1, Number.POSITIVE_INFINITY, Number.NaN]) { + expect(() => shape.parse({ ...budget, usedBytes })).toThrow(); + } + expect(() => shape.parse({ ...budget, limitBytes: Number.POSITIVE_INFINITY })).toThrow(); + }); + + it("config.get: carries each platform's defaultModels as lists of names per class, and refuses anything else", () => { + const shape = OPERATIONS["config.get"].output.shape; + const parse = (defaultModels: unknown) => ({ + ios: shape.ios.shape.defaultModels.safeParse(defaultModels), + android: shape.android.shape.defaultModels.safeParse(defaultModels), + }); + + for (const parsed of Object.values(parse({ phone: ["iPhone 15", "iPhone 14"], tv: ["x"] }))) { + expect(parsed.success).toBe(true); + expect(parsed.data).toEqual({ phone: ["iPhone 15", "iPhone 14"], tv: ["x"] }); + } + for (const refused of [{ phone: [] }, { phone: [""] }, { fridge: ["x"] }, { phone: "x" }]) { + expect(parse(refused).ios.success).toBe(false); + expect(parse(refused).android.success).toBe(false); + } + }); + + it("config.get: keeps both slim RAM sizes of the resource strategy when they are set", () => { + const ramBudget = { + androidBytesPerDevice: 4, + androidSlimBytesPerDevice: 2, + iosBytesPerDevice: 3, + iosSlimBytesPerDevice: 1, + }; + const capacity = { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + ramBudget, + }, + }; + const shape = OPERATIONS["config.get"].output.shape.capacity; + + expect(shape.parse(capacity)).toEqual(capacity); + }); +}); diff --git a/src/contract/operations-round-trips-lease-status.test.ts b/src/contract/operations-round-trips-lease-status.test.ts new file mode 100644 index 000000000..e761df7b3 --- /dev/null +++ b/src/contract/operations-round-trips-lease-status.test.ts @@ -0,0 +1,518 @@ +import { describe, expect, it } from "vitest"; +import { OPERATIONS } from "./index.js"; + +describe("operation input/output round trips", () => { + it("lease.request: round-trips a representative request and rejects legacy aliases", () => { + const input = OPERATIONS["lease.request"].input.parse({ + model: "iPhone 17 Pro", + platform: "ios", + osVersion: "18.0", + }); + expect(input).toMatchObject({ + model: "iPhone 17 Pro", + platform: "ios", + osVersion: "18.0", + }); + + expect(() => + OPERATIONS["lease.request"].input.parse({ device: "iPhone 17 Pro", platform: "ios" }), + ).toThrow(); + expect(() => + OPERATIONS["lease.request"].input.parse({ + request: { model: "iPhone 17 Pro", platform: "ios" }, + }), + ).toThrow(); + }); + + it("lease.request: accepts ttlMs on any request (ADR 0004 §4)", () => { + expect( + OPERATIONS["lease.request"].input.parse({ + model: "iPhone 17 Pro", + platform: "ios", + ttlMs: 60_000, + }), + ).toMatchObject({ ttlMs: 60_000 }); + }); + + it.each(["slim", "full"] as const)("lease.request: accepts mode %s", (mode) => { + expect( + OPERATIONS["lease.request"].input.parse({ model: "iPhone 17 Pro", mode, platform: "ios" }), + ).toMatchObject({ mode }); + }); + + it.each([ + ["full", { full: true }], + ["a mode other than slim or full", { mode: "fast" }], + ["the lease mode ADR 0004 retired", { mode: "held" }], + ])("lease.request: rejects %s", (_label, extra) => { + expect( + OPERATIONS["lease.request"].input.safeParse({ + model: "iPhone 17 Pro", + platform: "ios", + ...extra, + }).success, + ).toBe(false); + }); + + it("lease.request/lease.renew: a non-positive ttlMs is a schema-level BAD_REQUEST", () => { + // why: The upper bound (`lease.maxTtlMs`) is the dispatcher's to enforce -- it is a daemon + // config value this module cannot see -- but "a TTL is a positive number" is shape. + expect(() => + OPERATIONS["lease.request"].input.parse({ + model: "iPhone 17 Pro", + platform: "ios", + ttlMs: 0, + }), + ).toThrow(); + expect(() => OPERATIONS["lease.renew"].input.parse({ leaseId: "lse_1", ttlMs: -1 })).toThrow(); + }); + + it("lease.cancel: keeps a platform of ios or android and refuses any other", () => { + const input = OPERATIONS["lease.cancel"].input; + + expect(input.parse({ platform: "android", requesterId: "agent" })).toEqual({ + platform: "android", + requesterId: "agent", + }); + expect(input.parse({ platform: "ios" })).toEqual({ platform: "ios" }); + expect(input.safeParse({ platform: "tvos" }).success).toBe(false); + }); + + it("lease.request: round-trips a representative output grant", () => { + const grant = { + device: { + physical: false, + id: "dev_1", + driverDeviceId: "sim-1", + spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "18.0" }, + mode: "full", + state: "leased", + driverData: { udid: "abc" }, + createdAt: 1, + }, + environment: {}, + lease: { + id: "lease_1", + deviceId: "dev_1", + requesterId: "req_1", + ownerId: "req_1", + grantedAt: 1, + lastRenewedAt: 1, + idChosenByRequester: false, + ttlMs: 1, + ttlDeadline: 2, + }, + timing: { + estimatedProvisionMs: 1, + estimatedBootMs: 1, + estimatedReclaimMs: 1, + estimatedReadyMs: 1, + }, + }; + expect(OPERATIONS["lease.request"].output.parse(grant)).toBeDefined(); + }); + + it("lease.request: rejects a malformed input (missing platform)", () => { + expect(() => OPERATIONS["lease.request"].input.parse({ model: "iPhone 17" })).toThrow(); + }); + + it("doctor.run: round-trips a driver-advisory finding", () => { + const report = { + findings: [{ kind: "driver-advisory", platform: "ios", code: "slim-disabled", message: "x" }], + sources: [], + }; + expect(OPERATIONS["doctor.run"].output.parse(report)).toBeDefined(); + }); + + it("doctor.run: round-trips a prerequisite-missing finding field for field", () => { + const finding = { + kind: "prerequisite-missing", + platform: "android", + prerequisite: "android-emulator", + message: "The Android emulator package is not installed.", + remedy: "Run `sdkmanager --install emulator`.", + }; + expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ + findings: [finding], + sources: [], + }); + const { remedy: _remedy, ...withoutRemedy } = finding; + expect(() => + OPERATIONS["doctor.run"].output.parse({ findings: [withoutRemedy], sources: [] }), + ).toThrow(); + }); + + it("doctor.run: round-trips a service-broken finding field for field, and refuses an empty missing list or a missing name, remedy or unknown manager", () => { + const finding = { + kind: "service-broken", + manager: "launchd", + name: "com.callstack.simlock.worker", + missing: ["/Users/me/.nvm/versions/node/v22.13.0/bin/node"], + remedy: "Run `simlock daemon install` again.", + }; + expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ + findings: [finding], + sources: [], + }); + expect(() => + OPERATIONS["doctor.run"].output.parse({ + findings: [{ ...finding, missing: [] }], + sources: [], + }), + ).toThrow(); + for (const broken of [{ name: undefined }, { manager: "pm2" }, { remedy: undefined }]) { + expect(() => + OPERATIONS["doctor.run"].output.parse({ + findings: [{ ...finding, ...broken }], + sources: [], + }), + ).toThrow(); + } + }); + + it("status.get: round-trips a representative status snapshot", () => { + const status = { + devices: [], + leases: [], + capacity: { + ios: { + atRamBudget: false, + running: 0, + maxRunning: 1, + reserved: 0, + overLimit: false, + limit: 1, + warm: 0, + used: 0, + }, + android: { + atRamBudget: false, + running: 0, + maxRunning: 1, + reserved: 0, + overLimit: false, + limit: 1, + warm: 0, + used: 0, + }, + global: { running: 0, maxRunning: 2, reserved: 0, overLimit: false, warm: 0 }, + }, + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { + os: "macOS", + osVersion: "15.5", + arch: "arm64", + tools: [{ platform: "ios", name: "xcode", version: "16.4", build: "16F6" }], + }, + queueDepth: 0, + warmPool: { enabled: true, reserveRunning: { android: 0, ios: 1 }, targets: [] }, + }; + expect(OPERATIONS["status.get"].output.parse(status)).toBeDefined(); + }); + + describe("status.get memoryPressure", () => { + const host = { os: "macOS", osVersion: "15.5", arch: "arm64", tools: [] }; + const entry = { + atRamBudget: false, + limit: 1, + maxRunning: 1, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }; + const parse = (memoryPressure: unknown) => + OPERATIONS["status.get"].output.safeParse({ + capacity: { + android: entry, + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: entry, + memoryPressure, + }, + daemon: { + health: "running", + lastUpdateFailure: null, + latestVersion: null, + mode: "worker", + update: null, + version: "0.0.0-test", + }, + devices: [], + host, + leases: [], + queueDepth: 0, + }).success; + const floor = { availablePercent: 8, floorPercent: 10, trigger: "floor" }; + + it("a worker's block is bounded: the percent a whole number from 0 to 100, the floor 0 to 100, the time not negative", () => { + expect(parse({ step: "none" })).toBe(true); + expect(parse({ cause: floor, since: 0, step: "new-devices-paused" })).toBe(true); + expect( + parse({ cause: { ...floor, availablePercent: 100 }, since: 1, step: "warming-paused" }), + ).toBe(true); + for (const cause of [ + { ...floor, availablePercent: 101 }, + { ...floor, availablePercent: -1 }, + { ...floor, availablePercent: 8.5 }, + { ...floor, floorPercent: 101 }, + { ...floor, floorPercent: -1 }, + { ...floor, level: "bogus" }, + { ...floor, trigger: "other" }, + ]) { + expect(parse({ cause, since: 1, step: "warming-paused" })).toBe(false); + } + for (const since of [-1, Number.POSITIVE_INFINITY, Number.NaN]) { + expect(parse({ cause: floor, since, step: "warming-paused" })).toBe(false); + } + expect(parse({ step: "paused" })).toBe(false); + }); + }); + + describe("status.get warmPool", () => { + const host = { os: "macOS", osVersion: "15.5", arch: "arm64", tools: [] }; + const warmPool = { + enabled: true, + reserveRunning: { android: 0, ios: 1 }, + targets: [ + { + booting: 1, + count: 2, + mode: "full", + model: "iPhone 17", + origin: "configured", + osVersion: "26.0", + platform: "ios", + ready: 1, + }, + { + booting: 0, + count: 1, + mode: "full", + model: "iPhone 17", + origin: "learned", + osVersion: "27.0", + platform: "ios", + ready: 0, + short: "runtime-missing", + }, + ], + }; + const capacityEntry = { + atRamBudget: false, + limit: 1, + maxRunning: 1, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }; + const running = { + capacity: { + android: capacityEntry, + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: capacityEntry, + }, + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + devices: [], + host, + leases: [], + queueDepth: 0, + }; + + it("round-trips the block field for field", () => { + expect(OPERATIONS["status.get"].output.parse({ ...running, warmPool }).warmPool).toEqual( + warmPool, + ); + }); + + it("parses a starting daemon's answer without it, and a running one that carries it", () => { + expect( + OPERATIONS["status.get"].output.parse({ ...running, warmPool }).warmPool, + ).toBeDefined(); + expect( + OPERATIONS["status.get"].output.parse({ + daemon: { + health: "starting", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host, + }), + ).toEqual({ + daemon: { + health: "starting", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host, + }); + }); + + it("rejects a short outside the ten reasons, and a target list longer than 320", () => { + const target = warmPool.targets[0]; + expect(() => + OPERATIONS["status.get"].output.parse({ + ...running, + warmPool: { ...warmPool, targets: [{ ...target, short: "tired" }] }, + }), + ).toThrow(); + expect(() => + OPERATIONS["status.get"].output.parse({ + ...running, + warmPool: { ...warmPool, targets: Array.from({ length: 321 }, () => target) }, + }), + ).toThrow(); + }); + }); + + it.each([ + "disabled", + "no-driver", + "runtime-missing", + "unknown-model", + "unresolvable", + "boot-failed", + "device-limit", + "running-limit", + "reserve", + "ram-budget", + ])("status.get: accepts the warm pool reason %s", (short) => { + const target = { + booting: 0, + count: 1, + mode: "full", + model: "iPhone 17", + origin: "configured", + platform: "ios", + ready: 0, + short, + }; + const status = { + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, + warmPool: { enabled: true, reserveRunning: { android: 0, ios: 0 }, targets: [target] }, + }; + + expect(OPERATIONS["status.get"].output.parse(status).warmPool?.targets[0]?.short).toBe(short); + }); + + it.each(["runtime-missing", "unknown-model", "over-limit"])( + "doctor.run: accepts a warm-pool-target-unreachable finding with reason %s", + (reason) => { + const finding = { + kind: "warm-pool-target-unreachable", + message: "m", + reason, + remedy: "r", + target: "t", + }; + + expect( + OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] }).findings, + ).toEqual([finding]); + }, + ); + + it("doctor.run: round-trips a warm-pool-target-unreachable finding field for field", () => { + const finding = { + kind: "warm-pool-target-unreachable", + message: "iOS 27.0 is not installed", + platform: "ios", + reason: "runtime-missing", + remedy: "run simlock component install ios 27.0", + target: "iPhone 17 / 27.0 / full", + }; + expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ + findings: [finding], + sources: [], + }); + }); + + it("status.get: cuts installs to their first 16 and each component to 64 characters, on the status and on a worker view", () => { + const installs = Array.from({ length: 20 }, (_, index) => ({ + component: `${String(index).padStart(2, "0")}${"x".repeat(100)}`, + platform: "ios", + since: index, + state: "waiting", + waiters: 1, + })); + const cut = installs + .slice(0, 16) + .map((install) => ({ ...install, component: install.component.slice(0, 64) })); + const capacityEntry = { + atRamBudget: false, + running: 0, + maxRunning: 1, + reserved: 0, + overLimit: false, + limit: 1, + warm: 0, + used: 0, + }; + + const parsed = OPERATIONS["status.get"].output.parse({ + capacity: { + android: capacityEntry, + global: { running: 0, maxRunning: 1, reserved: 0, overLimit: false, warm: 0 }, + ios: capacityEntry, + }, + daemon: { + health: "running", + mode: "gateway", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + devices: [], + host: { os: "Linux", osVersion: "6.8.0", arch: "x64", tools: [] }, + installs, + leases: [], + queueDepth: 0, + workers: [ + { + catalog: [], + connection: "connected", + devices: [], + drained: false, + id: "wrk_1", + installs, + lastSeenAt: 1, + leases: [], + }, + ], + }); + + expect(parsed.installs).toEqual(cut); + expect(parsed.workers?.[0]?.installs).toEqual(cut); + }); +}); diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 9f0062918..370fb2191 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -1,16 +1,11 @@ import { describe, expect, it } from "vitest"; -import { z } from "zod"; - import { OPERATIONS, type Effect, type OperationName, - requestedClass, requestedDevice, -} from "./operations.js"; -import { PUSH_SCHEMAS } from "./pushes.js"; -import type { Role } from "./roles.js"; -import { platformCatalogSchema } from "./schemas.js"; + type Role, +} from "./index.js"; const ROLE_MATRIX: ReadonlyArray<{ readonly name: OperationName; @@ -481,1663 +476,3 @@ describe("lease.request metadata", () => { expect(notText.error?.issues[0]?.path, "a value that is not text").toEqual(["metadata", "n"]); }); }); - -describe("operation input/output round trips", () => { - it("lease.request: round-trips a representative request and rejects legacy aliases", () => { - const input = OPERATIONS["lease.request"].input.parse({ - model: "iPhone 17 Pro", - platform: "ios", - osVersion: "18.0", - }); - expect(input).toMatchObject({ - model: "iPhone 17 Pro", - platform: "ios", - osVersion: "18.0", - }); - - expect(() => - OPERATIONS["lease.request"].input.parse({ device: "iPhone 17 Pro", platform: "ios" }), - ).toThrow(); - expect(() => - OPERATIONS["lease.request"].input.parse({ - request: { model: "iPhone 17 Pro", platform: "ios" }, - }), - ).toThrow(); - }); - - it("lease.request: accepts ttlMs on any request (ADR 0004 §4)", () => { - expect( - OPERATIONS["lease.request"].input.parse({ - model: "iPhone 17 Pro", - platform: "ios", - ttlMs: 60_000, - }), - ).toMatchObject({ ttlMs: 60_000 }); - }); - - it.each(["slim", "full"] as const)("lease.request: accepts mode %s", (mode) => { - expect( - OPERATIONS["lease.request"].input.parse({ model: "iPhone 17 Pro", mode, platform: "ios" }), - ).toMatchObject({ mode }); - }); - - it.each([ - ["full", { full: true }], - ["a mode other than slim or full", { mode: "fast" }], - ["the lease mode ADR 0004 retired", { mode: "held" }], - ])("lease.request: rejects %s", (_label, extra) => { - expect( - OPERATIONS["lease.request"].input.safeParse({ - model: "iPhone 17 Pro", - platform: "ios", - ...extra, - }).success, - ).toBe(false); - }); - - it("lease.request/lease.renew: a non-positive ttlMs is a schema-level BAD_REQUEST", () => { - // why: The upper bound (`lease.maxTtlMs`) is the dispatcher's to enforce -- it is a daemon - // config value this module cannot see -- but "a TTL is a positive number" is shape. - expect(() => - OPERATIONS["lease.request"].input.parse({ - model: "iPhone 17 Pro", - platform: "ios", - ttlMs: 0, - }), - ).toThrow(); - expect(() => OPERATIONS["lease.renew"].input.parse({ leaseId: "lse_1", ttlMs: -1 })).toThrow(); - }); - - it("lease.cancel: keeps a platform of ios or android and refuses any other", () => { - const input = OPERATIONS["lease.cancel"].input; - - expect(input.parse({ platform: "android", requesterId: "agent" })).toEqual({ - platform: "android", - requesterId: "agent", - }); - expect(input.parse({ platform: "ios" })).toEqual({ platform: "ios" }); - expect(input.safeParse({ platform: "tvos" }).success).toBe(false); - }); - - it("lease.request: round-trips a representative output grant", () => { - const grant = { - device: { - physical: false, - id: "dev_1", - driverDeviceId: "sim-1", - spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "18.0" }, - mode: "full", - state: "leased", - driverData: { udid: "abc" }, - createdAt: 1, - }, - environment: {}, - lease: { - id: "lease_1", - deviceId: "dev_1", - requesterId: "req_1", - ownerId: "req_1", - grantedAt: 1, - lastRenewedAt: 1, - idChosenByRequester: false, - ttlMs: 1, - ttlDeadline: 2, - }, - timing: { - estimatedProvisionMs: 1, - estimatedBootMs: 1, - estimatedReclaimMs: 1, - estimatedReadyMs: 1, - }, - }; - expect(OPERATIONS["lease.request"].output.parse(grant)).toBeDefined(); - }); - - it("lease.request: rejects a malformed input (missing platform)", () => { - expect(() => OPERATIONS["lease.request"].input.parse({ model: "iPhone 17" })).toThrow(); - }); - - it("doctor.run: round-trips a driver-advisory finding", () => { - const report = { - findings: [{ kind: "driver-advisory", platform: "ios", code: "slim-disabled", message: "x" }], - sources: [], - }; - expect(OPERATIONS["doctor.run"].output.parse(report)).toBeDefined(); - }); - - it("doctor.run: round-trips a prerequisite-missing finding field for field", () => { - const finding = { - kind: "prerequisite-missing", - platform: "android", - prerequisite: "android-emulator", - message: "The Android emulator package is not installed.", - remedy: "Run `sdkmanager --install emulator`.", - }; - expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ - findings: [finding], - sources: [], - }); - const { remedy: _remedy, ...withoutRemedy } = finding; - expect(() => - OPERATIONS["doctor.run"].output.parse({ findings: [withoutRemedy], sources: [] }), - ).toThrow(); - }); - - it("doctor.run: round-trips a service-broken finding field for field, and refuses an empty missing list or a missing name, remedy or unknown manager", () => { - const finding = { - kind: "service-broken", - manager: "launchd", - name: "com.callstack.simlock.worker", - missing: ["/Users/me/.nvm/versions/node/v22.13.0/bin/node"], - remedy: "Run `simlock daemon install` again.", - }; - expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ - findings: [finding], - sources: [], - }); - expect(() => - OPERATIONS["doctor.run"].output.parse({ - findings: [{ ...finding, missing: [] }], - sources: [], - }), - ).toThrow(); - for (const broken of [{ name: undefined }, { manager: "pm2" }, { remedy: undefined }]) { - expect(() => - OPERATIONS["doctor.run"].output.parse({ - findings: [{ ...finding, ...broken }], - sources: [], - }), - ).toThrow(); - } - }); - - it("status.get: round-trips a representative status snapshot", () => { - const status = { - devices: [], - leases: [], - capacity: { - ios: { - atRamBudget: false, - running: 0, - maxRunning: 1, - reserved: 0, - overLimit: false, - limit: 1, - warm: 0, - used: 0, - }, - android: { - atRamBudget: false, - running: 0, - maxRunning: 1, - reserved: 0, - overLimit: false, - limit: 1, - warm: 0, - used: 0, - }, - global: { running: 0, maxRunning: 2, reserved: 0, overLimit: false, warm: 0 }, - }, - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { - os: "macOS", - osVersion: "15.5", - arch: "arm64", - tools: [{ platform: "ios", name: "xcode", version: "16.4", build: "16F6" }], - }, - queueDepth: 0, - warmPool: { enabled: true, reserveRunning: { android: 0, ios: 1 }, targets: [] }, - }; - expect(OPERATIONS["status.get"].output.parse(status)).toBeDefined(); - }); - - describe("status.get memoryPressure", () => { - const host = { os: "macOS", osVersion: "15.5", arch: "arm64", tools: [] }; - const entry = { - atRamBudget: false, - limit: 1, - maxRunning: 1, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }; - const parse = (memoryPressure: unknown) => - OPERATIONS["status.get"].output.safeParse({ - capacity: { - android: entry, - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: entry, - memoryPressure, - }, - daemon: { - health: "running", - lastUpdateFailure: null, - latestVersion: null, - mode: "worker", - update: null, - version: "0.0.0-test", - }, - devices: [], - host, - leases: [], - queueDepth: 0, - }).success; - const floor = { availablePercent: 8, floorPercent: 10, trigger: "floor" }; - - it("a worker's block is bounded: the percent a whole number from 0 to 100, the floor 0 to 100, the time not negative", () => { - expect(parse({ step: "none" })).toBe(true); - expect(parse({ cause: floor, since: 0, step: "new-devices-paused" })).toBe(true); - expect( - parse({ cause: { ...floor, availablePercent: 100 }, since: 1, step: "warming-paused" }), - ).toBe(true); - for (const cause of [ - { ...floor, availablePercent: 101 }, - { ...floor, availablePercent: -1 }, - { ...floor, availablePercent: 8.5 }, - { ...floor, floorPercent: 101 }, - { ...floor, floorPercent: -1 }, - { ...floor, level: "bogus" }, - { ...floor, trigger: "other" }, - ]) { - expect(parse({ cause, since: 1, step: "warming-paused" })).toBe(false); - } - for (const since of [-1, Number.POSITIVE_INFINITY, Number.NaN]) { - expect(parse({ cause: floor, since, step: "warming-paused" })).toBe(false); - } - expect(parse({ step: "paused" })).toBe(false); - }); - }); - - describe("status.get warmPool", () => { - const host = { os: "macOS", osVersion: "15.5", arch: "arm64", tools: [] }; - const warmPool = { - enabled: true, - reserveRunning: { android: 0, ios: 1 }, - targets: [ - { - booting: 1, - count: 2, - mode: "full", - model: "iPhone 17", - origin: "configured", - osVersion: "26.0", - platform: "ios", - ready: 1, - }, - { - booting: 0, - count: 1, - mode: "full", - model: "iPhone 17", - origin: "learned", - osVersion: "27.0", - platform: "ios", - ready: 0, - short: "runtime-missing", - }, - ], - }; - const capacityEntry = { - atRamBudget: false, - limit: 1, - maxRunning: 1, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }; - const running = { - capacity: { - android: capacityEntry, - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: capacityEntry, - }, - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - devices: [], - host, - leases: [], - queueDepth: 0, - }; - - it("round-trips the block field for field", () => { - expect(OPERATIONS["status.get"].output.parse({ ...running, warmPool }).warmPool).toEqual( - warmPool, - ); - }); - - it("parses a starting daemon's answer without it, and a running one that carries it", () => { - expect( - OPERATIONS["status.get"].output.parse({ ...running, warmPool }).warmPool, - ).toBeDefined(); - expect( - OPERATIONS["status.get"].output.parse({ - daemon: { - health: "starting", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host, - }), - ).toEqual({ - daemon: { - health: "starting", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host, - }); - }); - - it("rejects a short outside the ten reasons, and a target list longer than 320", () => { - const target = warmPool.targets[0]; - expect(() => - OPERATIONS["status.get"].output.parse({ - ...running, - warmPool: { ...warmPool, targets: [{ ...target, short: "tired" }] }, - }), - ).toThrow(); - expect(() => - OPERATIONS["status.get"].output.parse({ - ...running, - warmPool: { ...warmPool, targets: Array.from({ length: 321 }, () => target) }, - }), - ).toThrow(); - }); - }); - - it.each([ - "disabled", - "no-driver", - "runtime-missing", - "unknown-model", - "unresolvable", - "boot-failed", - "device-limit", - "running-limit", - "reserve", - "ram-budget", - ])("status.get: accepts the warm pool reason %s", (short) => { - const target = { - booting: 0, - count: 1, - mode: "full", - model: "iPhone 17", - origin: "configured", - platform: "ios", - ready: 0, - short, - }; - const status = { - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, - warmPool: { enabled: true, reserveRunning: { android: 0, ios: 0 }, targets: [target] }, - }; - - expect(OPERATIONS["status.get"].output.parse(status).warmPool?.targets[0]?.short).toBe(short); - }); - - it.each(["runtime-missing", "unknown-model", "over-limit"])( - "doctor.run: accepts a warm-pool-target-unreachable finding with reason %s", - (reason) => { - const finding = { - kind: "warm-pool-target-unreachable", - message: "m", - reason, - remedy: "r", - target: "t", - }; - - expect( - OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] }).findings, - ).toEqual([finding]); - }, - ); - - it("doctor.run: round-trips a warm-pool-target-unreachable finding field for field", () => { - const finding = { - kind: "warm-pool-target-unreachable", - message: "iOS 27.0 is not installed", - platform: "ios", - reason: "runtime-missing", - remedy: "run simlock component install ios 27.0", - target: "iPhone 17 / 27.0 / full", - }; - expect(OPERATIONS["doctor.run"].output.parse({ findings: [finding], sources: [] })).toEqual({ - findings: [finding], - sources: [], - }); - }); - - it("status.get: cuts installs to their first 16 and each component to 64 characters, on the status and on a worker view", () => { - const installs = Array.from({ length: 20 }, (_, index) => ({ - component: `${String(index).padStart(2, "0")}${"x".repeat(100)}`, - platform: "ios", - since: index, - state: "waiting", - waiters: 1, - })); - const cut = installs - .slice(0, 16) - .map((install) => ({ ...install, component: install.component.slice(0, 64) })); - const capacityEntry = { - atRamBudget: false, - running: 0, - maxRunning: 1, - reserved: 0, - overLimit: false, - limit: 1, - warm: 0, - used: 0, - }; - - const parsed = OPERATIONS["status.get"].output.parse({ - capacity: { - android: capacityEntry, - global: { running: 0, maxRunning: 1, reserved: 0, overLimit: false, warm: 0 }, - ios: capacityEntry, - }, - daemon: { - health: "running", - mode: "gateway", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - devices: [], - host: { os: "Linux", osVersion: "6.8.0", arch: "x64", tools: [] }, - installs, - leases: [], - queueDepth: 0, - workers: [ - { - catalog: [], - connection: "connected", - devices: [], - drained: false, - id: "wrk_1", - installs, - lastSeenAt: 1, - leases: [], - }, - ], - }); - - expect(parsed.installs).toEqual(cut); - expect(parsed.workers?.[0]?.installs).toEqual(cut); - }); - - it("status.get: round-trips a gateway's aggregate, workers and all (ADR 0005 §20)", () => { - const capacityEntry = { - atRamBudget: false, - running: 1, - maxRunning: 2, - reserved: 0, - overLimit: false, - limit: 2, - warm: 0, - used: 1, - }; - const capacity = { - ios: capacityEntry, - android: capacityEntry, - global: { running: 1, maxRunning: 4, reserved: 0, overLimit: false, warm: 0 }, - }; - const lease = { - id: "lease_1", - deviceId: "dev_1", - requesterId: "agent-1", - ownerId: "agent-1", - grantedAt: 1, - ttlMs: 2, - ttlDeadline: 3, - lastRenewedAt: 1, - idChosenByRequester: false, - workerId: "wrk_1", - }; - const parsed = OPERATIONS["status.get"].output.parse({ - devices: [ - { - physical: false, - id: "dev_1", - spec: { physical: false, platform: "ios", model: "iPhone 17", osVersion: "26.0" }, - mode: "full", - servesDefaultMode: true, - state: "leased", - workerId: "wrk_1", - }, - ], - leases: [lease], - capacity, - daemon: { - health: "running", - mode: "gateway", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { os: "Linux", osVersion: "6.8.0", arch: "x64", tools: [] }, - queueDepth: 0, - workers: [ - { - id: "wrk_1", - label: "mac-mini-1", - connection: "connected", - drained: false, - lastSeenAt: 10, - health: "running", - version: "0.3.0", - capacity, - host: { - os: "macOS", - osVersion: "15.5", - arch: "arm64", - tools: [{ platform: "android", name: "emulator", version: "35.4.9" }], - }, - queueDepth: 0, - leases: [lease], - devices: [], - catalog: [ - { - platform: "ios", - models: ["iPhone 17"], - runtimes: ["26.0"], - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["26.0"] }, - }, - ], - }, - { - id: "wrk_2", - connection: "incompatible", - drained: false, - lastSeenAt: 11, - protocol: { gateway: { min: 4, max: 4 }, worker: { min: 3, max: 3 } }, - leases: [], - devices: [], - catalog: [], - }, - ], - }); - expect(parsed.workers?.[1]?.protocol?.worker).toEqual({ min: 3, max: 3 }); - expect(parsed.workers?.[0]?.host?.tools[0]?.name).toBe("emulator"); - expect(parsed.devices?.[0]?.workerId).toBe("wrk_1"); - expect(parsed.leases?.[0]?.workerId).toBe("wrk_1"); - }); - - it("catalog.get: round-trips a gateway's per-entry worker annotations (ADR 0005 §21)", () => { - const parsed = OPERATIONS["catalog.get"].output.parse({ - platforms: [ - { - platform: "ios", - models: ["iPhone 17"], - runtimes: ["26.0"], - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "iPhone 17": ["26.0"] }, - modelWorkers: { "iPhone 17": ["wrk_1", "wrk_2"] }, - runtimeWorkers: { "26.0": ["wrk_1"] }, - }, - ], - }); - expect(parsed.platforms[0]?.modelWorkers).toEqual({ "iPhone 17": ["wrk_1", "wrk_2"] }); - }); - - it("catalog.get rejects a platform entry without modelRuntimes", () => { - const entry = { - platform: "ios", - models: ["iPhone 17"], - runtimes: ["26.0"], - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - }; - expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).toThrow( - /modelRuntimes/, - ); - expect(() => - OPERATIONS["catalog.get"].output.parse({ - platforms: [{ ...entry, modelRuntimes: { "iPhone 17": ["26.0"] } }], - }), - ).not.toThrow(); - }); - - it("catalog.get rejects an entry without modelClasses and one whose class is not one of the seven", () => { - const entry = { - platform: "ios", - models: ["iPhone 17"], - runtimes: ["26.0"], - classDefaults: {}, - modelAliases: {}, - modelRuntimes: { "iPhone 17": ["26.0"] }, - }; - const parse = (modelClasses?: unknown) => - platformCatalogSchema.parse(modelClasses === undefined ? entry : { ...entry, modelClasses }); - - expect(() => parse()).toThrow(/modelClasses/); - expect(() => parse({ "iPhone 17": "phablet" })).toThrow(/modelClasses/); - expect(() => parse({ ["x".repeat(257)]: "phone" })).toThrow(/modelClasses/); - for (const deviceClass of ["phone", "tablet", "watch", "tv", "vision", "auto", "desktop"]) { - expect(parse({ "iPhone 17": deviceClass }).modelClasses).toEqual({ - "iPhone 17": deviceClass, - }); - } - }); - - it("catalog.get rejects a platform entry without modelAliases, and takes images as optional", () => { - const entry = { - platform: "android", - models: ["Pixel 8"], - runtimes: ["35"], - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { "Pixel 8": ["35"] }, - }; - expect(() => OPERATIONS["catalog.get"].output.parse({ platforms: [entry] })).toThrow( - /modelAliases/, - ); - const parsed = OPERATIONS["catalog.get"].output.parse({ - platforms: [ - { - ...entry, - modelAliases: { "Pixel 8": ["pixel_8"] }, - images: [{ runtime: "35", tag: "google_apis", abi: "arm64-v8a" }], - }, - ], - }); - expect(parsed.platforms[0]?.images).toEqual([ - { runtime: "35", tag: "google_apis", abi: "arm64-v8a" }, - ]); - }); - - it("catalog.get bounds every string and list in modelAliases, customModels, images and slimCategories", () => { - const base = { - platform: "android", - models: ["Pixel 8"], - runtimes: ["35"], - modelRuntimes: { "Pixel 8": ["35"] }, - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - }; - const image = { runtime: "35", tag: "google_apis", abi: "arm64-v8a" }; - const names = (count: number) => Array.from({ length: count }, (_, index) => `p${index}`); - const aliasedModels = (count: number) => - Object.fromEntries(names(count).map((name) => [name, ["a"]])); - const classedModels = (count: number) => - Object.fromEntries(names(count).map((name) => [name, "phone"])); - const bounds = [ - [ - { modelAliases: { ["x".repeat(256)]: ["a"] } }, - { modelAliases: { ["x".repeat(257)]: ["a"] } }, - ], - [ - { modelAliases: { "Pixel 8": ["x".repeat(256)] } }, - { modelAliases: { "Pixel 8": ["x".repeat(257)] } }, - ], - [{ modelAliases: { "Pixel 8": names(32) } }, { modelAliases: { "Pixel 8": names(33) } }], - [{ modelAliases: aliasedModels(4096) }, { modelAliases: aliasedModels(4097) }], - [ - { modelClasses: { ["x".repeat(256)]: "phone" } }, - { modelClasses: { ["x".repeat(257)]: "phone" } }, - ], - [{ modelClasses: classedModels(4096) }, { modelClasses: classedModels(4097) }], - [{ customModels: ["x".repeat(256)] }, { customModels: ["x".repeat(257)] }], - [{ customModels: names(4096) }, { customModels: names(4097) }], - [{ slimCategories: names(32) }, { slimCategories: names(33) }], - [{ slimCategories: ["a" + "x".repeat(31)] }, { slimCategories: ["a" + "x".repeat(32)] }], - [{ slimCategories: ["a-1"] }, { slimCategories: ["Store"] }], - [{ defaultMode: "slim" }, { defaultMode: "lean" }], - [ - { images: [{ ...image, runtime: "x".repeat(128) }] }, - { images: [{ ...image, runtime: "x".repeat(129) }] }, - ], - [ - { images: [{ ...image, tag: "x".repeat(128) }] }, - { images: [{ ...image, tag: "x".repeat(129) }] }, - ], - [ - { images: [{ ...image, abi: "x".repeat(128) }] }, - { images: [{ ...image, abi: "x".repeat(129) }] }, - ], - [ - { images: Array.from({ length: 1024 }, () => image) }, - { images: Array.from({ length: 1025 }, () => image) }, - ], - ] as const; - for (const [accepted, refused] of bounds) { - const parse = (extra: object) => - OPERATIONS["catalog.get"].output.parse({ platforms: [{ ...base, ...extra }] }); - expect(() => parse(accepted)).not.toThrow(); - expect(() => parse(refused)).toThrow(); - } - }); - - it("worker.remove reports whether there was a view to forget; drain never lies", () => { - // why: `remove` is idempotent: forgetting a worker the gateway has already forgotten is done, - // not an error -- so `false` is a legitimate outcome, the same shape `token.revoke` uses. - expect(OPERATIONS["worker.remove"].output.parse({ workerId: "wrk_1", removed: false })).toEqual( - { workerId: "wrk_1", removed: false }, - ); - // why: Drain and undrain each report the state they establish, and only that state: an - // `undrain` answering `drained: true` would be a contradiction rather than an outcome. - expect(OPERATIONS["worker.drain"].output.parse({ workerId: "wrk_1", drained: true })).toEqual({ - workerId: "wrk_1", - drained: true, - }); - expect(() => - OPERATIONS["worker.drain"].output.parse({ workerId: "wrk_1", drained: false }), - ).toThrow(); - expect(() => - OPERATIONS["worker.undrain"].output.parse({ workerId: "wrk_1", drained: true }), - ).toThrow(); - }); - - it("list.get: accepts each kind's array shape", () => { - expect(OPERATIONS["list.get"].output.parse([])).toEqual([]); - expect(OPERATIONS["list.get"].output.parse([{ name: "idle-shutdown" }])).toEqual([ - { name: "idle-shutdown" }, - ]); - }); - - it("device.exec: parses a command, keeps stdin optional, and refuses anything else", () => { - // why: ADR 0005 §19a. `.strict()` is doing real work here: a client that sent `input` or `env` - // hoping either would reach the child must be told no, not have it silently dropped. - expect( - OPERATIONS["device.exec"].input.parse({ - args: ["shell", "getprop"], - leaseId: "lse_1", - stdin: "y\n", - tool: "adb", - }), - ).toEqual({ args: ["shell", "getprop"], leaseId: "lse_1", stdin: "y\n", tool: "adb" }); - - expect( - OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "simctl" }), - ).toEqual({ args: [], leaseId: "lse_1", tool: "simctl" }); - - // why: `tool` is an open string, exactly as `driver.passthrough`'s is: which wrappers exist is - // the drivers' answer, so a name none of them wraps parses fine here and comes back as - // `UNKNOWN_PASSTHROUGH_TOOL` from the driver catalog rather than as a malformed body. - expect( - OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "bash" }), - ).toEqual({ args: [], leaseId: "lse_1", tool: "bash" }); - expect(() => - OPERATIONS["device.exec"].input.parse({ args: [], leaseId: "lse_1", tool: "" }), - ).toThrow(); - expect(() => - OPERATIONS["device.exec"].input.parse({ - args: [], - leaseId: "lse_1", - tool: "simctl", - env: {}, - }), - ).toThrow(); - expect(() => OPERATIONS["device.exec"].input.parse({ args: [], tool: "simctl" })).toThrow(); - - expect(OPERATIONS["device.exec"].output.parse({ exitCode: 0 })).toEqual({ exitCode: 0 }); - }); - - it("device.rotate: accepts a leaseId and requesterId of 1 to 256 characters and each orientation, and refuses an empty or 257-character one, another orientation or an unknown key", () => { - const parse = (input: object) => OPERATIONS["device.rotate"].input.parse(input); - const base = { leaseId: "lse_1", orientation: "portrait" }; - - for (const orientation of [ - "portrait", - "portrait_upside_down", - "landscape_left", - "landscape_right", - ]) { - expect(parse({ ...base, orientation })).toEqual({ ...base, orientation }); - } - expect(parse({ ...base, leaseId: "x".repeat(256), requesterId: "y".repeat(256) })).toEqual({ - ...base, - leaseId: "x".repeat(256), - requesterId: "y".repeat(256), - }); - - for (const refused of [ - { ...base, leaseId: "" }, - { ...base, leaseId: "x".repeat(257) }, - { ...base, requesterId: "" }, - { ...base, requesterId: "y".repeat(257) }, - { ...base, requesterId: "a\u001b[2Jb" }, - { ...base, orientation: "upside_down" }, - { ...base, angle: 90 }, - ]) { - expect(() => parse(refused)).toThrow(); - } - }); - - it("baseline.remove: accepts a workerId of 1 to 128 characters, and refuses an empty or 129-character one, a non-string and an unknown key", () => { - const parse = (input: object) => OPERATIONS["baseline.remove"].input.parse(input); - const base = { name: "checkout" }; - - expect(parse(base)).toEqual(base); - for (const workerId of ["w", "x".repeat(128)]) { - expect(parse({ ...base, workerId })).toEqual({ ...base, workerId }); - } - for (const refused of [ - { ...base, workerId: "" }, - { ...base, workerId: "x".repeat(129) }, - { ...base, workerId: 7 }, - { ...base, worker: "wrk_a" }, - ]) { - expect(() => parse(refused)).toThrow(); - } - }); - - it("the output push carries a frame id, a stream, and a chunk", () => { - expect(PUSH_SCHEMAS.output.parse({ chunk: "hello", requestId: 7, stream: "stdout" })).toEqual({ - chunk: "hello", - requestId: 7, - stream: "stdout", - }); - expect(() => - PUSH_SCHEMAS.output.parse({ chunk: "hello", requestId: 7, stream: "stdin" }), - ).toThrow(); - expect(() => PUSH_SCHEMAS.output.parse({ chunk: "hello", stream: "stdout" })).toThrow(); - }); - - it("the progress push carries the downloading stage with its component, whether it waits, and an optional whole percent", () => { - const pushes = [ - { component: "26.4", stage: "downloading", waiting: true }, - { component: "26.4", percent: 41, stage: "downloading", waiting: false }, - ]; - for (const progress of pushes) { - expect(PUSH_SCHEMAS.progress.parse({ progress, requestId: 7 })).toEqual({ - progress, - requestId: 7, - }); - } - }); - - it.each([ - ["a percent above 100", { percent: 101 }, { percent: 100 }], - ["a percent below 0", { percent: -1 }, { percent: 0 }], - ["a percent that is not whole", { percent: 41.7 }, { percent: 41 }], - [ - "a component over 64 characters", - { component: "x".repeat(65) }, - { component: "x".repeat(64) }, - ], - ])("refuses a downloading progress push with %s", (_case, outside, inside) => { - const base = { component: "26.4", stage: "downloading", waiting: false }; - const parse = (fields: object) => - PUSH_SCHEMAS.progress.safeParse({ progress: { ...base, ...fields }, requestId: 7 }).success; - expect(parse(outside)).toBe(false); - expect(parse(inside)).toBe(true); - }); - - it("config.get: round-trips a representative config, carries a driver's own keys under ios and android as sent, and refuses a platform section whose core keys are missing or wrong", () => { - const config = { - capabilities: {}, - mode: "worker", - gateway: { - disconnectedRetentionMs: 86_400_000, - execTimeoutMs: 660_000, - leaseRequestTimeoutMs: 300_000, - routing: "warm-then-free", - alerts: { destinations: [] }, - admission: { command: ["bun", "policy.ts"], timeoutMs: 5_000 }, - }, - capacity: { strategy: "fixed", config: { maxRunning: 4 } }, - downloads: { policy: "on-request", acceptAndroidLicenses: false, timeoutMs: 1_000 }, - idle: { shutdownAfterMs: 1, deleteAfterMs: 2 }, - update: { check: true }, - warmPool: { - enabled: true, - maxConcurrentBoots: 1, - reserveRunning: { android: 0, ios: 0 }, - targets: [], - quarantine: { - maxRetries: 1, - retryBackoffMs: 1, - retryBackoffMultiplier: 1, - maxRetryBackoffMs: 1, - }, - }, - devices: { autoEnroll: false, deletedRetentionMs: 1 }, - lease: { - defaultTtlMs: 1, - maxTtlMs: 1, - identity: { ios: "fresh", android: "reusable" }, - requestRetentionMs: 1, - endedIdRetentionMs: 1, - maxRequestRecords: 1, - maxPerRequester: 1, - endWaitMs: 600_000, - }, - exec: { timeoutMs: 1 }, - diskPressure: { freeBytesThreshold: 1 }, - memoryPressure: { - clearAfterMs: 60_000, - enabled: true, - pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, - pauseWarming: { level: "warn", minAvailablePercent: 20 }, - }, - drivers: {}, - eventBuffer: { capacity: 1 }, - log: { level: "info", rotateBytes: 1 }, - http: { enabled: false, host: "127.0.0.1", port: 4700 }, - health: { - enabled: true, - probeIntervalMs: 1, - stableObservations: 1, - maxRecoveryAttempts: 1, - recoveryBackoffMs: 1, - maxConcurrentRecoveries: 1, - }, - ios: { - defaultMode: "full", - defaultModels: {}, - slim: { bootTimeoutMs: 1 }, - simulator: { audio: false, headless: false }, - aKeyOfALaterDriver: { nested: [1, "two"] }, - }, - android: { - defaultModels: {}, - emulator: { headless: true, gpu: "host", audio: false, bootAnimation: false }, - }, - stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, - }; - const output = OPERATIONS["config.get"].output; - const parsed = output.parse(config); - expect([parsed.ios, parsed.android]).toEqual([config.ios, config.android]); - expect([ - output.safeParse({ ...config, ios: { ...config.ios, defaultMode: "fast" } }).success, - output.safeParse({ ...config, android: { emulator: config.android.emulator } }).success, - ]).toEqual([false, false]); - for (const maxPerRequester of [2, { ios: 2 }, { ios: 1, android: 1 }]) { - const parsed = OPERATIONS["config.get"].output.parse({ - ...config, - lease: { ...config.lease, maxPerRequester }, - }); - expect(parsed.lease.maxPerRequester).toEqual(maxPerRequester); - } - }); - - it("config.get: keeps every warm target key and rejects a target with a platform or a mode the config loader would not accept", () => { - const targets = [ - { - count: 2, - mode: "slim", - model: "iPhone 17", - osVersion: ">=18", - platform: "ios", - }, - { count: 1, mode: "full", model: "Pixel 9", platform: "android" }, - ]; - const warmPool = { targets, maxConcurrentBoots: 3, baselineLearnAfter: 5 }; - const output = OPERATIONS["config.get"].output; - const base = { - capabilities: {}, - mode: "worker", - gateway: { - disconnectedRetentionMs: 1, - execTimeoutMs: 1, - leaseRequestTimeoutMs: 1, - routing: "warm-then-free", - alerts: { destinations: [] }, - admission: { command: ["bun", "policy.ts"], timeoutMs: 5_000 }, - }, - capacity: { strategy: "fixed", config: { maxRunning: 4 } }, - downloads: { policy: "on-request", acceptAndroidLicenses: false, timeoutMs: 1 }, - idle: { shutdownAfterMs: 1, deleteAfterMs: 2 }, - update: { check: true }, - devices: { autoEnroll: false, deletedRetentionMs: 1 }, - lease: { - defaultTtlMs: 1, - maxTtlMs: 1, - identity: { ios: "fresh", android: "reusable" }, - requestRetentionMs: 1, - endedIdRetentionMs: 1, - maxRequestRecords: 1, - maxPerRequester: 1, - endWaitMs: 600_000, - }, - exec: { timeoutMs: 1 }, - drivers: {}, - diskPressure: { freeBytesThreshold: 1 }, - memoryPressure: { - clearAfterMs: 60_000, - enabled: true, - pauseNewDevices: { level: "critical", minAvailablePercent: 10 }, - pauseWarming: { level: "warn", minAvailablePercent: 20 }, - }, - eventBuffer: { capacity: 1 }, - log: { level: "info", rotateBytes: 1 }, - http: { enabled: false, host: "127.0.0.1", port: 4700 }, - health: { - enabled: true, - probeIntervalMs: 1, - stableObservations: 1, - maxRecoveryAttempts: 1, - recoveryBackoffMs: 1, - maxConcurrentRecoveries: 1, - }, - ios: { defaultMode: "full", defaultModels: {} }, - android: { defaultModels: {} }, - stalledTransition: { thresholdMultiplier: 1, minimumThresholdMs: 1 }, - }; - const config = (pool: object) => ({ - ...base, - warmPool: { - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - quarantine: { - maxRetries: 1, - retryBackoffMs: 1, - retryBackoffMultiplier: 1, - maxRetryBackoffMs: 1, - }, - ...pool, - }, - }); - - expect(output.parse(config(warmPool)).warmPool).toMatchObject(warmPool); - expect(() => - output.parse(config({ ...warmPool, targets: [{ ...targets[0], platform: "tvos" }] })), - ).toThrow(); - expect(() => - output.parse(config({ ...warmPool, targets: [{ ...targets[0], mode: "lean" }] })), - ).toThrow(); - expect(() => output.parse(config({ maxConcurrentBoots: 1 }))).toThrow(); - expect(() => output.parse(config({ targets: [] }))).toThrow(); - }); - - it("config.get: bounds the drivers section's driver names, keys and string values, and takes strings, numbers and booleans", () => { - const drivers = OPERATIONS["config.get"].output.shape.drivers; - const long = (n: number) => "x".repeat(n); - - expect( - drivers.safeParse({ [long(64)]: { [long(64)]: long(4096), n: 1, b: false } }).success, - ).toBe(true); - for (const refused of [ - { [long(65)]: { a: "x" } }, - { a: { [long(65)]: "x" } }, - { a: { k: long(4097) } }, - { a: { k: null } }, - { a: { k: ["x"] } }, - ]) { - expect(drivers.safeParse(refused).success).toBe(false); - } - }); - - it("status.get: rejects a RAM budget whose use is negative or not finite", () => { - const shape = OPERATIONS["status.get"].output.shape.capacity.unwrap().shape.ramBudget; - const budget = { limitBytes: 8, overLimit: false, usedBytes: 1 }; - - expect(shape.parse(budget)).toEqual(budget); - for (const usedBytes of [-1, Number.POSITIVE_INFINITY, Number.NaN]) { - expect(() => shape.parse({ ...budget, usedBytes })).toThrow(); - } - expect(() => shape.parse({ ...budget, limitBytes: Number.POSITIVE_INFINITY })).toThrow(); - }); - - it("config.get: carries each platform's defaultModels as lists of names per class, and refuses anything else", () => { - const shape = OPERATIONS["config.get"].output.shape; - const parse = (defaultModels: unknown) => ({ - ios: shape.ios.shape.defaultModels.safeParse(defaultModels), - android: shape.android.shape.defaultModels.safeParse(defaultModels), - }); - - for (const parsed of Object.values(parse({ phone: ["iPhone 15", "iPhone 14"], tv: ["x"] }))) { - expect(parsed.success).toBe(true); - expect(parsed.data).toEqual({ phone: ["iPhone 15", "iPhone 14"], tv: ["x"] }); - } - for (const refused of [{ phone: [] }, { phone: [""] }, { fridge: ["x"] }, { phone: "x" }]) { - expect(parse(refused).ios.success).toBe(false); - expect(parse(refused).android.success).toBe(false); - } - }); - - it("config.get: keeps both slim RAM sizes of the resource strategy when they are set", () => { - const ramBudget = { - androidBytesPerDevice: 4, - androidSlimBytesPerDevice: 2, - iosBytesPerDevice: 3, - iosSlimBytesPerDevice: 1, - }; - const capacity = { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - ramBudget, - }, - }; - const shape = OPERATIONS["config.get"].output.shape.capacity; - - expect(shape.parse(capacity)).toEqual(capacity); - }); -}); - -describe("lease.request names a model, a class, or nothing (ADR 0015 §1)", () => { - const input = OPERATIONS["lease.request"].input; - - it("refuses a request with both a class and a model, with a message naming the two fields", () => { - const result = input.safeParse({ class: "phone", model: "iPhone 16", platform: "ios" }); - - expect(result.success).toBe(false); - const message = result.success ? "" : result.error.issues.map((issue) => issue.message).join(); - expect(message).toContain("`model`"); - expect(message).toContain("`class`"); - }); - - it("accepts a request with a class only, a model only, or neither", () => { - expect(input.safeParse({ class: "tablet", platform: "ios" }).success).toBe(true); - expect(input.safeParse({ model: "iPhone 16", platform: "ios" }).success).toBe(true); - expect(input.safeParse({ platform: "ios" }).success).toBe(true); - }); - - it("refuses a class that is not one of the seven", () => { - expect(input.safeParse({ class: "phablet", platform: "ios" }).success).toBe(false); - }); - - it("requestedDevice copies the class and leaves out the model a request did not name", () => { - expect(requestedDevice(input.parse({ class: "watch", platform: "ios" }))).toStrictEqual({ - class: "watch", - platform: "ios", - }); - expect(requestedDevice(input.parse({ platform: "android" }))).toStrictEqual({ - platform: "android", - }); - expect(requestedDevice(input.parse({ model: "Pixel 8", platform: "android" }))).toStrictEqual({ - model: "Pixel 8", - platform: "android", - }); - }); - - it("requestedClass answers the class named, phone for neither field, and none for an exact model", () => { - expect(requestedClass({ class: "tv" })).toBe("tv"); - expect(requestedClass({})).toBe("phone"); - expect(requestedClass({ model: "iPhone 16" })).toBeUndefined(); - }); -}); - -describe("lease.request physical (ADR 0022)", () => { - const input = OPERATIONS["lease.request"].input; - - it.each([ - ["mode", { mode: "slim" }], - ["imageTag", { imageTag: "x" }], - ["allowDownload", { allowDownload: true }], - ])("a physical request with %s is BAD_REQUEST naming the field", (field, extra) => { - expect(input.safeParse({ physical: true, platform: "ios" }).success).toBe(true); - - const result = input.safeParse({ ...extra, physical: true, platform: "ios" }); - - expect(result.success).toBe(false); - const message = result.success ? "" : result.error.issues.map((issue) => issue.message).join(); - expect(message).toContain(`\`${field}\``); - }); - - it.each([ - ["mode", { mode: "slim" }], - ["imageTag", { imageTag: "x" }], - ["allowDownload", { allowDownload: true }], - ])("a physical request with %s reports the issue at that field's path", (field, extra) => { - const result = input.safeParse({ ...extra, physical: true, platform: "ios" }); - - expect(result.success ? [] : result.error.issues.map((issue) => issue.path)).toStrictEqual([ - [field], - ]); - }); - - it("a physical request with allowDownload false passes", () => { - const result = input.safeParse({ allowDownload: false, physical: true, platform: "ios" }); - - expect(result.success).toBe(true); - expect(result.success ? (result.data as { physical?: boolean }).physical : undefined).toBe( - true, - ); - }); - - it("requestedDevice carries physical true only when the input set it", () => { - expect(requestedDevice(input.parse({ physical: true, platform: "ios" }))).toStrictEqual({ - physical: true, - platform: "ios", - }); - expect(requestedDevice(input.parse({ physical: false, platform: "ios" }))).toStrictEqual({ - platform: "ios", - }); - expect(requestedDevice(input.parse({ platform: "ios" }))).toStrictEqual({ platform: "ios" }); - }); -}); - -describe("status.get physicalQueueDepth (ADR 0022)", () => { - const host = { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }; - const entry = { - atRamBudget: false, - limit: 1, - maxRunning: 1, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }; - const capacity = { - android: entry, - global: { maxRunning: 1, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: entry, - }; - const output = OPERATIONS["status.get"].output; - - it("a status carries the physical line's depth beside the virtual one", () => { - const parsed = output.parse({ - capacity, - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - devices: [], - host, - leases: [], - physicalQueueDepth: 1, - queueDepth: 2, - }); - - expect(parsed.queueDepth).toBe(2); - expect((parsed as { physicalQueueDepth?: number }).physicalQueueDepth).toBe(1); - }); - - it("a worker's entry in a gateway's status carries it too", () => { - const parsed = output.parse({ - capacity, - daemon: { - health: "running", - mode: "gateway", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - devices: [], - host, - leases: [], - queueDepth: 0, - workers: [ - { - capacity, - catalog: [], - connection: "connected", - devices: [], - drained: false, - health: "running", - host, - id: "wrk_1", - label: "mac-mini-1", - lastSeenAt: 10, - leases: [], - physicalQueueDepth: 1, - queueDepth: 0, - version: "0.3.0", - }, - ], - }); - - expect( - (parsed.workers?.[0] as { physicalQueueDepth?: number } | undefined)?.physicalQueueDepth, - ).toBe(1); - }); -}); - -describe("device.add and device.remove ids (ADR 0022 §2)", () => { - const long = "x".repeat(201); - - it.each([ - ["201 characters", long], - ["a space", "00008110 001A2B3C4D5E"], - ["a tab", "00008110\t001A2B3C4D5E"], - ["a control character", "00008110\u0007001A2B3C4D5E"], - ["a leading dash", "-00008110-001A2B3C4D5E"], - ["nothing", ""], - ])("refuses an id with %s", (_case, id) => { - expect(OPERATIONS["device.add"].input.safeParse({ id, platform: "ios" }).success).toBe(false); - expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(false); - }); - - it("takes an id of exactly 200 characters, and refuses a field it does not declare", () => { - const id = "x".repeat(200); - - expect(OPERATIONS["device.add"].input.safeParse({ id, platform: "ios" }).success).toBe(true); - expect( - OPERATIONS["device.add"].input.safeParse({ id: "A", platform: "ios", worker: "w1" }).success, - ).toBe(false); - expect(OPERATIONS["device.remove"].input.safeParse({ id: "A", yes: true }).success).toBe(false); - }); - - it.each([ - ["a unit separator", "A\u001fB"], - ["a delete", "A\u007fB"], - ["a C1 control", "A\u0080B"], - ["the last C1 control", "A\u009fB"], - ["a no-break space", "A\u00a0B"], - ["a newline", "A\nB"], - ])("refuses an id with %s", (_case, id) => { - expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(false); - }); - - it.each([ - ["a letter beyond ASCII", "caf\u00e9"], - ["a dash inside", "A-B"], - ["a tilde and an inverted exclamation mark", "A~\u00a1B"], - ])("takes an id with %s", (_case, id) => { - expect(OPERATIONS["device.remove"].input.safeParse({ id }).success).toBe(true); - }); - - it("says why an id is refused", () => { - const messages = (id: string) => - OPERATIONS["device.remove"].input - .safeParse({ id }) - .error?.issues.map((issue) => issue.message); - - expect(messages("A B")).toEqual(["must not contain whitespace or control characters"]); - expect(messages("-A")).toEqual(["must not start with -"]); - }); - - it("declares device.add and device.remove with their names, outputs and strict inputs", () => { - const output = { - appCount: 3, - class: "tablet", - driverDeviceId: "U", - id: "dev_1", - model: "iPad", - osVersion: "18", - platform: "ios", - reenrolled: true, - }; - - expect(OPERATIONS["device.add"].name).toBe("device.add"); - expect(OPERATIONS["device.remove"].name).toBe("device.remove"); - expect(OPERATIONS["device.add"].output.parse(output)).toEqual(output); - expect(OPERATIONS["device.add"].output.safeParse({ ...output, class: "watch" }).success).toBe( - false, - ); - expect(OPERATIONS["device.add"].output.safeParse({ ...output, appCount: -1 }).success).toBe( - false, - ); - expect(OPERATIONS["device.remove"].output.parse({ driverDeviceId: "U", id: "U" })).toEqual({ - driverDeviceId: "U", - id: "U", - }); - expect(OPERATIONS["device.add"].input.safeParse({ id: "A" }).success).toBe(false); - expect(OPERATIONS["device.add"].input.safeParse({ id: "A", platform: "windows" }).success).toBe( - false, - ); - expect(OPERATIONS["device.remove"].input.safeParse({}).success).toBe(false); - }); -}); - -describe("lease.request requires", () => { - const base = { model: "iPhone 17", platform: "ios" } as const; - const parse = (extra: Record) => - OPERATIONS["lease.request"].input.safeParse({ ...base, ...extra }); - const names = (count: number) => - Array.from({ length: count }, (_, index) => `cap-${String(index)}`); - - it.each([ - ["one name", ["agent-device"]], - ["an empty list", []], - ["8 names", names(8)], - ])("accepts %s", (_label, requires) => { - expect(parse({ requires }).success).toBe(true); - }); - - it("accepts a request without requires, and one with requires: [] parses to the same device", () => { - expect(parse({}).success).toBe(true); - const input = parse({ requires: [] }); - expect(input.success && requestedDevice(input.data)).toEqual({ - model: "iPhone 17", - platform: "ios", - }); - }); - - it.each([ - ["an upper-case name", ["Agent-Device"]], - ["a duplicate name", ["agent-device", "agent-device"]], - ["9 names", names(9)], - ])("refuses requires with %s with BAD_REQUEST-shaped issues on requires", (_label, requires) => { - const parsed = parse({ requires }); - - expect(parsed.success).toBe(false); - expect( - !parsed.success && parsed.error.issues.every((issue) => issue.path[0] === "requires"), - ).toBe(true); - }); - - it("refuses requires together with idempotencyKey, naming both, where each alone is accepted", () => { - expect(parse({ idempotencyKey: "key-1" }).success).toBe(true); - expect(parse({ requires: ["agent-device"] }).success).toBe(true); - - const both = parse({ idempotencyKey: "key-1", requires: ["agent-device"] }); - - expect(both.success).toBe(false); - expect(!both.success && both.error.issues.map((issue) => issue.message)).toContain( - "requires cannot be combined with idempotencyKey", - ); - }); -}); - -describe("relay.request on a byte channel", () => { - const request = { - hasBody: false, - headers: [], - method: "GET", - relayId: "r-1", - target: "agent-device", - tenant: "gw.g.a", - url: "http://gateway.test/x", - }; - const parse = (input: Record) => - OPERATIONS["relay.request"].input.safeParse(input).success; - - it("accepts a channel of 1 to 32 characters of [A-Za-z0-9-] and refuses an empty, a 33-character and a bad-character one, and a request without one", () => { - expect(parse({ ...request, channel: "a" })).toBe(true); - expect(parse({ ...request, channel: "Ab-9".repeat(8) })).toBe(true); - expect(parse({ ...request, channel: "" })).toBe(false); - expect(parse({ ...request, channel: "a".repeat(33) })).toBe(false); - expect(parse({ ...request, channel: "a b" })).toBe(false); - expect(parse({ ...request, channel: "a_b" })).toBe(false); - expect(parse(request)).toBe(false); - }); - - it("accepts twoWay with GET and no body, and refuses it with another method, with a body, or as false, where the same request without twoWay is accepted", () => { - const channel = { ...request, channel: "a" }; - expect(parse({ ...channel, twoWay: true })).toBe(true); - expect(parse({ ...channel, method: "POST" })).toBe(true); - expect(parse({ ...channel, method: "POST", twoWay: true })).toBe(false); - expect(parse({ ...channel, hasBody: true, method: "PUT" })).toBe(true); - expect(parse({ ...channel, hasBody: true, method: "PUT", twoWay: true })).toBe(false); - expect(parse({ ...channel, twoWay: false })).toBe(false); - }); - - it("OPERATIONS has no relay.body and PUSH_SCHEMAS has no relay-chunk", () => { - expect(Object.keys(OPERATIONS)).not.toContain("relay.body"); - expect(Object.keys(PUSH_SCHEMAS)).not.toContain("relay-chunk"); - expect(Object.keys(PUSH_SCHEMAS)).toContain("relay-head"); - }); -}); - -describe("usage.get input", () => { - const parse = (input: unknown) => OPERATIONS["usage.get"].input.safeParse(input); - const issues = (input: unknown) => { - const result = parse(input); - return result.success ? [] : result.error.issues.map((issue) => issue.message); - }; - const DAY = 24 * 60 * 60 * 1000; - - it("takes epoch milliseconds with from before to, up to ninety days apart, and names the bound it refuses", () => { - expect(parse({ from: 0, to: 1 }).success).toBe(true); - expect(parse({ from: 1_000, to: 1_000 + 90 * DAY }).success).toBe(true); - expect(parse({ from: 1_000, to: 1_001 + 90 * DAY }).success).toBe(false); - expect(parse({ from: 5, to: 5 }).success).toBe(false); - expect(parse({ from: 6, to: 5 }).success).toBe(false); - expect(issues({ from: 6, to: 5 })).toEqual(["from must be before to"]); - expect(issues({ from: 0, to: 91 * DAY })).toEqual([ - "the window must not be longer than 90 days", - ]); - }); - - it("refuses a bound that is not an integer, is negative, is above 8.64e15 or is a string, and takes the last date a Date holds", () => { - expect(parse({ from: 8.64e15 - 1, to: 8.64e15 }).success).toBe(true); - expect(parse({ from: 0.5, to: 10 }).success).toBe(false); - expect(parse({ from: 0, to: 10.5 }).success).toBe(false); - expect(parse({ from: -1, to: 10 }).success).toBe(false); - expect(parse({ from: 8.64e15, to: 8.64e15 + 2 }).success).toBe(false); - expect(parse({ from: "0", to: 10 }).success).toBe(false); - expect(parse({ from: 0 }).success).toBe(false); - }); -}); - -describe("events.replay input", () => { - const parse = (input: unknown) => OPERATIONS["events.replay"].input.safeParse(input); - - it("accepts beforeId and limit in bounds, and refuses limit 0, 501 and 1.5, a beforeId of evt_ or abc, and beforeId or limit with sinceTs; accepts a workerId of 1 to 128 characters with any of them, and refuses an empty one or 129", () => { - expect(parse({ beforeId: "evt_abc", limit: 1 }).success).toBe(true); - expect(parse({ limit: 500 }).success).toBe(true); - expect(parse({ sinceTs: 5 }).success).toBe(true); - for (const workerId of ["w", "w".repeat(128)]) { - expect(parse({ workerId }).success).toBe(true); - expect(parse({ limit: 5, workerId }).success).toBe(true); - expect(parse({ sinceTs: 5, workerId }).success).toBe(true); - } - for (const workerId of ["", "w".repeat(129)]) expect(parse({ workerId }).success).toBe(false); - - for (const limit of [0, 501, 1.5]) expect(parse({ limit }).success).toBe(false); - for (const beforeId of ["evt_", "abc"]) expect(parse({ beforeId }).success).toBe(false); - expect(parse({ beforeId: "evt_abc", sinceTs: 5 }).success).toBe(false); - expect(parse({ limit: 5, sinceTs: 5 }).success).toBe(false); - }); -}); - -describe("daemon.update", () => { - const SCHEMA = "https://json-schema.org/draft/2020-12/schema"; - const exactVersion = { - type: "string", - maxLength: 64, - pattern: "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z.-]+)?$", - }; - const version = { type: "string", maxLength: 64 }; - const stateBase = { - from: version, - to: version, - forced: { type: "boolean" }, - startedAt: { type: "integer", minimum: 0, maximum: 9007199254740991 }, - }; - const state = (stage: string, extra: Record = {}) => ({ - type: "object", - properties: { stage: { type: "string", const: stage }, ...extra, ...stateBase }, - required: ["stage", ...Object.keys(extra), "from", "to", "forced", "startedAt"], - additionalProperties: false, - }); - - it("keeps the input and the answer the shapes every daemon version accepts, because it is a frozen call", () => { - expect(z.toJSONSchema(OPERATIONS["daemon.update"].input, { io: "input" })).toEqual({ - $schema: SCHEMA, - type: "object", - properties: { - version: exactVersion, - force: { type: "boolean" }, - query: { type: "boolean" }, - }, - }); - expect(z.toJSONSchema(OPERATIONS["daemon.update"].output)).toEqual({ - $schema: SCHEMA, - type: "object", - properties: { - started: { type: "boolean" }, - upToDate: { type: "boolean" }, - resolved: { anyOf: [exactVersion, { type: "null" }] }, - version, - health: { type: "string", enum: ["starting", "running", "failed"] }, - update: { - anyOf: [ - { - oneOf: [ - state("waiting", { - leases: { type: "integer", minimum: 0, maximum: 9007199254740991 }, - }), - state("installing"), - state("restarting"), - ], - }, - { type: "null" }, - ], - }, - lastUpdateFailure: { - anyOf: [ - { - type: "object", - properties: { - stage: { type: "string", enum: ["installing", "restarting"] }, - from: version, - to: version, - message: { type: "string", maxLength: 2000 }, - at: { type: "integer", minimum: 0, maximum: 9007199254740991 }, - }, - required: ["stage", "from", "to", "message", "at"], - additionalProperties: false, - }, - { type: "null" }, - ], - }, - latestVersion: { anyOf: [version, { type: "null" }] }, - }, - required: [ - "started", - "upToDate", - "resolved", - "version", - "health", - "update", - "lastUpdateFailure", - "latestVersion", - ], - additionalProperties: false, - }); - }); -}); From c54532b1d203cd57b1b67da1bffab2c0799849d8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:33:01 +0100 Subject: [PATCH 58/83] test(contract): split schemas.test.ts by topic Protocol-bump: none (test-only split) --- src/contract/schemas-fit-and-doctor.test.ts | 250 ++++++++ src/contract/schemas-worker-status.test.ts | 430 +++++++++++++ src/contract/schemas.test.ts | 674 +------------------- 3 files changed, 681 insertions(+), 673 deletions(-) create mode 100644 src/contract/schemas-fit-and-doctor.test.ts create mode 100644 src/contract/schemas-worker-status.test.ts diff --git a/src/contract/schemas-fit-and-doctor.test.ts b/src/contract/schemas-fit-and-doctor.test.ts new file mode 100644 index 000000000..e43158f05 --- /dev/null +++ b/src/contract/schemas-fit-and-doctor.test.ts @@ -0,0 +1,250 @@ +import { describe, expect, it } from "vitest"; +import { + doctorReportSchema, + fitHostFacts, + fitPlatformCatalog, + grantedDeviceSchema, + hostFactsSchema, + leaseRequestRecordSchema, + platformCatalogSchema, +} from "./index.js"; + +describe("fitHostFacts", () => { + it("cuts over-long host strings, leaves out tools that do not fit, and stops at the list's maximum", () => { + const fitted = fitHostFacts({ + arch: "a".repeat(200), + os: "o".repeat(200), + osVersion: "1".repeat(200), + tools: [ + { name: "emulator", platform: "android" as const, version: "9".repeat(129) }, + { name: "cmdline-tools", platform: "android" as const, version: "" }, + ...Array.from({ length: 40 }, (_, index) => ({ + name: `tool-${index}`, + platform: "android" as const, + version: "1", + })), + ], + }); + + expect(fitted.arch).toHaveLength(128); + expect(fitted.os).toHaveLength(128); + expect(fitted.osVersion).toHaveLength(128); + expect(fitted.tools).toHaveLength(32); + expect(fitted.tools[0]?.name).toBe("tool-0"); + expect(() => hostFactsSchema.parse(fitted)).not.toThrow(); + }); + + it("leaves facts that already fit as they are", () => { + const host = { + arch: "arm64", + os: "macOS", + osVersion: "15.5", + tools: [{ build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }], + }; + + expect(fitHostFacts(host)).toEqual(host); + }); +}); + +describe("fitPlatformCatalog", () => { + const entry = (customModels: string[]) => ({ + customModels, + defaultRuntime: "35", + modelAliases: {}, + modelClasses: {}, + classDefaults: {}, + modelRuntimes: Object.fromEntries(customModels.map((model) => [model, ["35"]])), + models: customModels, + platform: "android" as const, + runtimes: ["35"], + }); + + it("stops customModels at its maximum, in an entry the schema accepts", () => { + const names = Array.from({ length: 4097 }, (_, index) => `m${index}`); + + const fitted = fitPlatformCatalog(entry(names)); + + expect(fitted.customModels).toEqual(names.slice(0, 4096)); + expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); + }); + + it("leaves out customModels when no name fits", () => { + const fitted = fitPlatformCatalog(entry(["x".repeat(257)])); + + expect(fitted).not.toHaveProperty("customModels"); + expect(fitted.models).toEqual(["x".repeat(257)]); + }); + + it("drops a modelClasses key too long for the schema and keeps the rest", () => { + const fitted = fitPlatformCatalog({ + ...entry([]), + modelClasses: { ["x".repeat(257)]: "phone" as const, "Pixel 8": "phone" as const }, + }); + + expect(fitted.modelClasses).toEqual({ "Pixel 8": "phone" }); + expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); + }); + + it("drops a classDefaults model name too long for the schema and keeps the rest", () => { + const fitted = fitPlatformCatalog({ + ...entry([]), + classDefaults: { phone: "Pixel 8", tv: "x".repeat(257) }, + }); + + expect(fitted.classDefaults).toEqual({ phone: "Pixel 8" }); + expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); + }); + + it("leaves an entry that already fits as it is", () => { + const fits = entry(["My Tablet"]); + + expect(fitPlatformCatalog(fits)).toEqual(fits); + }); +}); + +describe("leaseRequestRecordSchema", () => { + it("keeps the image tag and baseline a stored request named", () => { + const record = leaseRequestRecordSchema.parse({ + createdAt: 0, + id: "req_1", + ownerId: "agent", + request: { + baseline: "checkout", + imageTag: "google_apis", + model: "Pixel 8", + platform: "android", + }, + requesterId: "agent", + state: "open", + }); + + expect(record.request.imageTag).toBe("google_apis"); + expect(record.request.baseline).toBe("checkout"); + }); + + it.each([ + ["a character outside letters, digits, '_', '.' and '-'", "google apis"], + ["more than 64 characters", "a".repeat(65)], + ])("refuses a stored request or a device spec whose image tag has %s", (_label, imageTag) => { + const record = { + createdAt: 0, + id: "req_1", + ownerId: "agent", + request: { imageTag, model: "Pixel 8", platform: "android" }, + requesterId: "agent", + state: "open", + }; + const device = { + physical: false, + driverDeviceId: "emulator-5554", + id: "dev_1", + mode: "full", + spec: { physical: false, imageTag, model: "Pixel 8", osVersion: "34", platform: "android" }, + }; + + expect(leaseRequestRecordSchema.safeParse(record).success).toBe(false); + expect(grantedDeviceSchema.safeParse(device).success).toBe(false); + }); +}); + +describe("platformCatalogSchema classDefaults", () => { + const entry = { + defaultRuntime: "35", + modelAliases: {}, + modelClasses: {}, + modelRuntimes: {}, + models: [], + platform: "android" as const, + runtimes: ["35"], + }; + + it("requires classDefaults", () => { + expect(platformCatalogSchema.safeParse(entry).success).toBe(false); + expect(platformCatalogSchema.safeParse({ ...entry, classDefaults: {} }).success).toBe(true); + }); + + it("accepts a model name for a class", () => { + const parsed = platformCatalogSchema.parse({ ...entry, classDefaults: { phone: "Pixel 9" } }); + + expect(parsed.classDefaults).toEqual({ phone: "Pixel 9" }); + }); + + it("refuses a key that is not a class", () => { + expect( + platformCatalogSchema.safeParse({ ...entry, classDefaults: { fridge: "Pixel 9" } }).success, + ).toBe(false); + }); + + it("refuses a model name longer than the catalog's name bound", () => { + expect( + platformCatalogSchema.safeParse({ ...entry, classDefaults: { phone: "x".repeat(257) } }) + .success, + ).toBe(false); + }); +}); + +describe("doctorReportSchema's module-unavailable finding", () => { + const finding = { + kind: "module-unavailable", + module: "agent-device", + message: "found 0.20.3, need 0.21.28 or later", + remedy: "install agent-device 0.21.28 or later", + }; + const parses = (candidate: unknown) => + doctorReportSchema.safeParse({ findings: [candidate], sources: [] }).success; + + it("parses a module-unavailable finding, and refuses one without module, without message or without remedy", () => { + expect(doctorReportSchema.parse({ findings: [finding], sources: [] })).toStrictEqual({ + findings: [finding], + sources: [], + }); + + const { module: _module, ...withoutModule } = finding; + const { message: _message, ...withoutMessage } = finding; + const { remedy: _remedy, ...withoutRemedy } = finding; + expect(parses(withoutModule)).toBe(false); + expect(parses(withoutMessage)).toBe(false); + expect(parses(withoutRemedy)).toBe(false); + }); + + it("refuses a module that fails the capability name check", () => { + for (const module of ["", "Agent-Device", "1abc", "has space", `a${"b".repeat(32)}`]) { + expect(parses({ ...finding, module })).toBe(false); + } + expect(parses({ ...finding, module: `a${"b".repeat(31)}` })).toBe(true); + }); + + it("refuses the old agent-device-unavailable kind", () => { + const { module: _module, ...rest } = finding; + expect(parses({ ...rest, kind: "agent-device-unavailable" })).toBe(false); + expect(parses({ ...finding, kind: "agent-device-unavailable" })).toBe(false); + }); +}); + +describe("doctorReportSchema's sources", () => { + const source = { platform: "ios", kind: "seed", model: "iPhone 17", osVersion: "27.0" }; + const parses = (candidate: unknown) => + doctorReportSchema.safeParse({ findings: [], sources: [candidate] }).success; + + it("parses a source with its optional fields, and refuses an unknown kind, a size that is negative or not whole, an over-long staleReason, model or osVersion, and an orphan that is not true", () => { + expect( + doctorReportSchema.parse({ + findings: [], + sources: [{ ...source, imageTag: "api-35", sizeBytes: 0, staleReason: "x".repeat(300) }], + }), + ).toStrictEqual({ + findings: [], + sources: [{ ...source, imageTag: "api-35", sizeBytes: 0, staleReason: "x".repeat(300) }], + }); + expect(parses({ ...source, orphan: true })).toBe(true); + expect(parses({ ...source, kind: "copy" })).toBe(false); + expect(parses({ ...source, sizeBytes: -1 })).toBe(false); + expect(parses({ ...source, sizeBytes: 1.5 })).toBe(false); + expect(parses({ ...source, staleReason: "x".repeat(301) })).toBe(false); + expect(parses({ ...source, model: "x".repeat(201) })).toBe(false); + expect(parses({ ...source, osVersion: "x".repeat(201) })).toBe(false); + expect(parses({ ...source, imageTag: "x".repeat(201) })).toBe(false); + expect(parses({ ...source, orphan: false })).toBe(false); + expect(doctorReportSchema.safeParse({ findings: [] }).success).toBe(false); + }); +}); diff --git a/src/contract/schemas-worker-status.test.ts b/src/contract/schemas-worker-status.test.ts new file mode 100644 index 000000000..6827bebc7 --- /dev/null +++ b/src/contract/schemas-worker-status.test.ts @@ -0,0 +1,430 @@ +import { describe, expect, it } from "vitest"; +import { + statusGet, + capabilityReason, + deviceRecordSchema, + doctorReportSchema, + eventEnvelopeSchema, + fitHostLoad, + hostLoadSchema, + statusWarmPoolSchema, + waitingRequestSchema, + workerViewSchema, +} from "./index.js"; + +describe("a worker status's capabilities list", () => { + const status = (capabilities: unknown) => + statusGet.output.safeParse({ + capabilities, + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "26.0", tools: [] }, + }); + const entry = (overrides: Record = {}) => ({ + name: "agent-device", + state: "unavailable", + reason: "found 0.20.1, need 0.21.28 or later", + ...overrides, + }); + + it("passes with a 32-character name and a 500-character reason", () => { + expect(status([entry()]).success).toBe(true); + expect(status([entry({ name: `a${"b".repeat(31)}`, reason: "r".repeat(500) })]).success).toBe( + true, + ); + }); + + it("passes an available and an off entry that carry no reason", () => { + expect( + status([ + { name: "agent-device", state: "available" }, + { name: "other", state: "off" }, + ]).success, + ).toBe(true); + }); + + const refused: ReadonlyArray = [ + ["17 entries", Array.from({ length: 17 }, (_, i) => entry({ name: `cap-${String(i)}` }))], + ["a name with an upper-case letter", [entry({ name: "Agent-device" })]], + ["a name starting with a digit", [entry({ name: "1agent" })]], + ["a name starting with -", [entry({ name: "-agent" })]], + ["a 33-character name", [entry({ name: `a${"b".repeat(32)}` })]], + ["a duplicate name", [entry(), entry({ reason: "other" })]], + ["a reason on an available entry", [entry({ state: "available" })]], + ["a reason on an off entry", [entry({ state: "off" })]], + ["an unavailable entry with no reason", [{ name: "agent-device", state: "unavailable" }]], + ["an unavailable entry with an empty reason", [entry({ reason: "" })]], + ["a 501-character reason", [entry({ reason: "r".repeat(501) })]], + ["a reason with a newline", [entry({ reason: "a\nb" })]], + ["a reason with an escape", [entry({ reason: "a\u001bb" })]], + ["a reason with a U+2028", [entry({ reason: "a\u2028b" })]], + ]; + + it.each(refused)("fails the schema with %s", (_title, capabilities) => { + expect(status([entry()]).success).toBe(true); + expect(status(capabilities).success).toBe(false); + }); +}); + +describe("capabilityReason", () => { + it("answers a 600-character reason with a newline, an escape, a DEL and a U+2028 as 500 characters with spaces in their place", () => { + const text = `a\nb\u001bc\u007fd\u2028${"x".repeat(600)}`; + + const cleaned = capabilityReason(text); + + expect(cleaned.length).toBe(500); + expect(cleaned.startsWith("a b c d ")).toBe(true); + }); + + it("answers a reason that is only control characters as unknown", () => { + expect(capabilityReason("\n\u001b\u007f\u2028")).toBe("unknown"); + expect(capabilityReason("")).toBe("unknown"); + }); + + it("never splits a surrogate pair when it cuts to 500 units", () => { + const cleaned = capabilityReason(`${"x".repeat(499)}\u{1F600}`); + + expect(cleaned).toBe("x".repeat(499)); + }); +}); + +describe("a status answer's load block", () => { + const load = { + ageMs: 4_120, + cpu: { busyPercent: 34.2, cores: 10, limited: false }, + disk: { freeBytes: 120_689_623_040, totalBytes: 530_870_820_864 }, + memory: { + availableBytes: 10_522_669_875, + limited: false, + pressure: "normal", + swapUsedBytes: 1_288_490_188, + totalBytes: 34_359_738_368, + }, + }; + const answer = (block: unknown) => + statusGet.output.safeParse({ + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "26.0", tools: [] }, + load: block, + }); + + it("round-trips field for field", () => { + const parsed = answer(load); + + expect(parsed.success).toBe(true); + expect(parsed.data?.load).toStrictEqual(load); + }); + + it.each([ + ["a negative disk figure", { ...load, disk: { ...load.disk, freeBytes: -1 } }], + ["a negative age", { ...load, ageMs: -1 }], + ["an ageMs over one day", { ...load, ageMs: 86_400_001 }], + ["a busy % over 100", { ...load, cpu: { ...load.cpu, busyPercent: 100.1 } }], + ["cores of 0", { ...load, cpu: { ...load.cpu, cores: 0 } }], + ["an unknown pressure", { ...load, memory: { ...load.memory, pressure: "severe" } }], + ["a negative swap", { ...load, memory: { ...load.memory, swapUsedBytes: -5 } }], + ])("is refused with %s", (_why, refused) => { + expect(answer(load).success).toBe(true); + + expect(answer(refused).success).toBe(false); + }); + + it("accepts an ageMs of exactly one day, half a core, and parts left out", () => { + expect(answer({ ageMs: 86_400_000 }).success).toBe(true); + expect(answer({ ageMs: 0, cpu: { ...load.cpu, cores: 0.5 } }).success).toBe(true); + }); +}); + +describe("fitHostLoad", () => { + const memory = { + availableBytes: 4_000, + limited: false, + swapUsedBytes: 0, + totalBytes: 16_000, + }; + + it("caps ageMs at one day and drops a part whose figures break the schema, keeping the others", () => { + const fitted = fitHostLoad({ + ageMs: 90_000_000, + cpu: { busyPercent: 150, cores: 4, limited: false }, + disk: { freeBytes: -1, totalBytes: 500 }, + memory, + }); + + expect(fitted).toStrictEqual({ ageMs: 86_400_000, memory }); + expect(() => hostLoadSchema.parse(fitted)).not.toThrow(); + }); + + it("drops a memory part whose figures break the schema, keeping the others", () => { + const disk = { freeBytes: 1, totalBytes: 2 }; + + const fitted = fitHostLoad({ ageMs: 5, disk, memory: { ...memory, totalBytes: -1 } }); + + expect(fitted).toStrictEqual({ ageMs: 5, disk }); + }); + + it("leaves a reading that already fits as it is, and nothing as nothing", () => { + const reading = { + ageMs: 15, + cpu: { busyPercent: 12.5, cores: 0.5, limited: true }, + disk: { freeBytes: 1, totalBytes: 2 }, + memory, + }; + + expect(fitHostLoad(reading)).toStrictEqual(reading); + expect(fitHostLoad(undefined)).toBeUndefined(); + }); +}); + +describe("the worker view's load", () => { + const view = { + connection: "connected" as const, + drained: false, + id: "wrk_1", + lastSeenAt: 1, + }; + const load = { + cpu: { busyPercent: 34.2, cores: 10, limited: false }, + disk: { freeBytes: 120, totalBytes: 500 }, + memory: { + availableBytes: 4_000, + limited: false, + pressure: "normal" as const, + swapUsedBytes: 100, + totalBytes: 16_000, + }, + readAt: 8_000, + }; + + it("the worker view schema round-trips load, and refuses a negative readAt, a negative figure or a busy % over 100", () => { + expect(workerViewSchema.parse({ ...view, load })).toEqual({ ...view, load }); + expect(workerViewSchema.parse({ ...view, load: { readAt: 0 } })).toEqual({ + ...view, + load: { readAt: 0 }, + }); + + expect(() => workerViewSchema.parse({ ...view, load: { ...load, readAt: -1 } })).toThrow(); + expect(() => + workerViewSchema.parse({ ...view, load: { ...load, disk: { ...load.disk, freeBytes: -1 } } }), + ).toThrow(); + expect(() => + workerViewSchema.parse({ + ...view, + load: { ...load, cpu: { ...load.cpu, busyPercent: 100.5 } }, + }), + ).toThrow(); + }); +}); + +describe("the worker view's baselines", () => { + const view = { + connection: "connected" as const, + drained: false, + id: "wrk_1", + lastSeenAt: 1, + }; + const entry = { + mode: "full" as const, + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android" as const, + savedAt: "2026-11-02T13:10:00Z", + version: 2, + }; + + it("the worker view schema takes 1000 baselines and refuses 1001, and an entry's workerId of 1 to 128 characters and refuses an empty or 129-character one", () => { + const parse = (baselines: unknown[]) => workerViewSchema.parse({ ...view, baselines }); + + expect(parse([])).toEqual({ ...view, baselines: [] }); + expect(parse(Array.from({ length: 1000 }, () => entry))).toHaveProperty("baselines.999"); + expect(() => parse(Array.from({ length: 1001 }, () => entry))).toThrow(); + expect(parse([{ ...entry, workerId: "x".repeat(128) }])).toHaveProperty( + "baselines.0.workerId", + "x".repeat(128), + ); + expect(() => parse([{ ...entry, workerId: "" }])).toThrow(); + expect(() => parse([{ ...entry, workerId: "x".repeat(129) }])).toThrow(); + }); +}); + +describe("waitingRequestSchema's reason", () => { + const reasons = [ + "behind-earlier-request", + "device-limit", + "ram-budget", + "global-running-limit", + "platform-running-limit", + "waiting-for-device", + "no-free-physical", + ] as const; + const queued = { + createdAt: 1, + id: "req_1", + queuePosition: 1, + requesterId: "agent", + spec: { model: "iPhone 16", platform: "ios" }, + stage: "queued", + } as const; + + it("accepts each of the eight reasons on a queued entry and keeps it", () => { + expect(waitingRequestSchema.safeParse(queued).success).toBe(true); + for (const reason of reasons) { + expect(waitingRequestSchema.parse({ ...queued, reason })).toMatchObject({ reason }); + } + }); + + it("refuses a reason outside the eight on a queued entry", () => { + expect(waitingRequestSchema.safeParse({ ...queued, reason: "bogus" }).success).toBe(false); + }); + + it("refuses any reason on a starting entry", () => { + const { queuePosition: _position, ...rest } = queued; + const starting = { ...rest, stage: "starting" } as const; + + expect(waitingRequestSchema.safeParse(starting).success).toBe(true); + for (const reason of reasons) { + expect(waitingRequestSchema.safeParse({ ...starting, reason }).success).toBe(false); + } + }); +}); + +describe("waitingRequestSchema's workers", () => { + const reasons = [ + "disconnected", + "incompatible", + "not-ready", + "unhealthy", + "drained", + "busy", + "no-driver", + "unknown-model", + "runtime-missing", + "capability-missing", + "full", + "no-free-physical", + "not-chosen", + ] as const; + const queued = { + createdAt: 1, + id: "req_1", + queuePosition: 1, + requesterId: "agent", + spec: { model: "iPhone 16", platform: "ios" }, + stage: "queued", + } as const; + const withWorkers = (workers: unknown) => waitingRequestSchema.safeParse({ ...queued, workers }); + + it("accepts each of the thirteen reasons and a capacity reason with full, and keeps them", () => { + for (const reason of reasons) { + expect(withWorkers([{ reason, workerId: "wrk_a" }]).success).toBe(true); + } + expect( + waitingRequestSchema.parse({ + ...queued, + workers: [{ capacityReason: "ram-budget", reason: "full", workerId: "wrk_a" }], + }), + ).toMatchObject({ workers: [{ capacityReason: "ram-budget", reason: "full" }] }); + }); + + it("refuses an unknown reason, a capacity reason with drained, an unknown capacity reason and one workerId twice", () => { + expect(withWorkers([{ reason: "full", workerId: "wrk_a" }]).success).toBe(true); + expect(withWorkers([{ reason: "bogus", workerId: "wrk_a" }]).success).toBe(false); + expect( + withWorkers([{ capacityReason: "ram-budget", reason: "drained", workerId: "wrk_a" }]).success, + ).toBe(false); + expect( + withWorkers([{ capacityReason: "bogus", reason: "full", workerId: "wrk_a" }]).success, + ).toBe(false); + expect( + withWorkers([ + { reason: "drained", workerId: "wrk_a" }, + { reason: "full", workerId: "wrk_a" }, + ]).success, + ).toBe(false); + }); +}); + +describe("opaque fields stay optional on the wire", () => { + it("accepts an answer whose opaque driverData or payload key is absent (JSON drops undefined)", () => { + const record = { + physical: false, + id: "device-1", + driverDeviceId: "SIM-1", + spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, + mode: "full", + state: "ready", + createdAt: 0, + }; + const orphan = { + findings: [ + { kind: "orphan-device", platform: "ios", device: { deviceId: "SIM-1", address: "a" } }, + ], + sources: [], + }; + const envelope = { id: "evt_1", seq: 1, timestamp: 1, event: "x", module: "m" }; + expect(deviceRecordSchema.safeParse(record).success).toBe(true); + expect(doctorReportSchema.safeParse(orphan).success).toBe(true); + expect(eventEnvelopeSchema.safeParse(envelope).success).toBe(true); + }); +}); + +describe("statusWarmPoolSchema", () => { + const entry = { + booting: 0, + count: 1, + mode: "full", + model: "Pixel 8", + origin: "learned", + osVersion: "35", + platform: "android", + ready: 1, + }; + const block = (targets: readonly unknown[]) => ({ + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + targets, + }); + const { origin: _origin, ...withoutOrigin } = entry; + + it("accepts origin configured and learned, an optional imageTag, an optional baseline, and 320 entries, and refuses an entry without origin or with another word, a tag imageTagSchema refuses, a baseline name the name rule refuses, and 321 entries", () => { + const cases: readonly { name: string; targets: readonly unknown[]; accepted: boolean }[] = [ + { accepted: true, name: "learned", targets: [entry] }, + { accepted: true, name: "configured", targets: [{ ...entry, origin: "configured" }] }, + { accepted: true, name: "a tag", targets: [{ ...entry, imageTag: "google_apis_playstore" }] }, + { accepted: true, name: "320 entries", targets: Array.from({ length: 320 }, () => entry) }, + { accepted: true, name: "a baseline", targets: [{ ...entry, baseline: "checkout" }] }, + { accepted: false, name: "no origin", targets: [withoutOrigin] }, + { accepted: false, name: "another word", targets: [{ ...entry, origin: "remembered" }] }, + { accepted: false, name: "a bad tag", targets: [{ ...entry, imageTag: "no spaces" }] }, + { accepted: false, name: "a bad baseline", targets: [{ ...entry, baseline: "Not A Name" }] }, + { accepted: false, name: "321 entries", targets: Array.from({ length: 321 }, () => entry) }, + ]; + + expect( + cases.map(({ name, targets }) => [ + name, + statusWarmPoolSchema.safeParse(block(targets)).success, + ]), + ).toEqual(cases.map(({ accepted, name }) => [name, accepted])); + expect( + statusWarmPoolSchema.parse(block([{ ...entry, imageTag: "google_apis" }])).targets[0], + ).toMatchObject({ imageTag: "google_apis", origin: "learned" }); + expect( + statusWarmPoolSchema.parse(block([{ ...entry, baseline: "checkout" }])).targets[0], + ).toMatchObject({ baseline: "checkout", origin: "learned" }); + }); +}); diff --git a/src/contract/schemas.test.ts b/src/contract/schemas.test.ts index 61da6367f..b2dc956a7 100644 --- a/src/contract/schemas.test.ts +++ b/src/contract/schemas.test.ts @@ -1,25 +1,10 @@ import { describe, expect, it } from "vitest"; - -import { statusGet } from "./operations.js"; import { - capabilityReason, deviceRecordSchema, - doctorReportSchema, - eventEnvelopeSchema, - fitHostFacts, - fitHostLoad, - fitPlatformCatalog, grantedDeviceSchema, - hostFactsSchema, - hostLoadSchema, leaseGrantSchema, - leaseRequestRecordSchema, - platformCatalogSchema, - statusWarmPoolSchema, statusDeviceSchema, - waitingRequestSchema, - workerViewSchema, -} from "./schemas.js"; +} from "./index.js"; /** * Regression coverage for the defect fixed alongside ADR 0003 §1: a lease grant's device must @@ -524,660 +509,3 @@ describe("a device's type on the wire (ADR 0022 §1)", () => { expect(() => grantedDeviceSchema.parse({ ...virtual, class: "phone" })).toThrow(); }); }); - -describe("fitHostFacts", () => { - it("cuts over-long host strings, leaves out tools that do not fit, and stops at the list's maximum", () => { - const fitted = fitHostFacts({ - arch: "a".repeat(200), - os: "o".repeat(200), - osVersion: "1".repeat(200), - tools: [ - { name: "emulator", platform: "android" as const, version: "9".repeat(129) }, - { name: "cmdline-tools", platform: "android" as const, version: "" }, - ...Array.from({ length: 40 }, (_, index) => ({ - name: `tool-${index}`, - platform: "android" as const, - version: "1", - })), - ], - }); - - expect(fitted.arch).toHaveLength(128); - expect(fitted.os).toHaveLength(128); - expect(fitted.osVersion).toHaveLength(128); - expect(fitted.tools).toHaveLength(32); - expect(fitted.tools[0]?.name).toBe("tool-0"); - expect(() => hostFactsSchema.parse(fitted)).not.toThrow(); - }); - - it("leaves facts that already fit as they are", () => { - const host = { - arch: "arm64", - os: "macOS", - osVersion: "15.5", - tools: [{ build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }], - }; - - expect(fitHostFacts(host)).toEqual(host); - }); -}); - -describe("fitPlatformCatalog", () => { - const entry = (customModels: string[]) => ({ - customModels, - defaultRuntime: "35", - modelAliases: {}, - modelClasses: {}, - classDefaults: {}, - modelRuntimes: Object.fromEntries(customModels.map((model) => [model, ["35"]])), - models: customModels, - platform: "android" as const, - runtimes: ["35"], - }); - - it("stops customModels at its maximum, in an entry the schema accepts", () => { - const names = Array.from({ length: 4097 }, (_, index) => `m${index}`); - - const fitted = fitPlatformCatalog(entry(names)); - - expect(fitted.customModels).toEqual(names.slice(0, 4096)); - expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); - }); - - it("leaves out customModels when no name fits", () => { - const fitted = fitPlatformCatalog(entry(["x".repeat(257)])); - - expect(fitted).not.toHaveProperty("customModels"); - expect(fitted.models).toEqual(["x".repeat(257)]); - }); - - it("drops a modelClasses key too long for the schema and keeps the rest", () => { - const fitted = fitPlatformCatalog({ - ...entry([]), - modelClasses: { ["x".repeat(257)]: "phone" as const, "Pixel 8": "phone" as const }, - }); - - expect(fitted.modelClasses).toEqual({ "Pixel 8": "phone" }); - expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); - }); - - it("drops a classDefaults model name too long for the schema and keeps the rest", () => { - const fitted = fitPlatformCatalog({ - ...entry([]), - classDefaults: { phone: "Pixel 8", tv: "x".repeat(257) }, - }); - - expect(fitted.classDefaults).toEqual({ phone: "Pixel 8" }); - expect(() => platformCatalogSchema.parse(fitted)).not.toThrow(); - }); - - it("leaves an entry that already fits as it is", () => { - const fits = entry(["My Tablet"]); - - expect(fitPlatformCatalog(fits)).toEqual(fits); - }); -}); - -describe("leaseRequestRecordSchema", () => { - it("keeps the image tag and baseline a stored request named", () => { - const record = leaseRequestRecordSchema.parse({ - createdAt: 0, - id: "req_1", - ownerId: "agent", - request: { - baseline: "checkout", - imageTag: "google_apis", - model: "Pixel 8", - platform: "android", - }, - requesterId: "agent", - state: "open", - }); - - expect(record.request.imageTag).toBe("google_apis"); - expect(record.request.baseline).toBe("checkout"); - }); - - it.each([ - ["a character outside letters, digits, '_', '.' and '-'", "google apis"], - ["more than 64 characters", "a".repeat(65)], - ])("refuses a stored request or a device spec whose image tag has %s", (_label, imageTag) => { - const record = { - createdAt: 0, - id: "req_1", - ownerId: "agent", - request: { imageTag, model: "Pixel 8", platform: "android" }, - requesterId: "agent", - state: "open", - }; - const device = { - physical: false, - driverDeviceId: "emulator-5554", - id: "dev_1", - mode: "full", - spec: { physical: false, imageTag, model: "Pixel 8", osVersion: "34", platform: "android" }, - }; - - expect(leaseRequestRecordSchema.safeParse(record).success).toBe(false); - expect(grantedDeviceSchema.safeParse(device).success).toBe(false); - }); -}); - -describe("platformCatalogSchema classDefaults", () => { - const entry = { - defaultRuntime: "35", - modelAliases: {}, - modelClasses: {}, - modelRuntimes: {}, - models: [], - platform: "android" as const, - runtimes: ["35"], - }; - - it("requires classDefaults", () => { - expect(platformCatalogSchema.safeParse(entry).success).toBe(false); - expect(platformCatalogSchema.safeParse({ ...entry, classDefaults: {} }).success).toBe(true); - }); - - it("accepts a model name for a class", () => { - const parsed = platformCatalogSchema.parse({ ...entry, classDefaults: { phone: "Pixel 9" } }); - - expect(parsed.classDefaults).toEqual({ phone: "Pixel 9" }); - }); - - it("refuses a key that is not a class", () => { - expect( - platformCatalogSchema.safeParse({ ...entry, classDefaults: { fridge: "Pixel 9" } }).success, - ).toBe(false); - }); - - it("refuses a model name longer than the catalog's name bound", () => { - expect( - platformCatalogSchema.safeParse({ ...entry, classDefaults: { phone: "x".repeat(257) } }) - .success, - ).toBe(false); - }); -}); - -describe("doctorReportSchema's module-unavailable finding", () => { - const finding = { - kind: "module-unavailable", - module: "agent-device", - message: "found 0.20.3, need 0.21.28 or later", - remedy: "install agent-device 0.21.28 or later", - }; - const parses = (candidate: unknown) => - doctorReportSchema.safeParse({ findings: [candidate], sources: [] }).success; - - it("parses a module-unavailable finding, and refuses one without module, without message or without remedy", () => { - expect(doctorReportSchema.parse({ findings: [finding], sources: [] })).toStrictEqual({ - findings: [finding], - sources: [], - }); - - const { module: _module, ...withoutModule } = finding; - const { message: _message, ...withoutMessage } = finding; - const { remedy: _remedy, ...withoutRemedy } = finding; - expect(parses(withoutModule)).toBe(false); - expect(parses(withoutMessage)).toBe(false); - expect(parses(withoutRemedy)).toBe(false); - }); - - it("refuses a module that fails the capability name check", () => { - for (const module of ["", "Agent-Device", "1abc", "has space", `a${"b".repeat(32)}`]) { - expect(parses({ ...finding, module })).toBe(false); - } - expect(parses({ ...finding, module: `a${"b".repeat(31)}` })).toBe(true); - }); - - it("refuses the old agent-device-unavailable kind", () => { - const { module: _module, ...rest } = finding; - expect(parses({ ...rest, kind: "agent-device-unavailable" })).toBe(false); - expect(parses({ ...finding, kind: "agent-device-unavailable" })).toBe(false); - }); -}); - -describe("doctorReportSchema's sources", () => { - const source = { platform: "ios", kind: "seed", model: "iPhone 17", osVersion: "27.0" }; - const parses = (candidate: unknown) => - doctorReportSchema.safeParse({ findings: [], sources: [candidate] }).success; - - it("parses a source with its optional fields, and refuses an unknown kind, a size that is negative or not whole, an over-long staleReason, model or osVersion, and an orphan that is not true", () => { - expect( - doctorReportSchema.parse({ - findings: [], - sources: [{ ...source, imageTag: "api-35", sizeBytes: 0, staleReason: "x".repeat(300) }], - }), - ).toStrictEqual({ - findings: [], - sources: [{ ...source, imageTag: "api-35", sizeBytes: 0, staleReason: "x".repeat(300) }], - }); - expect(parses({ ...source, orphan: true })).toBe(true); - expect(parses({ ...source, kind: "copy" })).toBe(false); - expect(parses({ ...source, sizeBytes: -1 })).toBe(false); - expect(parses({ ...source, sizeBytes: 1.5 })).toBe(false); - expect(parses({ ...source, staleReason: "x".repeat(301) })).toBe(false); - expect(parses({ ...source, model: "x".repeat(201) })).toBe(false); - expect(parses({ ...source, osVersion: "x".repeat(201) })).toBe(false); - expect(parses({ ...source, imageTag: "x".repeat(201) })).toBe(false); - expect(parses({ ...source, orphan: false })).toBe(false); - expect(doctorReportSchema.safeParse({ findings: [] }).success).toBe(false); - }); -}); - -describe("a worker status's capabilities list", () => { - const status = (capabilities: unknown) => - statusGet.output.safeParse({ - capabilities, - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "26.0", tools: [] }, - }); - const entry = (overrides: Record = {}) => ({ - name: "agent-device", - state: "unavailable", - reason: "found 0.20.1, need 0.21.28 or later", - ...overrides, - }); - - it("passes with a 32-character name and a 500-character reason", () => { - expect(status([entry()]).success).toBe(true); - expect(status([entry({ name: `a${"b".repeat(31)}`, reason: "r".repeat(500) })]).success).toBe( - true, - ); - }); - - it("passes an available and an off entry that carry no reason", () => { - expect( - status([ - { name: "agent-device", state: "available" }, - { name: "other", state: "off" }, - ]).success, - ).toBe(true); - }); - - const refused: ReadonlyArray = [ - ["17 entries", Array.from({ length: 17 }, (_, i) => entry({ name: `cap-${String(i)}` }))], - ["a name with an upper-case letter", [entry({ name: "Agent-device" })]], - ["a name starting with a digit", [entry({ name: "1agent" })]], - ["a name starting with -", [entry({ name: "-agent" })]], - ["a 33-character name", [entry({ name: `a${"b".repeat(32)}` })]], - ["a duplicate name", [entry(), entry({ reason: "other" })]], - ["a reason on an available entry", [entry({ state: "available" })]], - ["a reason on an off entry", [entry({ state: "off" })]], - ["an unavailable entry with no reason", [{ name: "agent-device", state: "unavailable" }]], - ["an unavailable entry with an empty reason", [entry({ reason: "" })]], - ["a 501-character reason", [entry({ reason: "r".repeat(501) })]], - ["a reason with a newline", [entry({ reason: "a\nb" })]], - ["a reason with an escape", [entry({ reason: "a\u001bb" })]], - ["a reason with a U+2028", [entry({ reason: "a\u2028b" })]], - ]; - - it.each(refused)("fails the schema with %s", (_title, capabilities) => { - expect(status([entry()]).success).toBe(true); - expect(status(capabilities).success).toBe(false); - }); -}); - -describe("capabilityReason", () => { - it("answers a 600-character reason with a newline, an escape, a DEL and a U+2028 as 500 characters with spaces in their place", () => { - const text = `a\nb\u001bc\u007fd\u2028${"x".repeat(600)}`; - - const cleaned = capabilityReason(text); - - expect(cleaned.length).toBe(500); - expect(cleaned.startsWith("a b c d ")).toBe(true); - }); - - it("answers a reason that is only control characters as unknown", () => { - expect(capabilityReason("\n\u001b\u007f\u2028")).toBe("unknown"); - expect(capabilityReason("")).toBe("unknown"); - }); - - it("never splits a surrogate pair when it cuts to 500 units", () => { - const cleaned = capabilityReason(`${"x".repeat(499)}\u{1F600}`); - - expect(cleaned).toBe("x".repeat(499)); - }); -}); - -describe("a status answer's load block", () => { - const load = { - ageMs: 4_120, - cpu: { busyPercent: 34.2, cores: 10, limited: false }, - disk: { freeBytes: 120_689_623_040, totalBytes: 530_870_820_864 }, - memory: { - availableBytes: 10_522_669_875, - limited: false, - pressure: "normal", - swapUsedBytes: 1_288_490_188, - totalBytes: 34_359_738_368, - }, - }; - const answer = (block: unknown) => - statusGet.output.safeParse({ - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "26.0", tools: [] }, - load: block, - }); - - it("round-trips field for field", () => { - const parsed = answer(load); - - expect(parsed.success).toBe(true); - expect(parsed.data?.load).toStrictEqual(load); - }); - - it.each([ - ["a negative disk figure", { ...load, disk: { ...load.disk, freeBytes: -1 } }], - ["a negative age", { ...load, ageMs: -1 }], - ["an ageMs over one day", { ...load, ageMs: 86_400_001 }], - ["a busy % over 100", { ...load, cpu: { ...load.cpu, busyPercent: 100.1 } }], - ["cores of 0", { ...load, cpu: { ...load.cpu, cores: 0 } }], - ["an unknown pressure", { ...load, memory: { ...load.memory, pressure: "severe" } }], - ["a negative swap", { ...load, memory: { ...load.memory, swapUsedBytes: -5 } }], - ])("is refused with %s", (_why, refused) => { - expect(answer(load).success).toBe(true); - - expect(answer(refused).success).toBe(false); - }); - - it("accepts an ageMs of exactly one day, half a core, and parts left out", () => { - expect(answer({ ageMs: 86_400_000 }).success).toBe(true); - expect(answer({ ageMs: 0, cpu: { ...load.cpu, cores: 0.5 } }).success).toBe(true); - }); -}); - -describe("fitHostLoad", () => { - const memory = { - availableBytes: 4_000, - limited: false, - swapUsedBytes: 0, - totalBytes: 16_000, - }; - - it("caps ageMs at one day and drops a part whose figures break the schema, keeping the others", () => { - const fitted = fitHostLoad({ - ageMs: 90_000_000, - cpu: { busyPercent: 150, cores: 4, limited: false }, - disk: { freeBytes: -1, totalBytes: 500 }, - memory, - }); - - expect(fitted).toStrictEqual({ ageMs: 86_400_000, memory }); - expect(() => hostLoadSchema.parse(fitted)).not.toThrow(); - }); - - it("drops a memory part whose figures break the schema, keeping the others", () => { - const disk = { freeBytes: 1, totalBytes: 2 }; - - const fitted = fitHostLoad({ ageMs: 5, disk, memory: { ...memory, totalBytes: -1 } }); - - expect(fitted).toStrictEqual({ ageMs: 5, disk }); - }); - - it("leaves a reading that already fits as it is, and nothing as nothing", () => { - const reading = { - ageMs: 15, - cpu: { busyPercent: 12.5, cores: 0.5, limited: true }, - disk: { freeBytes: 1, totalBytes: 2 }, - memory, - }; - - expect(fitHostLoad(reading)).toStrictEqual(reading); - expect(fitHostLoad(undefined)).toBeUndefined(); - }); -}); - -describe("the worker view's load", () => { - const view = { - connection: "connected" as const, - drained: false, - id: "wrk_1", - lastSeenAt: 1, - }; - const load = { - cpu: { busyPercent: 34.2, cores: 10, limited: false }, - disk: { freeBytes: 120, totalBytes: 500 }, - memory: { - availableBytes: 4_000, - limited: false, - pressure: "normal" as const, - swapUsedBytes: 100, - totalBytes: 16_000, - }, - readAt: 8_000, - }; - - it("the worker view schema round-trips load, and refuses a negative readAt, a negative figure or a busy % over 100", () => { - expect(workerViewSchema.parse({ ...view, load })).toEqual({ ...view, load }); - expect(workerViewSchema.parse({ ...view, load: { readAt: 0 } })).toEqual({ - ...view, - load: { readAt: 0 }, - }); - - expect(() => workerViewSchema.parse({ ...view, load: { ...load, readAt: -1 } })).toThrow(); - expect(() => - workerViewSchema.parse({ ...view, load: { ...load, disk: { ...load.disk, freeBytes: -1 } } }), - ).toThrow(); - expect(() => - workerViewSchema.parse({ - ...view, - load: { ...load, cpu: { ...load.cpu, busyPercent: 100.5 } }, - }), - ).toThrow(); - }); -}); - -describe("the worker view's baselines", () => { - const view = { - connection: "connected" as const, - drained: false, - id: "wrk_1", - lastSeenAt: 1, - }; - const entry = { - mode: "full" as const, - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android" as const, - savedAt: "2026-11-02T13:10:00Z", - version: 2, - }; - - it("the worker view schema takes 1000 baselines and refuses 1001, and an entry's workerId of 1 to 128 characters and refuses an empty or 129-character one", () => { - const parse = (baselines: unknown[]) => workerViewSchema.parse({ ...view, baselines }); - - expect(parse([])).toEqual({ ...view, baselines: [] }); - expect(parse(Array.from({ length: 1000 }, () => entry))).toHaveProperty("baselines.999"); - expect(() => parse(Array.from({ length: 1001 }, () => entry))).toThrow(); - expect(parse([{ ...entry, workerId: "x".repeat(128) }])).toHaveProperty( - "baselines.0.workerId", - "x".repeat(128), - ); - expect(() => parse([{ ...entry, workerId: "" }])).toThrow(); - expect(() => parse([{ ...entry, workerId: "x".repeat(129) }])).toThrow(); - }); -}); - -describe("waitingRequestSchema's reason", () => { - const reasons = [ - "behind-earlier-request", - "device-limit", - "ram-budget", - "global-running-limit", - "platform-running-limit", - "waiting-for-device", - "no-free-physical", - ] as const; - const queued = { - createdAt: 1, - id: "req_1", - queuePosition: 1, - requesterId: "agent", - spec: { model: "iPhone 16", platform: "ios" }, - stage: "queued", - } as const; - - it("accepts each of the eight reasons on a queued entry and keeps it", () => { - expect(waitingRequestSchema.safeParse(queued).success).toBe(true); - for (const reason of reasons) { - expect(waitingRequestSchema.parse({ ...queued, reason })).toMatchObject({ reason }); - } - }); - - it("refuses a reason outside the eight on a queued entry", () => { - expect(waitingRequestSchema.safeParse({ ...queued, reason: "bogus" }).success).toBe(false); - }); - - it("refuses any reason on a starting entry", () => { - const { queuePosition: _position, ...rest } = queued; - const starting = { ...rest, stage: "starting" } as const; - - expect(waitingRequestSchema.safeParse(starting).success).toBe(true); - for (const reason of reasons) { - expect(waitingRequestSchema.safeParse({ ...starting, reason }).success).toBe(false); - } - }); -}); - -describe("waitingRequestSchema's workers", () => { - const reasons = [ - "disconnected", - "incompatible", - "not-ready", - "unhealthy", - "drained", - "busy", - "no-driver", - "unknown-model", - "runtime-missing", - "capability-missing", - "full", - "no-free-physical", - "not-chosen", - ] as const; - const queued = { - createdAt: 1, - id: "req_1", - queuePosition: 1, - requesterId: "agent", - spec: { model: "iPhone 16", platform: "ios" }, - stage: "queued", - } as const; - const withWorkers = (workers: unknown) => waitingRequestSchema.safeParse({ ...queued, workers }); - - it("accepts each of the thirteen reasons and a capacity reason with full, and keeps them", () => { - for (const reason of reasons) { - expect(withWorkers([{ reason, workerId: "wrk_a" }]).success).toBe(true); - } - expect( - waitingRequestSchema.parse({ - ...queued, - workers: [{ capacityReason: "ram-budget", reason: "full", workerId: "wrk_a" }], - }), - ).toMatchObject({ workers: [{ capacityReason: "ram-budget", reason: "full" }] }); - }); - - it("refuses an unknown reason, a capacity reason with drained, an unknown capacity reason and one workerId twice", () => { - expect(withWorkers([{ reason: "full", workerId: "wrk_a" }]).success).toBe(true); - expect(withWorkers([{ reason: "bogus", workerId: "wrk_a" }]).success).toBe(false); - expect( - withWorkers([{ capacityReason: "ram-budget", reason: "drained", workerId: "wrk_a" }]).success, - ).toBe(false); - expect( - withWorkers([{ capacityReason: "bogus", reason: "full", workerId: "wrk_a" }]).success, - ).toBe(false); - expect( - withWorkers([ - { reason: "drained", workerId: "wrk_a" }, - { reason: "full", workerId: "wrk_a" }, - ]).success, - ).toBe(false); - }); -}); - -describe("opaque fields stay optional on the wire", () => { - it("accepts an answer whose opaque driverData or payload key is absent (JSON drops undefined)", () => { - const record = { - physical: false, - id: "device-1", - driverDeviceId: "SIM-1", - spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, - mode: "full", - state: "ready", - createdAt: 0, - }; - const orphan = { - findings: [ - { kind: "orphan-device", platform: "ios", device: { deviceId: "SIM-1", address: "a" } }, - ], - sources: [], - }; - const envelope = { id: "evt_1", seq: 1, timestamp: 1, event: "x", module: "m" }; - expect(deviceRecordSchema.safeParse(record).success).toBe(true); - expect(doctorReportSchema.safeParse(orphan).success).toBe(true); - expect(eventEnvelopeSchema.safeParse(envelope).success).toBe(true); - }); -}); - -describe("statusWarmPoolSchema", () => { - const entry = { - booting: 0, - count: 1, - mode: "full", - model: "Pixel 8", - origin: "learned", - osVersion: "35", - platform: "android", - ready: 1, - }; - const block = (targets: readonly unknown[]) => ({ - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - targets, - }); - const { origin: _origin, ...withoutOrigin } = entry; - - it("accepts origin configured and learned, an optional imageTag, an optional baseline, and 320 entries, and refuses an entry without origin or with another word, a tag imageTagSchema refuses, a baseline name the name rule refuses, and 321 entries", () => { - const cases: readonly { name: string; targets: readonly unknown[]; accepted: boolean }[] = [ - { accepted: true, name: "learned", targets: [entry] }, - { accepted: true, name: "configured", targets: [{ ...entry, origin: "configured" }] }, - { accepted: true, name: "a tag", targets: [{ ...entry, imageTag: "google_apis_playstore" }] }, - { accepted: true, name: "320 entries", targets: Array.from({ length: 320 }, () => entry) }, - { accepted: true, name: "a baseline", targets: [{ ...entry, baseline: "checkout" }] }, - { accepted: false, name: "no origin", targets: [withoutOrigin] }, - { accepted: false, name: "another word", targets: [{ ...entry, origin: "remembered" }] }, - { accepted: false, name: "a bad tag", targets: [{ ...entry, imageTag: "no spaces" }] }, - { accepted: false, name: "a bad baseline", targets: [{ ...entry, baseline: "Not A Name" }] }, - { accepted: false, name: "321 entries", targets: Array.from({ length: 321 }, () => entry) }, - ]; - - expect( - cases.map(({ name, targets }) => [ - name, - statusWarmPoolSchema.safeParse(block(targets)).success, - ]), - ).toEqual(cases.map(({ accepted, name }) => [name, accepted])); - expect( - statusWarmPoolSchema.parse(block([{ ...entry, imageTag: "google_apis" }])).targets[0], - ).toMatchObject({ imageTag: "google_apis", origin: "learned" }); - expect( - statusWarmPoolSchema.parse(block([{ ...entry, baseline: "checkout" }])).targets[0], - ).toMatchObject({ baseline: "checkout", origin: "learned" }); - }); -}); From 473a5baefd51e1a39e7fb12c5447359cad7fb4a7 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:46 +0100 Subject: [PATCH 59/83] test(drivers): split catalog.test.ts by topic --- .../android/catalog-device-list.test.ts | 270 ++++ .../catalog-library-and-resolve.test.ts | 453 +++++++ .../android/catalog-refusals-and-fit.test.ts | 333 +++++ src/drivers/android/catalog.test.ts | 1148 +---------------- src/drivers/android/test-catalog.ts | 144 +++ 5 files changed, 1204 insertions(+), 1144 deletions(-) create mode 100644 src/drivers/android/catalog-device-list.test.ts create mode 100644 src/drivers/android/catalog-library-and-resolve.test.ts create mode 100644 src/drivers/android/catalog-refusals-and-fit.test.ts create mode 100644 src/drivers/android/test-catalog.ts diff --git a/src/drivers/android/catalog-device-list.test.ts b/src/drivers/android/catalog-device-list.test.ts new file mode 100644 index 000000000..85e502025 --- /dev/null +++ b/src/drivers/android/catalog-device-list.test.ts @@ -0,0 +1,270 @@ +import { describe, expect, it } from "vitest"; + +import { MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import type { ProcessRunner, ScriptedProcessExpectation } from "../../ports/index.js"; + +import { binaries, androidFilesystem, createPart } from "./test-catalog.js"; +import { + ANDROID_HOME_DIRECTORY as home, + ANDROID_SDK_ROOT as sdk, + customDeviceXml, + pixelDevices, + processResult, + writeDevicesXml, +} from "./testing.js"; + +// why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build +// `Catalog` directly over a memory filesystem and a scripted `avdmanager`. +describe("Catalog", () => { + describe("avdmanager list device reads", () => { + const toolsPackage = `${sdk}/cmdline-tools/latest`; + const sdklibJar = `${toolsPackage}/lib/sdklib/sdklib.core.jar`; + const userDevicesXml = `${home}/.android/devices.xml`; + const imageDevicesXml = `${sdk}/system-images/android-34/google_apis/arm64-v8a/devices.xml`; + const addOnDevicesXml = `${sdk}/add-ons/addon-acme/devices.xml`; + const extraDirectory = `${sdk}/extras/acme/device_profiles`; + const extraDevicesXml = `${extraDirectory}/devices.xml`; + const rereadDevices = `Available devices:\nid: 0 or "reread_phone"\n Name: Reread Phone\n OEM : Acme\n`; + + /** why: sdklib reads `extras///devices.xml` only when the item's source.properties says `Extra.Path=DeviceProfiles`. */ + async function profilesExtra(filesystem: MemoryFilesystem, devicesXml?: string): Promise { + await filesystem.mkdirp(extraDirectory); + await filesystem.writeFileAtomic( + `${extraDirectory}/source.properties`, + "Pkg.Revision=1\nExtra.Path=DeviceProfiles\n", + ); + if (devicesXml !== undefined) await filesystem.writeFileAtomic(extraDevicesXml, devicesXml); + } + + /** why: every input the read is keyed on is present, so a change to any of them is a change, not a creation. */ + async function readsHarness( + options: { + readonly avdmanager?: string; + readonly before?: (filesystem: MemoryFilesystem) => Promise; + readonly results?: readonly ScriptedProcessExpectation[]; + } = {}, + ) { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp(`${toolsPackage}/lib/sdklib`); + await filesystem.writeFileAtomic(sdklibJar, "jar"); + await filesystem.writeFileAtomic(`${toolsPackage}/source.properties`, "Pkg.Revision=20.0\n"); + await filesystem.writeFileAtomic(imageDevicesXml, ""); + await filesystem.mkdirp(`${sdk}/add-ons/addon-acme`); + await filesystem.writeFileAtomic(addOnDevicesXml, ""); + await options.before?.(filesystem); + const avdmanager = options.avdmanager ?? binaries.avdmanager; + const runner = new ScriptedProcessRunner( + options.results ?? [ + processResult(avdmanager, ["list", "device"], pixelDevices), + processResult(avdmanager, ["list", "device"], rereadDevices), + ], + ); + const part = createPart(filesystem, runner, { avdmanager }); + const listRuns = () => + runner.calls.filter( + (call) => call.command === avdmanager && call.args.join(" ") === "list device", + ); + return { part, filesystem, listRuns, runner }; + } + + it("runs `avdmanager list device` once, with a 30 s limit, for two catalogs read while none of its inputs changed", async () => { + const { part, listRuns } = await readsHarness(); + + const first = await part.listCatalog(); + const second = await part.listCatalog(); + + expect(first.models).toContain("Pixel 5"); + expect(second.models).toContain("Pixel 5"); + expect(second.models).not.toContain("Reread Phone"); + expect(listRuns()).toHaveLength(1); + expect(listRuns()[0]?.options.timeoutMs).toBe(30_000); + }); + + it("runs `avdmanager list device` again after any file it reads device profiles from changed", async () => { + const legacyTools = `${sdk}/tools`; + const legacyJar = `${legacyTools}/lib/sdklib-26.0.0-dev.jar`; + const cases: readonly { + readonly change: string; + readonly avdmanager?: string; + readonly before?: (filesystem: MemoryFilesystem) => Promise; + readonly act: (filesystem: MemoryFilesystem) => Promise; + }[] = [ + { + act: async (filesystem) => + filesystem.defineAttributes(sdklibJar, { modifiedAtMs: 5_000 }), + change: "the sdklib jar's modified time", + }, + { + act: async (filesystem) => filesystem.writeFileAtomic(sdklibJar, "jar, updated"), + change: "the sdklib jar's size", + }, + { + act: async (filesystem) => + filesystem.writeFileAtomic( + `${toolsPackage}/source.properties`, + "Pkg.Revision=21.0.1\n", + ), + change: "the command-line tools' source.properties", + }, + { + act: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Made", 2048)), + change: "devices.xml created", + }, + { + act: async (filesystem) => + writeDevicesXml( + filesystem, + customDeviceXml("Kept", 2048) + customDeviceXml("New", 2048), + ), + before: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Kept", 2048)), + change: "devices.xml edited", + }, + { + act: async (filesystem) => filesystem.rm(userDevicesXml), + before: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Kept", 2048)), + change: "devices.xml deleted", + }, + { + act: async (filesystem) => + filesystem.mkdirp(`${sdk}/system-images/android-35/google_apis/arm64-v8a`), + change: "a system image installed", + }, + { + act: async (filesystem) => { + await filesystem.mkdirp(`${sdk}/system-images/android-36/android-wear/arm64-v8a`); + await filesystem.writeFileAtomic( + `${sdk}/system-images/android-36/android-wear/arm64-v8a/devices.xml`, + "", + ); + }, + change: "a system image installed with its own devices.xml", + }, + { + act: async (filesystem) => + filesystem.writeFileAtomic(imageDevicesXml, ""), + change: "an installed system image's devices.xml", + }, + { + act: async (filesystem) => + filesystem.writeFileAtomic(addOnDevicesXml, ""), + change: "an add-on's devices.xml", + }, + { + act: async (filesystem) => filesystem.writeFileAtomic(extraDevicesXml, ""), + before: async (filesystem) => profilesExtra(filesystem), + change: "an extra's devices.xml created", + }, + { + act: async (filesystem) => + filesystem.writeFileAtomic(extraDevicesXml, ""), + before: async (filesystem) => profilesExtra(filesystem, ""), + change: "an extra's devices.xml edited", + }, + { + act: async (filesystem) => filesystem.rm(extraDevicesXml), + before: async (filesystem) => profilesExtra(filesystem, ""), + change: "an extra's devices.xml removed", + }, + { + act: async (filesystem) => + filesystem.writeFileAtomic( + `${extraDirectory}/source.properties`, + "Extra.Path=DeviceProfiles\n", + ), + before: async (filesystem) => { + await profilesExtra(filesystem, ""); + await filesystem.writeFileAtomic( + `${extraDirectory}/source.properties`, + "Extra.Path=usb_driver\n", + ); + }, + change: "an extra's source.properties made it a device-profiles extra", + }, + { + act: async (filesystem) => filesystem.writeFileAtomic(legacyJar, "legacy jar, updated"), + avdmanager: `${legacyTools}/bin/avdmanager`, + before: async (filesystem) => { + await filesystem.rm(toolsPackage); + await filesystem.mkdirp(`${legacyTools}/bin`); + await filesystem.writeFileAtomic(`${legacyTools}/bin/avdmanager`, "binary"); + await filesystem.writeFileAtomic(`${legacyTools}/bin/sdkmanager`, "binary"); + await filesystem.mkdirp(`${legacyTools}/lib`); + await filesystem.writeFileAtomic(legacyJar, "legacy jar"); + }, + change: "the legacy tools' sdklib jar", + }, + ]; + + const seen: Record = {}; + for (const row of cases) { + const { part, filesystem, listRuns } = await readsHarness({ + ...(row.avdmanager === undefined ? {} : { avdmanager: row.avdmanager }), + ...(row.before === undefined ? {} : { before: row.before }), + }); + const first = await part.listCatalog(); + await row.act(filesystem); + const second = await part.listCatalog(); + seen[row.change] = [ + first.models.includes("Reread Phone"), + second.models.includes("Reread Phone"), + listRuns().length, + ]; + } + + expect(seen).toEqual(Object.fromEntries(cases.map((row) => [row.change, [false, true, 2]]))); + }); + + it("runs `avdmanager list device` again after a read that failed", async () => { + const { part, listRuns } = await readsHarness({ + results: [ + { + match: { args: ["list", "device"], command: binaries.avdmanager }, + result: { code: 1, stderr: "Error: could not read device definitions", stdout: "" }, + }, + processResult(binaries.avdmanager, ["list", "device"], rereadDevices), + ], + }); + + await expect(part.listCatalog()).rejects.toThrow("could not read device definitions"); + const catalog = await part.listCatalog(); + + expect(catalog.models).toContain("Reread Phone"); + expect(listRuns()).toHaveLength(2); + }); + + it("runs one `avdmanager list device` for two catalogs read at once", async () => { + const scripted = new ScriptedProcessRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ]); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + // why: the run stays in flight until both catalogs have asked, so the second cannot be served from a finished read. + const runner: ProcessRunner = { + run: async (command, args, options) => { + await held; + return scripted.run(command, args, options); + }, + spawn: (command, args, options) => scripted.spawn(command, args, options), + spawnStreaming: (command, args, options) => scripted.spawnStreaming(command, args, options), + }; + const part = createPart(await androidFilesystem(), runner); + + const catalogs = Promise.allSettled([part.listCatalog(), part.listCatalog()]); + // why: the filesystem fake settles in microtasks, so one macrotask lets both catalogs + // reach the read before the first run ends. + await new Promise((resolve) => setImmediate(resolve)); + release(); + + expect( + (await catalogs).map( + (catalog) => catalog.status === "fulfilled" && catalog.value.models.includes("Pixel 5"), + ), + ).toEqual([true, true]); + expect(scripted.calls.filter((call) => call.args.join(" ") === "list device")).toHaveLength( + 1, + ); + }); + }); +}); diff --git a/src/drivers/android/catalog-library-and-resolve.test.ts b/src/drivers/android/catalog-library-and-resolve.test.ts new file mode 100644 index 000000000..4b9818a33 --- /dev/null +++ b/src/drivers/android/catalog-library-and-resolve.test.ts @@ -0,0 +1,453 @@ +import { describe, expect, it } from "vitest"; + +import { ScriptedProcessRunner } from "../../ports/index.js"; + +import type { DeviceProfileSourceDiagnostic } from "./device-profile-source.js"; +import { binaries, androidFilesystem, createPart, fitHarness } from "./test-catalog.js"; +import { + ANDROID_HOME_DIRECTORY as home, + customDeviceXml, + type FitImage, + HOST_IMAGES, + pixelDevices, + processResult, + testLibraryProfile, + twoPixelDevices, + writeDevicesXml, +} from "./testing.js"; + +// why: the verbatim source.properties of the real android-Baklava image installed beside +// android-36 and android-37.0 on a reporting host, so the fixture holds what the SDK really stores. +const BAKLAVA_SOURCE_PROPERTIES = + "Pkg.Desc=System Image arm64-v8a with Google APIs.\n" + + "Pkg.Revision=4\n" + + "AndroidVersion.ApiLevel=35\n" + + "AndroidVersion.CodeName=Baklava\n" + + "SystemImage.Abi=arm64-v8a\n" + + "SystemImage.TagId=google_apis\n"; + +// why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build +// `Catalog` directly over a memory filesystem and a scripted `avdmanager`. +describe("Catalog", () => { + describe("library catalog", () => { + const classes: Record = { + "Automotive (1024p landscape)": "auto", + "Automotive (1080p landscape)": "auto", + "Large Desktop": "desktop", + "Medium Desktop": "desktop", + "Medium Phone": "phone", + "Medium Tablet": "tablet", + "Pixel 6": "phone", + "Pixel 7": "phone", + "Pixel 8": "phone", + "Pixel 9": "phone", + "Pixel 9 Pro XL": "phone", + "Small Phone": "phone", + "Television (1080p)": "tv", + "Television (4K)": "tv", + "Wear OS Large Round": "watch", + "Wear OS Small Round": "watch", + }; + + it("surfaces malformed devices.xml as a diagnostic and falls through to UnknownModelError, never throwing from the parse itself", async () => { + const filesystem = await androidFilesystem(); + await filesystem.mkdirp(`${home}/.android`); + await filesystem.writeFileAtomic(`${home}/.android/devices.xml`, "not xml at all {{{"); + const runner = new ScriptedProcessRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ]); + const diagnostics: DeviceProfileSourceDiagnostic[] = []; + const part = createPart(filesystem, runner, { + onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), + }); + + await expect( + part.resolveSpec({ + model: "Nonexistent Model", + osVersion: "34", + platform: "android", + }), + ).rejects.toMatchObject({ name: "UnknownModelError" }); + + expect(diagnostics).toEqual([ + expect.objectContaining({ kind: "device-profile-source-unreadable" }), + ]); + }); + + it("lists Pixel 9 first in the catalog, once, as `phone`, not custom, with `pixel_9` as its other name", async () => { + const filesystem = await androidFilesystem(); + await writeDevicesXml( + filesystem, + customDeviceXml("pixel_9", 4096) + customDeviceXml("PIXEL 9", 4096), + ); + const runner = new ScriptedProcessRunner([ + processResult(binaries.avdmanager, ["list", "device"], twoPixelDevices), + ]); + const part = createPart(filesystem, runner); + + const catalog = await part.listCatalog(); + + expect(catalog.models[0]).toBe("Pixel 9"); + expect(catalog.models.filter((model) => model.toLocaleLowerCase() === "pixel 9")).toEqual([ + "Pixel 9", + ]); + expect(catalog.modelClasses["Pixel 9"]).toBe("phone"); + expect(catalog.customModels ?? []).not.toContain("Pixel 9"); + expect(catalog.modelAliases["Pixel 9"]).toEqual(["pixel_9"]); + }); + + it("every library model resolves, and the catalog lists all 16 with their table classes, when `avdmanager list device` lists none of them", async () => { + const runner = new ScriptedProcessRunner([ + processResult(binaries.avdmanager, ["list", "device"], "Available devices:\n"), + ]); + const part = createPart( + await androidFilesystem({ + images: [ + ["36", "google_apis", "arm64-v8a"], + ["36", "android-tv", "arm64-v8a"], + ["36", "android-wear", "arm64-v8a"], + ["36", "android-automotive", "arm64-v8a"], + ["36", "android-automotive-playstore", "arm64-v8a"], + ["36", "android-desktop", "arm64-v8a"], + ], + }), + runner, + ); + + const catalog = await part.listCatalog(); + const resolved: string[] = []; + for (const model of Object.keys(classes)) { + resolved.push((await part.resolveSpec({ model, platform: "android" })).model); + } + + expect(resolved).toEqual(Object.keys(classes)); + expect(Object.keys(classes).filter((model) => !catalog.models.includes(model))).toEqual([]); + for (const [model, deviceClass] of Object.entries(classes)) { + expect(catalog.modelClasses[model], model).toBe(deviceClass); + } + expect(catalog.modelAliases["Television (4K)"]).toEqual(["tv_4k"]); + }); + + it("the catalog classes Medium Tablet `tablet` when `avdmanager list device` also lists `medium_tablet`", async () => { + const listing = + 'Available devices:\nid: 0 or "medium_tablet"\n Name: Medium Tablet\n OEM : Generic\n'; + const runner = new ScriptedProcessRunner([ + processResult(binaries.avdmanager, ["list", "device"], listing), + ]); + const part = createPart(await androidFilesystem(), runner); + + const catalog = await part.listCatalog(); + + expect(catalog.modelClasses["Medium Tablet"]).toBe("tablet"); + expect(catalog.models.filter((model) => model === "Medium Tablet")).toHaveLength(1); + }); + }); + + describe("each row of the In short table", () => { + interface Row { + readonly name: string; + readonly images?: readonly FitImage[]; + readonly model: string; + readonly os?: string; + readonly tag?: string; + readonly resolves?: string; + readonly resolvedTag?: string; + readonly refusedDownloadable?: boolean; + readonly refusedComponent?: string; + readonly pkg?: string; + } + const withGoogleTv: readonly FitImage[] = [...HOST_IMAGES, ["36", "google-tv", "arm64-v8a"]]; + const withWear: readonly FitImage[] = [...HOST_IMAGES, ["34", "android-wear", "arm64-v8a"]]; + const rows: readonly Row[] = [ + { model: "Television (1080p)", name: "TV 1080p", resolves: "34" }, + { model: "Television (1080p)", name: "TV 1080p at 36", os: "36", refusedDownloadable: false }, + { + model: "Television (1080p)", + name: "TV 1080p with image tag google_apis", + refusedDownloadable: false, + tag: "google_apis", + }, + { model: "Television (720p)", name: "TV 720p, SDK only", resolves: "34" }, + { + images: withGoogleTv, + model: "Television (1080p)", + name: "TV 1080p with a google-tv image at 36", + resolves: "36", + }, + { images: withWear, model: "Wear OS Large Round", name: "Wear OS", resolves: "34" }, + { model: "Pixel 9", name: "Pixel 9", resolves: "36" }, + { model: "Pixel 9", name: "Pixel 9 at 34", os: "34", refusedDownloadable: false }, + { + model: "Pixel 9", + name: "Pixel 9 at 37", + os: "37", + refusedComponent: "37", + refusedDownloadable: true, + }, + { model: "Television (1080p)", name: "TV 1080p at 37", os: "37", refusedDownloadable: false }, + { model: "Pixel 6", name: "Pixel 6", resolves: "35" }, + { model: "Pixel 6", name: "Pixel 6 at 36", os: "36", refusedDownloadable: false }, + { + model: "Pixel 6", + name: "Pixel 6 at 37", + os: "37", + refusedComponent: "37", + refusedDownloadable: true, + }, + { + model: "Pixel 6", + name: "Pixel 6 with image tag google_apis_playstore", + refusedDownloadable: false, + tag: "google_apis_playstore", + }, + { + model: "Medium Phone", + name: "Medium Phone with image tag google_apis_playstore", + resolvedTag: "google_apis_playstore", + resolves: "36", + tag: "google_apis_playstore", + }, + { model: "Medium Phone", name: "Medium Phone at 35", os: "35", resolves: "35" }, + { model: "Pixel 9 Pro", name: "Pixel 9 Pro, SDK only", resolves: "36" }, + { model: "My Tablet", name: "My Tablet, custom", resolves: "36" }, + ]; + + it.each(rows)("$name gives its After result", async (row) => { + const { part, refusal, resolve } = await fitHarness(row.images ?? HOST_IMAGES); + + if (row.refusedDownloadable !== undefined) { + expect(await refusal(row.model, row.os, row.tag)).toMatchObject({ + component: row.refusedComponent, + downloadable: row.refusedDownloadable, + name: "RuntimeMissingError", + }); + if (row.os !== undefined) { + expect((await part.listCatalog()).modelRuntimes[row.model]).not.toContain(row.os); + } + return; + } + const spec = await resolve(row.model, row.os, row.tag); + expect(spec).toEqual({ + ...(row.resolvedTag === undefined ? {} : { imageTag: row.resolvedTag }), + model: row.model, + osVersion: row.resolves, + physical: false, + platform: "android", + }); + expect((await part.listCatalog()).modelRuntimes[row.model]).toContain(spec.osVersion); + }); + + it("pairs My Tablet with 35 and 36 and not the TV image at 34, and the TV with 34 only", async () => { + const { part } = await fitHarness(HOST_IMAGES); + + const { modelRuntimes } = await part.listCatalog(); + + expect(modelRuntimes["My Tablet"]).toEqual(["35", "36"]); + expect(modelRuntimes["Television (1080p)"]).toEqual(["34"]); + expect(modelRuntimes["Pixel 9"]).toEqual(["35", "36"]); + expect(modelRuntimes["Pixel 6"]).toEqual(["35"]); + }); + }); + + describe("downloads", () => { + it("is not downloadable for an uninstalled level above the profile's API range", async () => { + const old = await fitHarness([["35", "google_apis", "arm64-v8a"]], { + profiles: [ + testLibraryProfile({ + apiRange: { max: 33, min: 24 }, + name: "Old Phone", + names: ["Old Phone"], + }), + ], + }); + expect((await old.refusal("Old Phone", "34")).downloadable).toBe(false); + }); + }); + + describe("the catalog", () => { + it("lists in modelRuntimes only levels with a fitting image, and every installed image elsewhere", async () => { + const { part } = await fitHarness([ + ["35", "google_apis", "arm64-v8a"], + ["37", "android-xr", "arm64-v8a"], + ]); + + const catalog = await part.listCatalog(); + + expect(catalog.modelRuntimes["Pixel 5"]).toEqual(["35"]); + expect(catalog.modelRuntimes["Television (1080p)"]).toEqual([]); + expect(catalog.runtimes).toEqual(["35", "37"]); + expect(catalog.defaultRuntime).toBe("37"); + expect(catalog.images).toEqual([ + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + { abi: "arm64-v8a", runtime: "37", tag: "android-xr" }, + ]); + }); + + it("resolves every pair it lists, and refuses a level it leaves out", async () => { + const { part, refusal, resolve } = await fitHarness([ + ...HOST_IMAGES, + ["36", "android-wear", "arm64-v8a"], + ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=35\n"], + ]); + const catalog = await part.listCatalog(); + const pairs = Object.entries(catalog.modelRuntimes).flatMap(([model, levels]) => + levels.map((level) => ({ level, model })), + ); + const left = Object.entries(catalog.modelRuntimes).flatMap(([model, levels]) => + catalog.runtimes + .filter((level) => !levels.includes(level)) + .map((level) => ({ level, model })), + ); + expect(pairs.length).toBeGreaterThan(0); + expect(left.length).toBeGreaterThan(0); + + for (const { level, model } of pairs) { + expect(await resolve(model, level)).toMatchObject({ model, osVersion: level }); + } + for (const { level, model } of left) { + expect(await refusal(model, level)).toMatchObject({ + downloadable: false, + name: "RuntimeMissingError", + }); + } + }); + + it("pairs a devices.xml profile avdmanager did not list with every installed phone level", async () => { + const { part, resolve } = await fitHarness(HOST_IMAGES, { + devicesXml: customDeviceXml("Legacy Phone", 2048), + }); + + const catalog = await part.listCatalog(); + + expect(catalog.modelRuntimes["Legacy Phone"]).toEqual(["35", "36"]); + expect(await resolve("Legacy Phone", "35")).toMatchObject({ osVersion: "35" }); + }); + + it("counts android-Baklava with AndroidVersion.ApiLevel=35 as a level Pixel 9 pairs with", async () => { + const { part, resolve } = await fitHarness([ + ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=35\n"], + ["34", "google_apis", "arm64-v8a"], + ]); + + const catalog = await part.listCatalog(); + + expect(catalog.modelRuntimes["Pixel 9"]).toEqual(["Baklava"]); + expect(await resolve("Pixel 9")).toMatchObject({ osVersion: "Baklava" }); + }); + + it("ranks a preview image by the API level it records, so a newer numeric level stays the newest", async () => { + const { part, resolve } = await fitHarness([ + ["35", "google_apis", "arm64-v8a"], + ["36", "google_apis", "arm64-v8a"], + ["37.0", "google_apis", "arm64-v8a"], + ["Baklava", "google_apis", "arm64-v8a", BAKLAVA_SOURCE_PROPERTIES], + ]); + + const catalog = await part.listCatalog(); + + // why: stated first so the assertions below can only fail on the *ranking* — the preview is + // genuinely a listed candidate, not something the catalog left out. + expect(catalog.runtimes).toContain("Baklava"); + expect(catalog.modelRuntimes["Pixel 9"]).toContain("Baklava"); + expect(catalog.defaultRuntime).toBe("37.0"); + expect(await resolve("Pixel 9")).toMatchObject({ osVersion: "37.0" }); + }); + + it("ranks a preview image above the numeric levels when the level it records is higher", async () => { + const { part } = await fitHarness([ + ["37.0", "google_apis", "arm64-v8a"], + ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=38\n"], + ]); + + expect((await part.listCatalog()).defaultRuntime).toBe("Baklava"); + }); + + it("ranks a preview folder by the highest level any of its images records, so an image recording none does not raise it", async () => { + const { part } = await fitHarness([ + ["36", "google_apis", "arm64-v8a"], + ["37.0", "google_apis", "arm64-v8a"], + ["Baklava", "google_apis", "arm64-v8a", BAKLAVA_SOURCE_PROPERTIES], + ["Baklava", "google_apis_playstore", "arm64-v8a"], + ]); + + expect((await part.listCatalog()).defaultRuntime).toBe("37.0"); + }); + + it.each([ + ["lower first", "35", "38"], + ["higher first", "38", "35"], + ])( + "ranks a preview folder by the highest level among its images, whichever is read first (%s)", + async (_order, first, second) => { + const { part } = await fitHarness([ + ["37.0", "google_apis", "arm64-v8a"], + ["Baklava", "google_apis", "arm64-v8a", `AndroidVersion.ApiLevel=${first}\n`], + ["Baklava", "google_apis_playstore", "arm64-v8a", `AndroidVersion.ApiLevel=${second}\n`], + ]); + + expect((await part.listCatalog()).defaultRuntime).toBe("Baklava"); + }, + ); + + it("uses the folder name for --image-tag and the catalog's images[].tag when SystemImage.TagId differs", async () => { + const { part, refusal, resolve } = await fitHarness([ + ["35", "custom-folder", "arm64-v8a", "SystemImage.TagId=google_apis_playstore\n"], + ]); + + const catalog = await part.listCatalog(); + + expect(catalog.images).toEqual([{ abi: "arm64-v8a", runtime: "35", tag: "custom-folder" }]); + expect(await resolve("Pixel 5", undefined, "custom-folder")).toMatchObject({ + imageTag: "custom-folder", + }); + expect(await refusal("Pixel 5", undefined, "google_apis_playstore")).toMatchObject({ + downloadable: false, + name: "RuntimeMissingError", + }); + }); + }); + + describe("resolveSpec", () => { + it("gives a request with no version the newest level with a fitting image", async () => { + const { resolve } = await fitHarness([ + ["34", "android-tv", "arm64-v8a"], + ["35", "android-tv", "arm64-v8a"], + ["36", "google_apis", "arm64-v8a"], + ]); + + expect(await resolve("Television (1080p)")).toMatchObject({ osVersion: "35" }); + }); + + it("fails a request with no version RUNTIME_MISSING, not downloadable, when no level has a fitting image", async () => { + const { refusal } = await fitHarness([["36", "google_apis", "arm64-v8a"]]); + + expect(await refusal("Television (1080p)")).toMatchObject({ + component: undefined, + downloadable: false, + name: "RuntimeMissingError", + }); + }); + + it("skips a newest level whose image of the tag misfits and takes the newest level where it fits", async () => { + const { resolve } = await fitHarness( + [ + ["34", "google_apis_playstore", "arm64-v8a"], + ["36", "google_apis_playstore", "arm64-v8a"], + ], + { + profiles: [ + testLibraryProfile({ + apiRange: { max: 35, min: 24 }, + name: "Capped", + names: ["Capped"], + }), + ], + }, + ); + + expect(await resolve("Capped", undefined, "google_apis_playstore")).toMatchObject({ + osVersion: "34", + }); + }); + }); +}); diff --git a/src/drivers/android/catalog-refusals-and-fit.test.ts b/src/drivers/android/catalog-refusals-and-fit.test.ts new file mode 100644 index 000000000..f88b139eb --- /dev/null +++ b/src/drivers/android/catalog-refusals-and-fit.test.ts @@ -0,0 +1,333 @@ +import { describe, expect, it } from "vitest"; + +import { binaries, fitHarness } from "./test-catalog.js"; +import { + ANDROID_HOME_DIRECTORY as home, + customFitXml, + type FitImage, + HOST_IMAGES, + testLibraryProfile, + userListing, +} from "./testing.js"; + +// why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build +// `Catalog` directly over a memory filesystem and a scripted `avdmanager`. +describe("Catalog", () => { + describe("refusal messages", () => { + const phones: readonly FitImage[] = [ + ["35", "google_apis", "arm64-v8a"], + ["36", "google_apis_playstore", "arm64-v8a"], + ]; + + it("names the API range for an API level below the profile's minimum", async () => { + const { refusal } = await fitHarness([["34", "google_apis", "arm64-v8a"]]); + + expect((await refusal("Pixel 9", "34")).message).toBe( + "No installed system image fits Pixel 9 at Android 34: it needs Android 35 or newer", + ); + }); + + it("names the API range for a level above the profile's maximum", async () => { + const { refusal } = await fitHarness([["35", "google_apis", "arm64-v8a"]], { + profiles: [ + testLibraryProfile({ + apiRange: { max: 33, min: 24 }, + name: "Old Phone", + names: ["Old Phone"], + }), + ], + }); + + expect((await refusal("Old Phone", "35")).message).toBe( + "No installed system image fits Old Phone at Android 35: it needs Android 33 or older", + ); + }); + + it.each([ + ["Television (1080p)", "it needs a TV image"], + ["Wear OS Large Round", "it needs a Wear OS image"], + ["Automotive (1080p landscape)", "it needs an automotive image"], + ["Medium Desktop", "it needs a desktop image"], + ["XR Headset", "it needs an XR image"], + ["AI Glasses", "it needs an AI glasses image"], + ])("names the kind for %s with only phone images", async (model, reason) => { + const { refusal } = await fitHarness(phones); + + expect((await refusal(model, "35")).message).toBe( + `No installed system image fits ${model} at Android 35: ${reason}`, + ); + }); + + it("names phone or tablet for a profile with no kind whose only image is a TV image", async () => { + const { refusal } = await fitHarness([["34", "android-tv", "arm64-v8a"]]); + + expect((await refusal("Pixel 5", "34")).message).toBe( + "No installed system image fits Pixel 5 at Android 34: it needs a phone or tablet image", + ); + }); + + it("names the tag for any other kind", async () => { + const { refusal } = await fitHarness(phones); + + expect((await refusal("Odd Device", "35")).message).toBe( + "No installed system image fits Odd Device at Android 35: it needs an image tagged foo-device", + ); + }); + + it.each([ + ["Pixel 6", "36", "google_apis_playstore", "Play Store"], + ["Pixel 5", "35", "tablet", "tablet"], + ])( + "says the model does not take the kind of image it was given (%s)", + async (model, osVersion, imageTag, noun) => { + const { refusal } = await fitHarness([[osVersion, imageTag, "arm64-v8a"]]); + + expect((await refusal(model, osVersion)).message).toBe( + `No installed system image fits ${model} at Android ${osVersion}: ${model} does not take ${noun} images`, + ); + }, + ); + + it("says none of the installed images fit it when no single check rules out every image", async () => { + const { refusal } = await fitHarness([ + ["35", "google_apis_playstore", "arm64-v8a"], + ["35", "android-tv", "arm64-v8a"], + ]); + + expect((await refusal("Pixel 6", "35")).message).toBe( + "No installed system image fits Pixel 6 at Android 35: none of the installed images fit it", + ); + }); + + it("gives the API range reason for an image that fails both API range and kind", async () => { + const { refusal } = await fitHarness([["34", "android-tv", "arm64-v8a"]]); + + expect((await refusal("Pixel 9", "34")).message).toBe( + "No installed system image fits Pixel 9 at Android 34: it needs Android 35 or newer", + ); + }); + + it("drops the level from the message when no version was asked, reading every installed image", async () => { + const { refusal } = await fitHarness([["36", "google_apis", "arm64-v8a"]]); + + expect((await refusal("Television (1080p)")).message).toBe( + "No installed system image fits Television (1080p): it needs a TV image", + ); + }); + + it("says an image tag's image does not fit it, and why", async () => { + const { refusal } = await fitHarness(HOST_IMAGES); + + expect((await refusal("Television (1080p)", undefined, "google_apis")).message).toBe( + "No installed system image fits Television (1080p): the google_apis image does not fit it: it needs a TV image", + ); + expect((await refusal("Television (1080p)", "35", "google_apis")).message).toBe( + "No installed system image fits Television (1080p) at Android 35: the google_apis image does not fit it: it needs a TV image", + ); + }); + + it("says the level is not installed for a level with nothing installed", async () => { + const { refusal } = await fitHarness(HOST_IMAGES); + + expect((await refusal("Pixel 9", "37")).message).toBe( + "No installed system image fits Pixel 9 at Android 37: Android 37 is not installed", + ); + }); + + it("says no system image is installed on a host with no image at all", async () => { + const { refusal } = await fitHarness([]); + + expect((await refusal("Pixel 9")).message).toBe( + "No installed system image fits Pixel 9: no system image is installed", + ); + }); + + it("keeps today's message for an image tag with no image of that tag installed", async () => { + const { refusal } = await fitHarness(HOST_IMAGES); + + expect((await refusal("Pixel 5", "35", "default")).message).toBe( + "No default system image is installed for Android API 35", + ); + }); + }); + + describe("an image tag whose image misfits", () => { + it("fails RUNTIME_MISSING, not downloadable, and starts no sdkmanager", async () => { + const { refusal, runner } = await fitHarness(HOST_IMAGES); + + const failure = await refusal("Pixel 6", undefined, "google_apis_playstore"); + + expect(failure).toMatchObject({ downloadable: false, name: "RuntimeMissingError" }); + expect(failure.message).toContain("the google_apis_playstore image does not fit it"); + expect(runner.calls.map((call) => call.command)).not.toContain(binaries.sdkmanager); + }); + }); + + describe("custom profile fit from devices.xml", () => { + const wearImage: FitImage = [ + "34", + "android-wear", + "arm64-v8a", + "SystemImage.TagId=android-wear,google_apis_playstore\n", + ]; + + it("a custom profile listed by avdmanager gets the API range of its devices.xml entry: api-level 31- drops levels below 31 from its modelRuntimes", async () => { + const images: readonly FitImage[] = [ + ["30", "google_apis", "arm64-v8a"], + ["31", "google_apis", "arm64-v8a"], + ["30", "android-tv", "arm64-v8a"], + ["31", "android-tv", "arm64-v8a"], + ]; + const open = await fitHarness(images, { + devicesXml: customFitXml({ name: "Open Phone" }), + listing: userListing(["Open Phone"]), + }); + expect((await open.part.listCatalog()).modelRuntimes["Open Phone"]).toEqual(["30", "31"]); + const { part } = await fitHarness(images, { + devicesXml: + customFitXml({ apiLevel: "31-", name: "My Phone" }) + + customFitXml({ apiLevel: "31-", name: "My TV", tagId: "android-tv" }), + listing: userListing(["My Phone"], ["My TV", "android-tv"]), + }); + + const { modelRuntimes } = await part.listCatalog(); + + expect(modelRuntimes["My Phone"]).toEqual(["31"]); + expect(modelRuntimes["My TV"]).toEqual(["31"]); + }); + + it("a custom profile with playstore-enabled false drops Play Store images", async () => { + const images: readonly FitImage[] = [ + ["34", "google_apis_playstore", "arm64-v8a"], + ["35", "google_apis", "arm64-v8a"], + ]; + const allowed = await fitHarness(images, { + devicesXml: customFitXml({ name: "My Phone", playStore: "true" }), + listing: userListing(["My Phone"]), + }); + expect((await allowed.part.listCatalog()).modelRuntimes["My Phone"]).toEqual(["34", "35"]); + const { part } = await fitHarness(images, { + devicesXml: customFitXml({ name: "My Phone", playStore: "false" }), + listing: userListing(["My Phone"]), + }); + + expect((await part.listCatalog()).modelRuntimes["My Phone"]).toEqual(["35"]); + }); + + it("a custom Wear profile with playstore-enabled false keeps an image tagged android-wear,google_apis_playstore, listed by avdmanager or devices.xml-only", async () => { + const wear = customFitXml({ name: "My Watch", playStore: "false", tagId: "android-wear" }); + const listed = await fitHarness([wearImage], { + devicesXml: wear, + listing: userListing(["My Watch", "android-wear"]), + }); + const only = await fitHarness([wearImage], { devicesXml: wear, listing: userListing() }); + + expect((await listed.part.listCatalog()).modelRuntimes["My Watch"]).toEqual(["34"]); + expect((await only.part.listCatalog()).modelRuntimes["My Watch"]).toEqual(["34"]); + }); + + it("a custom profile with no type and a 10.1-inch screen that does not fold fits tablet images; a 6-inch one, one with folded dimensions and one with no diagonal do not", async () => { + const images: readonly FitImage[] = [ + ["35", "google_apis", "arm64-v8a"], + ["36", "tablet", "arm64-v8a"], + ]; + const { part } = await fitHarness(images, { + devicesXml: + customFitXml({ diagonal: "10.1", name: "Big" }) + + customFitXml({ diagonal: "6", name: "Small" }) + + customFitXml({ diagonal: "10.1", folded: true, name: "Folds" }) + + customFitXml({ name: "NoDiagonal" }), + listing: userListing(["Big"], ["Small"], ["Folds"], ["NoDiagonal"]), + }); + + const { modelRuntimes } = await part.listCatalog(); + + expect(modelRuntimes["Big"]).toEqual(["35", "36"]); + expect(modelRuntimes["Small"]).toEqual(["35"]); + expect(modelRuntimes["Folds"]).toEqual(["35"]); + expect(modelRuntimes["NoDiagonal"]).toEqual(["35"]); + }); + + it("a custom profile with nothing but a name pairs as after #489", async () => { + const { part } = await fitHarness(HOST_IMAGES, { + devicesXml: customFitXml({ name: "Old" }), + listing: userListing(["Old"]), + }); + + expect((await part.listCatalog()).modelRuntimes["Old"]).toEqual(["35", "36"]); + }); + + it("a custom profile's class still comes from avdmanager's Tag line", async () => { + const { part } = await fitHarness(HOST_IMAGES, { + devicesXml: customFitXml({ name: "My TV", tagId: "android-wear" }), + listing: userListing(["My TV", "android-tv"]), + }); + + expect((await part.listCatalog()).modelClasses["My TV"]).toBe("tv"); + }); + + it("a devices.xml profile avdmanager does not list fits by its own tag-id, range, Play Store flag and screen, and is classed by its tag-id", async () => { + const images: readonly FitImage[] = [ + ["30", "android-tv", "arm64-v8a"], + ["31", "android-tv", "arm64-v8a"], + ["30", "google_apis", "arm64-v8a"], + ["31", "google_apis_playstore", "arm64-v8a"], + ["32", "google_apis", "arm64-v8a"], + ["33", "tablet", "arm64-v8a"], + ]; + const { part } = await fitHarness(images, { + devicesXml: + customFitXml({ apiLevel: "31-", name: "Only TV", tagId: "android-tv" }) + + customFitXml({ name: "Only Phone", playStore: "false" }) + + customFitXml({ diagonal: "10.1", name: "Only Tablet" }), + listing: userListing(), + }); + + const catalog = await part.listCatalog(); + + expect(catalog.modelRuntimes["Only TV"]).toEqual(["31"]); + expect(catalog.modelRuntimes["Only Phone"]).toEqual(["30", "32"]); + expect(catalog.modelRuntimes["Only Tablet"]).toEqual(["30", "31", "32", "33"]); + expect(catalog.modelClasses["Only TV"]).toBe("tv"); + expect(catalog.modelClasses["Only Phone"]).toBe("phone"); + }); + + it("reads devices.xml from ANDROID_USER_HOME, then ANDROID_PREFS_ROOT, then ANDROID_SDK_HOME, then home", async () => { + const paths = [ + "/user-home/devices.xml", + "/prefs/.android/devices.xml", + "/sdk-home/.android/devices.xml", + `${home}/.android/devices.xml`, + ]; + const env = { + ANDROID_PREFS_ROOT: "/prefs", + ANDROID_SDK_HOME: "/sdk-home", + ANDROID_USER_HOME: "/user-home", + }; + const names = ["From user home", "From prefs root", "From sdk home", "From home"]; + const outcomes: string[][] = []; + for (let first = 0; first < paths.length; first++) { + const used: Record = {}; + if (first <= 0) used["ANDROID_USER_HOME"] = env.ANDROID_USER_HOME; + if (first <= 1) used["ANDROID_PREFS_ROOT"] = env.ANDROID_PREFS_ROOT; + if (first <= 2) used["ANDROID_SDK_HOME"] = env.ANDROID_SDK_HOME; + const { part, filesystem } = await fitHarness(HOST_IMAGES, { + extraEnv: used, + listing: userListing(), + }); + for (const [index, path] of paths.entries()) { + await filesystem.mkdirp(path.slice(0, path.lastIndexOf("/"))); + await filesystem.writeFileAtomic( + path, + `${customFitXml({ name: names[index] as string })}`, + ); + } + outcomes.push( + (await part.listCatalog()).models.filter((model) => model.startsWith("From ")), + ); + } + + expect(outcomes).toEqual(names.map((name) => [name])); + }); + }); +}); diff --git a/src/drivers/android/catalog.test.ts b/src/drivers/android/catalog.test.ts index 7e46932a0..4a4b164c8 100644 --- a/src/drivers/android/catalog.test.ts +++ b/src/drivers/android/catalog.test.ts @@ -1,38 +1,19 @@ import { describe, expect, it } from "vitest"; -import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; -import type { - ProcessHandle, - ProcessRunner, - ProcessRunOptions, - ScriptedProcessExpectation, -} from "../../ports/index.js"; -import { Catalog } from "./catalog.js"; -import type { - DeviceProfileSource, - DeviceProfileSourceDiagnostic, -} from "./device-profile-source.js"; -import { SystemImages } from "./system-images.js"; -import { LIBRARY_PROFILES, type LibraryDeviceProfile } from "./profiles/index.js"; +import { MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; + +import { LIBRARY_PROFILES } from "./profiles/index.js"; +import { avdmanager, binaries, androidFilesystem, createPart } from "./test-catalog.js"; import { ANDROID_HOME_DIRECTORY as home, ANDROID_SDK_ROOT as sdk, customDeviceXml, - customFitXml, - fitDevices, - type FitImage, - HOST_IMAGES, pixelDevices, processResult, - type Refusal, - testLibraryProfile, twoPixelDevices, - userListing, writeDevicesXml, } from "./testing.js"; -const avdmanager = `${sdk}/cmdline-tools/latest/bin/avdmanager`; -const binaries = { avdmanager, sdkmanager: `${sdk}/cmdline-tools/latest/bin/sdkmanager` } as const; const libraryModels = LIBRARY_PROFILES.map((profile) => profile.name); function libraryAliases(extra: Record): Record { @@ -67,135 +48,6 @@ class ReversedReaddirFilesystem extends MemoryFilesystem { } } -async function androidFilesystem( - options: { readonly images?: readonly (readonly [string, string, string])[] } = {}, - filesystem: MemoryFilesystem = new MemoryFilesystem(), -): Promise { - for (const [api, tag, abi] of options.images ?? [["34", "google_apis", "arm64-v8a"]]) { - await filesystem.mkdirp(`${sdk}/system-images/android-${api}/${tag}/${abi}`); - } - return filesystem; -} - -function createPart( - filesystem: MemoryFilesystem, - processRunner: ProcessRunner, - options: { - readonly avdmanager?: string; - readonly deviceProfileSources?: readonly DeviceProfileSource[]; - readonly extraEnv?: Readonly>; - readonly hostAbi?: string; - readonly onDiagnostic?: (diagnostic: DeviceProfileSourceDiagnostic) => void; - } = {}, -): Catalog { - return new Catalog({ - avdmanager: options.avdmanager ?? avdmanager, - avdmanagerEnv: () => ({ ANDROID_HOME: sdk }), - deviceProfileSources: options.deviceProfileSources, - env: { ANDROID_HOME: sdk, ...options.extraEnv }, - filesystem, - homeDirectory: home, - onDiagnostic: options.onDiagnostic, - processRunner, - sdkRoot: sdk, - systemImages: new SystemImages({ - acceptAndroidLicenses: false, - clock: new FakeClock(), - deviceRoot: `${home}/.simlock/devices/android`, - filesystem, - hostAbi: options.hostAbi ?? "arm64-v8a", - legacyAvdHome: `${home}/.android/avd`, - processRunner, - sdk: { - adb: `${sdk}/platform-tools/adb`, - emulator: `${sdk}/emulator/emulator`, - root: sdk, - sdkmanager: `${sdk}/cmdline-tools/latest/bin/sdkmanager`, - }, - }), - }); -} - -// why: the verbatim source.properties of the real android-Baklava image installed beside -// android-36 and android-37.0 on a reporting host, so the fixture holds what the SDK really stores. -const BAKLAVA_SOURCE_PROPERTIES = - "Pkg.Desc=System Image arm64-v8a with Google APIs.\n" + - "Pkg.Revision=4\n" + - "AndroidVersion.ApiLevel=35\n" + - "AndroidVersion.CodeName=Baklava\n" + - "SystemImage.Abi=arm64-v8a\n" + - "SystemImage.TagId=google_apis\n"; - -// why: `avdmanager list device` answers with the listing a test names however often it is asked, so a -// second read is counted rather than refused. -class ListingRunner extends ScriptedProcessRunner { - constructor(private readonly listing: string) { - super([]); - } - - override spawn( - command: string, - args: readonly string[], - options: ProcessRunOptions = {}, - ): ProcessHandle { - if (command === binaries.avdmanager && args[0] === "list") { - return new ScriptedProcessRunner([processResult(command, args, this.listing)]).spawn( - command, - args, - options, - ); - } - return super.spawn(command, args, options); - } -} - -async function fitHarness( - images: readonly FitImage[], - options: { - readonly devicesXml?: string; - readonly extraEnv?: Readonly>; - readonly hostAbi?: string; - readonly listing?: string; - readonly profiles?: readonly LibraryDeviceProfile[]; - } = {}, -) { - const filesystem = await androidFilesystem({ - images: images.map(([api, tag, abi]) => [api, tag, abi]), - }); - for (const [api, tag, abi, properties] of images) { - if (properties !== undefined) { - await filesystem.writeFileAtomic( - `${sdk}/system-images/android-${api}/${tag}/${abi}/source.properties`, - properties, - ); - } - } - if (options.devicesXml !== undefined) await writeDevicesXml(filesystem, options.devicesXml); - const runner = new ListingRunner(options.listing ?? fitDevices); - const part = createPart(filesystem, runner, { - ...(options.extraEnv === undefined ? {} : { extraEnv: options.extraEnv }), - hostAbi: options.hostAbi ?? "arm64-v8a", - ...(options.profiles === undefined - ? {} - : { deviceProfileSources: [{ profiles: async () => options.profiles ?? [] }] }), - }); - const resolve = (model: string, osVersion?: string, imageTag?: string) => - part.resolveSpec({ - model, - platform: "android", - ...(osVersion === undefined ? {} : { osVersion }), - ...(imageTag === undefined ? {} : { imageTag }), - }); - const refusal = (model: string, osVersion?: string, imageTag?: string) => - resolve(model, osVersion, imageTag).then( - () => { - throw new Error(`${model} ${osVersion ?? ""} ${imageTag ?? ""} resolved`); - }, - (error: unknown) => error as Refusal, - ); - return { filesystem, part, refusal, resolve, runner }; -} - // why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build // `Catalog` directly over a memory filesystem and a scripted `avdmanager`. describe("Catalog", () => { @@ -606,996 +458,4 @@ describe("Catalog", () => { }); }); }); - - describe("avdmanager list device reads", () => { - const toolsPackage = `${sdk}/cmdline-tools/latest`; - const sdklibJar = `${toolsPackage}/lib/sdklib/sdklib.core.jar`; - const userDevicesXml = `${home}/.android/devices.xml`; - const imageDevicesXml = `${sdk}/system-images/android-34/google_apis/arm64-v8a/devices.xml`; - const addOnDevicesXml = `${sdk}/add-ons/addon-acme/devices.xml`; - const extraDirectory = `${sdk}/extras/acme/device_profiles`; - const extraDevicesXml = `${extraDirectory}/devices.xml`; - const rereadDevices = `Available devices:\nid: 0 or "reread_phone"\n Name: Reread Phone\n OEM : Acme\n`; - - /** why: sdklib reads `extras///devices.xml` only when the item's source.properties says `Extra.Path=DeviceProfiles`. */ - async function profilesExtra(filesystem: MemoryFilesystem, devicesXml?: string): Promise { - await filesystem.mkdirp(extraDirectory); - await filesystem.writeFileAtomic( - `${extraDirectory}/source.properties`, - "Pkg.Revision=1\nExtra.Path=DeviceProfiles\n", - ); - if (devicesXml !== undefined) await filesystem.writeFileAtomic(extraDevicesXml, devicesXml); - } - - /** why: every input the read is keyed on is present, so a change to any of them is a change, not a creation. */ - async function readsHarness( - options: { - readonly avdmanager?: string; - readonly before?: (filesystem: MemoryFilesystem) => Promise; - readonly results?: readonly ScriptedProcessExpectation[]; - } = {}, - ) { - const filesystem = await androidFilesystem(); - await filesystem.mkdirp(`${toolsPackage}/lib/sdklib`); - await filesystem.writeFileAtomic(sdklibJar, "jar"); - await filesystem.writeFileAtomic(`${toolsPackage}/source.properties`, "Pkg.Revision=20.0\n"); - await filesystem.writeFileAtomic(imageDevicesXml, ""); - await filesystem.mkdirp(`${sdk}/add-ons/addon-acme`); - await filesystem.writeFileAtomic(addOnDevicesXml, ""); - await options.before?.(filesystem); - const avdmanager = options.avdmanager ?? binaries.avdmanager; - const runner = new ScriptedProcessRunner( - options.results ?? [ - processResult(avdmanager, ["list", "device"], pixelDevices), - processResult(avdmanager, ["list", "device"], rereadDevices), - ], - ); - const part = createPart(filesystem, runner, { avdmanager }); - const listRuns = () => - runner.calls.filter( - (call) => call.command === avdmanager && call.args.join(" ") === "list device", - ); - return { part, filesystem, listRuns, runner }; - } - - it("runs `avdmanager list device` once, with a 30 s limit, for two catalogs read while none of its inputs changed", async () => { - const { part, listRuns } = await readsHarness(); - - const first = await part.listCatalog(); - const second = await part.listCatalog(); - - expect(first.models).toContain("Pixel 5"); - expect(second.models).toContain("Pixel 5"); - expect(second.models).not.toContain("Reread Phone"); - expect(listRuns()).toHaveLength(1); - expect(listRuns()[0]?.options.timeoutMs).toBe(30_000); - }); - - it("runs `avdmanager list device` again after any file it reads device profiles from changed", async () => { - const legacyTools = `${sdk}/tools`; - const legacyJar = `${legacyTools}/lib/sdklib-26.0.0-dev.jar`; - const cases: readonly { - readonly change: string; - readonly avdmanager?: string; - readonly before?: (filesystem: MemoryFilesystem) => Promise; - readonly act: (filesystem: MemoryFilesystem) => Promise; - }[] = [ - { - act: async (filesystem) => - filesystem.defineAttributes(sdklibJar, { modifiedAtMs: 5_000 }), - change: "the sdklib jar's modified time", - }, - { - act: async (filesystem) => filesystem.writeFileAtomic(sdklibJar, "jar, updated"), - change: "the sdklib jar's size", - }, - { - act: async (filesystem) => - filesystem.writeFileAtomic( - `${toolsPackage}/source.properties`, - "Pkg.Revision=21.0.1\n", - ), - change: "the command-line tools' source.properties", - }, - { - act: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Made", 2048)), - change: "devices.xml created", - }, - { - act: async (filesystem) => - writeDevicesXml( - filesystem, - customDeviceXml("Kept", 2048) + customDeviceXml("New", 2048), - ), - before: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Kept", 2048)), - change: "devices.xml edited", - }, - { - act: async (filesystem) => filesystem.rm(userDevicesXml), - before: async (filesystem) => writeDevicesXml(filesystem, customDeviceXml("Kept", 2048)), - change: "devices.xml deleted", - }, - { - act: async (filesystem) => - filesystem.mkdirp(`${sdk}/system-images/android-35/google_apis/arm64-v8a`), - change: "a system image installed", - }, - { - act: async (filesystem) => { - await filesystem.mkdirp(`${sdk}/system-images/android-36/android-wear/arm64-v8a`); - await filesystem.writeFileAtomic( - `${sdk}/system-images/android-36/android-wear/arm64-v8a/devices.xml`, - "", - ); - }, - change: "a system image installed with its own devices.xml", - }, - { - act: async (filesystem) => - filesystem.writeFileAtomic(imageDevicesXml, ""), - change: "an installed system image's devices.xml", - }, - { - act: async (filesystem) => - filesystem.writeFileAtomic(addOnDevicesXml, ""), - change: "an add-on's devices.xml", - }, - { - act: async (filesystem) => filesystem.writeFileAtomic(extraDevicesXml, ""), - before: async (filesystem) => profilesExtra(filesystem), - change: "an extra's devices.xml created", - }, - { - act: async (filesystem) => - filesystem.writeFileAtomic(extraDevicesXml, ""), - before: async (filesystem) => profilesExtra(filesystem, ""), - change: "an extra's devices.xml edited", - }, - { - act: async (filesystem) => filesystem.rm(extraDevicesXml), - before: async (filesystem) => profilesExtra(filesystem, ""), - change: "an extra's devices.xml removed", - }, - { - act: async (filesystem) => - filesystem.writeFileAtomic( - `${extraDirectory}/source.properties`, - "Extra.Path=DeviceProfiles\n", - ), - before: async (filesystem) => { - await profilesExtra(filesystem, ""); - await filesystem.writeFileAtomic( - `${extraDirectory}/source.properties`, - "Extra.Path=usb_driver\n", - ); - }, - change: "an extra's source.properties made it a device-profiles extra", - }, - { - act: async (filesystem) => filesystem.writeFileAtomic(legacyJar, "legacy jar, updated"), - avdmanager: `${legacyTools}/bin/avdmanager`, - before: async (filesystem) => { - await filesystem.rm(toolsPackage); - await filesystem.mkdirp(`${legacyTools}/bin`); - await filesystem.writeFileAtomic(`${legacyTools}/bin/avdmanager`, "binary"); - await filesystem.writeFileAtomic(`${legacyTools}/bin/sdkmanager`, "binary"); - await filesystem.mkdirp(`${legacyTools}/lib`); - await filesystem.writeFileAtomic(legacyJar, "legacy jar"); - }, - change: "the legacy tools' sdklib jar", - }, - ]; - - const seen: Record = {}; - for (const row of cases) { - const { part, filesystem, listRuns } = await readsHarness({ - ...(row.avdmanager === undefined ? {} : { avdmanager: row.avdmanager }), - ...(row.before === undefined ? {} : { before: row.before }), - }); - const first = await part.listCatalog(); - await row.act(filesystem); - const second = await part.listCatalog(); - seen[row.change] = [ - first.models.includes("Reread Phone"), - second.models.includes("Reread Phone"), - listRuns().length, - ]; - } - - expect(seen).toEqual(Object.fromEntries(cases.map((row) => [row.change, [false, true, 2]]))); - }); - - it("runs `avdmanager list device` again after a read that failed", async () => { - const { part, listRuns } = await readsHarness({ - results: [ - { - match: { args: ["list", "device"], command: binaries.avdmanager }, - result: { code: 1, stderr: "Error: could not read device definitions", stdout: "" }, - }, - processResult(binaries.avdmanager, ["list", "device"], rereadDevices), - ], - }); - - await expect(part.listCatalog()).rejects.toThrow("could not read device definitions"); - const catalog = await part.listCatalog(); - - expect(catalog.models).toContain("Reread Phone"); - expect(listRuns()).toHaveLength(2); - }); - - it("runs one `avdmanager list device` for two catalogs read at once", async () => { - const scripted = new ScriptedProcessRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ]); - let release: () => void = () => undefined; - const held = new Promise((resolve) => { - release = resolve; - }); - // why: the run stays in flight until both catalogs have asked, so the second cannot be served from a finished read. - const runner: ProcessRunner = { - run: async (command, args, options) => { - await held; - return scripted.run(command, args, options); - }, - spawn: (command, args, options) => scripted.spawn(command, args, options), - spawnStreaming: (command, args, options) => scripted.spawnStreaming(command, args, options), - }; - const part = createPart(await androidFilesystem(), runner); - - const catalogs = Promise.allSettled([part.listCatalog(), part.listCatalog()]); - // why: the filesystem fake settles in microtasks, so one macrotask lets both catalogs - // reach the read before the first run ends. - await new Promise((resolve) => setImmediate(resolve)); - release(); - - expect( - (await catalogs).map( - (catalog) => catalog.status === "fulfilled" && catalog.value.models.includes("Pixel 5"), - ), - ).toEqual([true, true]); - expect(scripted.calls.filter((call) => call.args.join(" ") === "list device")).toHaveLength( - 1, - ); - }); - }); - - describe("library catalog", () => { - const classes: Record = { - "Automotive (1024p landscape)": "auto", - "Automotive (1080p landscape)": "auto", - "Large Desktop": "desktop", - "Medium Desktop": "desktop", - "Medium Phone": "phone", - "Medium Tablet": "tablet", - "Pixel 6": "phone", - "Pixel 7": "phone", - "Pixel 8": "phone", - "Pixel 9": "phone", - "Pixel 9 Pro XL": "phone", - "Small Phone": "phone", - "Television (1080p)": "tv", - "Television (4K)": "tv", - "Wear OS Large Round": "watch", - "Wear OS Small Round": "watch", - }; - - it("surfaces malformed devices.xml as a diagnostic and falls through to UnknownModelError, never throwing from the parse itself", async () => { - const filesystem = await androidFilesystem(); - await filesystem.mkdirp(`${home}/.android`); - await filesystem.writeFileAtomic(`${home}/.android/devices.xml`, "not xml at all {{{"); - const runner = new ScriptedProcessRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ]); - const diagnostics: DeviceProfileSourceDiagnostic[] = []; - const part = createPart(filesystem, runner, { - onDiagnostic: (diagnostic) => diagnostics.push(diagnostic), - }); - - await expect( - part.resolveSpec({ - model: "Nonexistent Model", - osVersion: "34", - platform: "android", - }), - ).rejects.toMatchObject({ name: "UnknownModelError" }); - - expect(diagnostics).toEqual([ - expect.objectContaining({ kind: "device-profile-source-unreadable" }), - ]); - }); - - it("lists Pixel 9 first in the catalog, once, as `phone`, not custom, with `pixel_9` as its other name", async () => { - const filesystem = await androidFilesystem(); - await writeDevicesXml( - filesystem, - customDeviceXml("pixel_9", 4096) + customDeviceXml("PIXEL 9", 4096), - ); - const runner = new ScriptedProcessRunner([ - processResult(binaries.avdmanager, ["list", "device"], twoPixelDevices), - ]); - const part = createPart(filesystem, runner); - - const catalog = await part.listCatalog(); - - expect(catalog.models[0]).toBe("Pixel 9"); - expect(catalog.models.filter((model) => model.toLocaleLowerCase() === "pixel 9")).toEqual([ - "Pixel 9", - ]); - expect(catalog.modelClasses["Pixel 9"]).toBe("phone"); - expect(catalog.customModels ?? []).not.toContain("Pixel 9"); - expect(catalog.modelAliases["Pixel 9"]).toEqual(["pixel_9"]); - }); - - it("every library model resolves, and the catalog lists all 16 with their table classes, when `avdmanager list device` lists none of them", async () => { - const runner = new ScriptedProcessRunner([ - processResult(binaries.avdmanager, ["list", "device"], "Available devices:\n"), - ]); - const part = createPart( - await androidFilesystem({ - images: [ - ["36", "google_apis", "arm64-v8a"], - ["36", "android-tv", "arm64-v8a"], - ["36", "android-wear", "arm64-v8a"], - ["36", "android-automotive", "arm64-v8a"], - ["36", "android-automotive-playstore", "arm64-v8a"], - ["36", "android-desktop", "arm64-v8a"], - ], - }), - runner, - ); - - const catalog = await part.listCatalog(); - const resolved: string[] = []; - for (const model of Object.keys(classes)) { - resolved.push((await part.resolveSpec({ model, platform: "android" })).model); - } - - expect(resolved).toEqual(Object.keys(classes)); - expect(Object.keys(classes).filter((model) => !catalog.models.includes(model))).toEqual([]); - for (const [model, deviceClass] of Object.entries(classes)) { - expect(catalog.modelClasses[model], model).toBe(deviceClass); - } - expect(catalog.modelAliases["Television (4K)"]).toEqual(["tv_4k"]); - }); - - it("the catalog classes Medium Tablet `tablet` when `avdmanager list device` also lists `medium_tablet`", async () => { - const listing = - 'Available devices:\nid: 0 or "medium_tablet"\n Name: Medium Tablet\n OEM : Generic\n'; - const runner = new ScriptedProcessRunner([ - processResult(binaries.avdmanager, ["list", "device"], listing), - ]); - const part = createPart(await androidFilesystem(), runner); - - const catalog = await part.listCatalog(); - - expect(catalog.modelClasses["Medium Tablet"]).toBe("tablet"); - expect(catalog.models.filter((model) => model === "Medium Tablet")).toHaveLength(1); - }); - }); - - describe("each row of the In short table", () => { - interface Row { - readonly name: string; - readonly images?: readonly FitImage[]; - readonly model: string; - readonly os?: string; - readonly tag?: string; - readonly resolves?: string; - readonly resolvedTag?: string; - readonly refusedDownloadable?: boolean; - readonly refusedComponent?: string; - readonly pkg?: string; - } - const withGoogleTv: readonly FitImage[] = [...HOST_IMAGES, ["36", "google-tv", "arm64-v8a"]]; - const withWear: readonly FitImage[] = [...HOST_IMAGES, ["34", "android-wear", "arm64-v8a"]]; - const rows: readonly Row[] = [ - { model: "Television (1080p)", name: "TV 1080p", resolves: "34" }, - { model: "Television (1080p)", name: "TV 1080p at 36", os: "36", refusedDownloadable: false }, - { - model: "Television (1080p)", - name: "TV 1080p with image tag google_apis", - refusedDownloadable: false, - tag: "google_apis", - }, - { model: "Television (720p)", name: "TV 720p, SDK only", resolves: "34" }, - { - images: withGoogleTv, - model: "Television (1080p)", - name: "TV 1080p with a google-tv image at 36", - resolves: "36", - }, - { images: withWear, model: "Wear OS Large Round", name: "Wear OS", resolves: "34" }, - { model: "Pixel 9", name: "Pixel 9", resolves: "36" }, - { model: "Pixel 9", name: "Pixel 9 at 34", os: "34", refusedDownloadable: false }, - { - model: "Pixel 9", - name: "Pixel 9 at 37", - os: "37", - refusedComponent: "37", - refusedDownloadable: true, - }, - { model: "Television (1080p)", name: "TV 1080p at 37", os: "37", refusedDownloadable: false }, - { model: "Pixel 6", name: "Pixel 6", resolves: "35" }, - { model: "Pixel 6", name: "Pixel 6 at 36", os: "36", refusedDownloadable: false }, - { - model: "Pixel 6", - name: "Pixel 6 at 37", - os: "37", - refusedComponent: "37", - refusedDownloadable: true, - }, - { - model: "Pixel 6", - name: "Pixel 6 with image tag google_apis_playstore", - refusedDownloadable: false, - tag: "google_apis_playstore", - }, - { - model: "Medium Phone", - name: "Medium Phone with image tag google_apis_playstore", - resolvedTag: "google_apis_playstore", - resolves: "36", - tag: "google_apis_playstore", - }, - { model: "Medium Phone", name: "Medium Phone at 35", os: "35", resolves: "35" }, - { model: "Pixel 9 Pro", name: "Pixel 9 Pro, SDK only", resolves: "36" }, - { model: "My Tablet", name: "My Tablet, custom", resolves: "36" }, - ]; - - it.each(rows)("$name gives its After result", async (row) => { - const { part, refusal, resolve } = await fitHarness(row.images ?? HOST_IMAGES); - - if (row.refusedDownloadable !== undefined) { - expect(await refusal(row.model, row.os, row.tag)).toMatchObject({ - component: row.refusedComponent, - downloadable: row.refusedDownloadable, - name: "RuntimeMissingError", - }); - if (row.os !== undefined) { - expect((await part.listCatalog()).modelRuntimes[row.model]).not.toContain(row.os); - } - return; - } - const spec = await resolve(row.model, row.os, row.tag); - expect(spec).toEqual({ - ...(row.resolvedTag === undefined ? {} : { imageTag: row.resolvedTag }), - model: row.model, - osVersion: row.resolves, - physical: false, - platform: "android", - }); - expect((await part.listCatalog()).modelRuntimes[row.model]).toContain(spec.osVersion); - }); - - it("pairs My Tablet with 35 and 36 and not the TV image at 34, and the TV with 34 only", async () => { - const { part } = await fitHarness(HOST_IMAGES); - - const { modelRuntimes } = await part.listCatalog(); - - expect(modelRuntimes["My Tablet"]).toEqual(["35", "36"]); - expect(modelRuntimes["Television (1080p)"]).toEqual(["34"]); - expect(modelRuntimes["Pixel 9"]).toEqual(["35", "36"]); - expect(modelRuntimes["Pixel 6"]).toEqual(["35"]); - }); - }); - - describe("downloads", () => { - it("is not downloadable for an uninstalled level above the profile's API range", async () => { - const old = await fitHarness([["35", "google_apis", "arm64-v8a"]], { - profiles: [ - testLibraryProfile({ - apiRange: { max: 33, min: 24 }, - name: "Old Phone", - names: ["Old Phone"], - }), - ], - }); - expect((await old.refusal("Old Phone", "34")).downloadable).toBe(false); - }); - }); - - describe("the catalog", () => { - it("lists in modelRuntimes only levels with a fitting image, and every installed image elsewhere", async () => { - const { part } = await fitHarness([ - ["35", "google_apis", "arm64-v8a"], - ["37", "android-xr", "arm64-v8a"], - ]); - - const catalog = await part.listCatalog(); - - expect(catalog.modelRuntimes["Pixel 5"]).toEqual(["35"]); - expect(catalog.modelRuntimes["Television (1080p)"]).toEqual([]); - expect(catalog.runtimes).toEqual(["35", "37"]); - expect(catalog.defaultRuntime).toBe("37"); - expect(catalog.images).toEqual([ - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - { abi: "arm64-v8a", runtime: "37", tag: "android-xr" }, - ]); - }); - - it("resolves every pair it lists, and refuses a level it leaves out", async () => { - const { part, refusal, resolve } = await fitHarness([ - ...HOST_IMAGES, - ["36", "android-wear", "arm64-v8a"], - ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=35\n"], - ]); - const catalog = await part.listCatalog(); - const pairs = Object.entries(catalog.modelRuntimes).flatMap(([model, levels]) => - levels.map((level) => ({ level, model })), - ); - const left = Object.entries(catalog.modelRuntimes).flatMap(([model, levels]) => - catalog.runtimes - .filter((level) => !levels.includes(level)) - .map((level) => ({ level, model })), - ); - expect(pairs.length).toBeGreaterThan(0); - expect(left.length).toBeGreaterThan(0); - - for (const { level, model } of pairs) { - expect(await resolve(model, level)).toMatchObject({ model, osVersion: level }); - } - for (const { level, model } of left) { - expect(await refusal(model, level)).toMatchObject({ - downloadable: false, - name: "RuntimeMissingError", - }); - } - }); - - it("pairs a devices.xml profile avdmanager did not list with every installed phone level", async () => { - const { part, resolve } = await fitHarness(HOST_IMAGES, { - devicesXml: customDeviceXml("Legacy Phone", 2048), - }); - - const catalog = await part.listCatalog(); - - expect(catalog.modelRuntimes["Legacy Phone"]).toEqual(["35", "36"]); - expect(await resolve("Legacy Phone", "35")).toMatchObject({ osVersion: "35" }); - }); - - it("counts android-Baklava with AndroidVersion.ApiLevel=35 as a level Pixel 9 pairs with", async () => { - const { part, resolve } = await fitHarness([ - ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=35\n"], - ["34", "google_apis", "arm64-v8a"], - ]); - - const catalog = await part.listCatalog(); - - expect(catalog.modelRuntimes["Pixel 9"]).toEqual(["Baklava"]); - expect(await resolve("Pixel 9")).toMatchObject({ osVersion: "Baklava" }); - }); - - it("ranks a preview image by the API level it records, so a newer numeric level stays the newest", async () => { - const { part, resolve } = await fitHarness([ - ["35", "google_apis", "arm64-v8a"], - ["36", "google_apis", "arm64-v8a"], - ["37.0", "google_apis", "arm64-v8a"], - ["Baklava", "google_apis", "arm64-v8a", BAKLAVA_SOURCE_PROPERTIES], - ]); - - const catalog = await part.listCatalog(); - - // why: stated first so the assertions below can only fail on the *ranking* — the preview is - // genuinely a listed candidate, not something the catalog left out. - expect(catalog.runtimes).toContain("Baklava"); - expect(catalog.modelRuntimes["Pixel 9"]).toContain("Baklava"); - expect(catalog.defaultRuntime).toBe("37.0"); - expect(await resolve("Pixel 9")).toMatchObject({ osVersion: "37.0" }); - }); - - it("ranks a preview image above the numeric levels when the level it records is higher", async () => { - const { part } = await fitHarness([ - ["37.0", "google_apis", "arm64-v8a"], - ["Baklava", "google_apis", "arm64-v8a", "AndroidVersion.ApiLevel=38\n"], - ]); - - expect((await part.listCatalog()).defaultRuntime).toBe("Baklava"); - }); - - it("ranks a preview folder by the highest level any of its images records, so an image recording none does not raise it", async () => { - const { part } = await fitHarness([ - ["36", "google_apis", "arm64-v8a"], - ["37.0", "google_apis", "arm64-v8a"], - ["Baklava", "google_apis", "arm64-v8a", BAKLAVA_SOURCE_PROPERTIES], - ["Baklava", "google_apis_playstore", "arm64-v8a"], - ]); - - expect((await part.listCatalog()).defaultRuntime).toBe("37.0"); - }); - - it.each([ - ["lower first", "35", "38"], - ["higher first", "38", "35"], - ])( - "ranks a preview folder by the highest level among its images, whichever is read first (%s)", - async (_order, first, second) => { - const { part } = await fitHarness([ - ["37.0", "google_apis", "arm64-v8a"], - ["Baklava", "google_apis", "arm64-v8a", `AndroidVersion.ApiLevel=${first}\n`], - ["Baklava", "google_apis_playstore", "arm64-v8a", `AndroidVersion.ApiLevel=${second}\n`], - ]); - - expect((await part.listCatalog()).defaultRuntime).toBe("Baklava"); - }, - ); - - it("uses the folder name for --image-tag and the catalog's images[].tag when SystemImage.TagId differs", async () => { - const { part, refusal, resolve } = await fitHarness([ - ["35", "custom-folder", "arm64-v8a", "SystemImage.TagId=google_apis_playstore\n"], - ]); - - const catalog = await part.listCatalog(); - - expect(catalog.images).toEqual([{ abi: "arm64-v8a", runtime: "35", tag: "custom-folder" }]); - expect(await resolve("Pixel 5", undefined, "custom-folder")).toMatchObject({ - imageTag: "custom-folder", - }); - expect(await refusal("Pixel 5", undefined, "google_apis_playstore")).toMatchObject({ - downloadable: false, - name: "RuntimeMissingError", - }); - }); - }); - - describe("resolveSpec", () => { - it("gives a request with no version the newest level with a fitting image", async () => { - const { resolve } = await fitHarness([ - ["34", "android-tv", "arm64-v8a"], - ["35", "android-tv", "arm64-v8a"], - ["36", "google_apis", "arm64-v8a"], - ]); - - expect(await resolve("Television (1080p)")).toMatchObject({ osVersion: "35" }); - }); - - it("fails a request with no version RUNTIME_MISSING, not downloadable, when no level has a fitting image", async () => { - const { refusal } = await fitHarness([["36", "google_apis", "arm64-v8a"]]); - - expect(await refusal("Television (1080p)")).toMatchObject({ - component: undefined, - downloadable: false, - name: "RuntimeMissingError", - }); - }); - - it("skips a newest level whose image of the tag misfits and takes the newest level where it fits", async () => { - const { resolve } = await fitHarness( - [ - ["34", "google_apis_playstore", "arm64-v8a"], - ["36", "google_apis_playstore", "arm64-v8a"], - ], - { - profiles: [ - testLibraryProfile({ - apiRange: { max: 35, min: 24 }, - name: "Capped", - names: ["Capped"], - }), - ], - }, - ); - - expect(await resolve("Capped", undefined, "google_apis_playstore")).toMatchObject({ - osVersion: "34", - }); - }); - }); - - describe("refusal messages", () => { - const phones: readonly FitImage[] = [ - ["35", "google_apis", "arm64-v8a"], - ["36", "google_apis_playstore", "arm64-v8a"], - ]; - - it("names the API range for an API level below the profile's minimum", async () => { - const { refusal } = await fitHarness([["34", "google_apis", "arm64-v8a"]]); - - expect((await refusal("Pixel 9", "34")).message).toBe( - "No installed system image fits Pixel 9 at Android 34: it needs Android 35 or newer", - ); - }); - - it("names the API range for a level above the profile's maximum", async () => { - const { refusal } = await fitHarness([["35", "google_apis", "arm64-v8a"]], { - profiles: [ - testLibraryProfile({ - apiRange: { max: 33, min: 24 }, - name: "Old Phone", - names: ["Old Phone"], - }), - ], - }); - - expect((await refusal("Old Phone", "35")).message).toBe( - "No installed system image fits Old Phone at Android 35: it needs Android 33 or older", - ); - }); - - it.each([ - ["Television (1080p)", "it needs a TV image"], - ["Wear OS Large Round", "it needs a Wear OS image"], - ["Automotive (1080p landscape)", "it needs an automotive image"], - ["Medium Desktop", "it needs a desktop image"], - ["XR Headset", "it needs an XR image"], - ["AI Glasses", "it needs an AI glasses image"], - ])("names the kind for %s with only phone images", async (model, reason) => { - const { refusal } = await fitHarness(phones); - - expect((await refusal(model, "35")).message).toBe( - `No installed system image fits ${model} at Android 35: ${reason}`, - ); - }); - - it("names phone or tablet for a profile with no kind whose only image is a TV image", async () => { - const { refusal } = await fitHarness([["34", "android-tv", "arm64-v8a"]]); - - expect((await refusal("Pixel 5", "34")).message).toBe( - "No installed system image fits Pixel 5 at Android 34: it needs a phone or tablet image", - ); - }); - - it("names the tag for any other kind", async () => { - const { refusal } = await fitHarness(phones); - - expect((await refusal("Odd Device", "35")).message).toBe( - "No installed system image fits Odd Device at Android 35: it needs an image tagged foo-device", - ); - }); - - it.each([ - ["Pixel 6", "36", "google_apis_playstore", "Play Store"], - ["Pixel 5", "35", "tablet", "tablet"], - ])( - "says the model does not take the kind of image it was given (%s)", - async (model, osVersion, imageTag, noun) => { - const { refusal } = await fitHarness([[osVersion, imageTag, "arm64-v8a"]]); - - expect((await refusal(model, osVersion)).message).toBe( - `No installed system image fits ${model} at Android ${osVersion}: ${model} does not take ${noun} images`, - ); - }, - ); - - it("says none of the installed images fit it when no single check rules out every image", async () => { - const { refusal } = await fitHarness([ - ["35", "google_apis_playstore", "arm64-v8a"], - ["35", "android-tv", "arm64-v8a"], - ]); - - expect((await refusal("Pixel 6", "35")).message).toBe( - "No installed system image fits Pixel 6 at Android 35: none of the installed images fit it", - ); - }); - - it("gives the API range reason for an image that fails both API range and kind", async () => { - const { refusal } = await fitHarness([["34", "android-tv", "arm64-v8a"]]); - - expect((await refusal("Pixel 9", "34")).message).toBe( - "No installed system image fits Pixel 9 at Android 34: it needs Android 35 or newer", - ); - }); - - it("drops the level from the message when no version was asked, reading every installed image", async () => { - const { refusal } = await fitHarness([["36", "google_apis", "arm64-v8a"]]); - - expect((await refusal("Television (1080p)")).message).toBe( - "No installed system image fits Television (1080p): it needs a TV image", - ); - }); - - it("says an image tag's image does not fit it, and why", async () => { - const { refusal } = await fitHarness(HOST_IMAGES); - - expect((await refusal("Television (1080p)", undefined, "google_apis")).message).toBe( - "No installed system image fits Television (1080p): the google_apis image does not fit it: it needs a TV image", - ); - expect((await refusal("Television (1080p)", "35", "google_apis")).message).toBe( - "No installed system image fits Television (1080p) at Android 35: the google_apis image does not fit it: it needs a TV image", - ); - }); - - it("says the level is not installed for a level with nothing installed", async () => { - const { refusal } = await fitHarness(HOST_IMAGES); - - expect((await refusal("Pixel 9", "37")).message).toBe( - "No installed system image fits Pixel 9 at Android 37: Android 37 is not installed", - ); - }); - - it("says no system image is installed on a host with no image at all", async () => { - const { refusal } = await fitHarness([]); - - expect((await refusal("Pixel 9")).message).toBe( - "No installed system image fits Pixel 9: no system image is installed", - ); - }); - - it("keeps today's message for an image tag with no image of that tag installed", async () => { - const { refusal } = await fitHarness(HOST_IMAGES); - - expect((await refusal("Pixel 5", "35", "default")).message).toBe( - "No default system image is installed for Android API 35", - ); - }); - }); - - describe("an image tag whose image misfits", () => { - it("fails RUNTIME_MISSING, not downloadable, and starts no sdkmanager", async () => { - const { refusal, runner } = await fitHarness(HOST_IMAGES); - - const failure = await refusal("Pixel 6", undefined, "google_apis_playstore"); - - expect(failure).toMatchObject({ downloadable: false, name: "RuntimeMissingError" }); - expect(failure.message).toContain("the google_apis_playstore image does not fit it"); - expect(runner.calls.map((call) => call.command)).not.toContain(binaries.sdkmanager); - }); - }); - - describe("custom profile fit from devices.xml", () => { - const wearImage: FitImage = [ - "34", - "android-wear", - "arm64-v8a", - "SystemImage.TagId=android-wear,google_apis_playstore\n", - ]; - - it("a custom profile listed by avdmanager gets the API range of its devices.xml entry: api-level 31- drops levels below 31 from its modelRuntimes", async () => { - const images: readonly FitImage[] = [ - ["30", "google_apis", "arm64-v8a"], - ["31", "google_apis", "arm64-v8a"], - ["30", "android-tv", "arm64-v8a"], - ["31", "android-tv", "arm64-v8a"], - ]; - const open = await fitHarness(images, { - devicesXml: customFitXml({ name: "Open Phone" }), - listing: userListing(["Open Phone"]), - }); - expect((await open.part.listCatalog()).modelRuntimes["Open Phone"]).toEqual(["30", "31"]); - const { part } = await fitHarness(images, { - devicesXml: - customFitXml({ apiLevel: "31-", name: "My Phone" }) + - customFitXml({ apiLevel: "31-", name: "My TV", tagId: "android-tv" }), - listing: userListing(["My Phone"], ["My TV", "android-tv"]), - }); - - const { modelRuntimes } = await part.listCatalog(); - - expect(modelRuntimes["My Phone"]).toEqual(["31"]); - expect(modelRuntimes["My TV"]).toEqual(["31"]); - }); - - it("a custom profile with playstore-enabled false drops Play Store images", async () => { - const images: readonly FitImage[] = [ - ["34", "google_apis_playstore", "arm64-v8a"], - ["35", "google_apis", "arm64-v8a"], - ]; - const allowed = await fitHarness(images, { - devicesXml: customFitXml({ name: "My Phone", playStore: "true" }), - listing: userListing(["My Phone"]), - }); - expect((await allowed.part.listCatalog()).modelRuntimes["My Phone"]).toEqual(["34", "35"]); - const { part } = await fitHarness(images, { - devicesXml: customFitXml({ name: "My Phone", playStore: "false" }), - listing: userListing(["My Phone"]), - }); - - expect((await part.listCatalog()).modelRuntimes["My Phone"]).toEqual(["35"]); - }); - - it("a custom Wear profile with playstore-enabled false keeps an image tagged android-wear,google_apis_playstore, listed by avdmanager or devices.xml-only", async () => { - const wear = customFitXml({ name: "My Watch", playStore: "false", tagId: "android-wear" }); - const listed = await fitHarness([wearImage], { - devicesXml: wear, - listing: userListing(["My Watch", "android-wear"]), - }); - const only = await fitHarness([wearImage], { devicesXml: wear, listing: userListing() }); - - expect((await listed.part.listCatalog()).modelRuntimes["My Watch"]).toEqual(["34"]); - expect((await only.part.listCatalog()).modelRuntimes["My Watch"]).toEqual(["34"]); - }); - - it("a custom profile with no type and a 10.1-inch screen that does not fold fits tablet images; a 6-inch one, one with folded dimensions and one with no diagonal do not", async () => { - const images: readonly FitImage[] = [ - ["35", "google_apis", "arm64-v8a"], - ["36", "tablet", "arm64-v8a"], - ]; - const { part } = await fitHarness(images, { - devicesXml: - customFitXml({ diagonal: "10.1", name: "Big" }) + - customFitXml({ diagonal: "6", name: "Small" }) + - customFitXml({ diagonal: "10.1", folded: true, name: "Folds" }) + - customFitXml({ name: "NoDiagonal" }), - listing: userListing(["Big"], ["Small"], ["Folds"], ["NoDiagonal"]), - }); - - const { modelRuntimes } = await part.listCatalog(); - - expect(modelRuntimes["Big"]).toEqual(["35", "36"]); - expect(modelRuntimes["Small"]).toEqual(["35"]); - expect(modelRuntimes["Folds"]).toEqual(["35"]); - expect(modelRuntimes["NoDiagonal"]).toEqual(["35"]); - }); - - it("a custom profile with nothing but a name pairs as after #489", async () => { - const { part } = await fitHarness(HOST_IMAGES, { - devicesXml: customFitXml({ name: "Old" }), - listing: userListing(["Old"]), - }); - - expect((await part.listCatalog()).modelRuntimes["Old"]).toEqual(["35", "36"]); - }); - - it("a custom profile's class still comes from avdmanager's Tag line", async () => { - const { part } = await fitHarness(HOST_IMAGES, { - devicesXml: customFitXml({ name: "My TV", tagId: "android-wear" }), - listing: userListing(["My TV", "android-tv"]), - }); - - expect((await part.listCatalog()).modelClasses["My TV"]).toBe("tv"); - }); - - it("a devices.xml profile avdmanager does not list fits by its own tag-id, range, Play Store flag and screen, and is classed by its tag-id", async () => { - const images: readonly FitImage[] = [ - ["30", "android-tv", "arm64-v8a"], - ["31", "android-tv", "arm64-v8a"], - ["30", "google_apis", "arm64-v8a"], - ["31", "google_apis_playstore", "arm64-v8a"], - ["32", "google_apis", "arm64-v8a"], - ["33", "tablet", "arm64-v8a"], - ]; - const { part } = await fitHarness(images, { - devicesXml: - customFitXml({ apiLevel: "31-", name: "Only TV", tagId: "android-tv" }) + - customFitXml({ name: "Only Phone", playStore: "false" }) + - customFitXml({ diagonal: "10.1", name: "Only Tablet" }), - listing: userListing(), - }); - - const catalog = await part.listCatalog(); - - expect(catalog.modelRuntimes["Only TV"]).toEqual(["31"]); - expect(catalog.modelRuntimes["Only Phone"]).toEqual(["30", "32"]); - expect(catalog.modelRuntimes["Only Tablet"]).toEqual(["30", "31", "32", "33"]); - expect(catalog.modelClasses["Only TV"]).toBe("tv"); - expect(catalog.modelClasses["Only Phone"]).toBe("phone"); - }); - - it("reads devices.xml from ANDROID_USER_HOME, then ANDROID_PREFS_ROOT, then ANDROID_SDK_HOME, then home", async () => { - const paths = [ - "/user-home/devices.xml", - "/prefs/.android/devices.xml", - "/sdk-home/.android/devices.xml", - `${home}/.android/devices.xml`, - ]; - const env = { - ANDROID_PREFS_ROOT: "/prefs", - ANDROID_SDK_HOME: "/sdk-home", - ANDROID_USER_HOME: "/user-home", - }; - const names = ["From user home", "From prefs root", "From sdk home", "From home"]; - const outcomes: string[][] = []; - for (let first = 0; first < paths.length; first++) { - const used: Record = {}; - if (first <= 0) used["ANDROID_USER_HOME"] = env.ANDROID_USER_HOME; - if (first <= 1) used["ANDROID_PREFS_ROOT"] = env.ANDROID_PREFS_ROOT; - if (first <= 2) used["ANDROID_SDK_HOME"] = env.ANDROID_SDK_HOME; - const { part, filesystem } = await fitHarness(HOST_IMAGES, { - extraEnv: used, - listing: userListing(), - }); - for (const [index, path] of paths.entries()) { - await filesystem.mkdirp(path.slice(0, path.lastIndexOf("/"))); - await filesystem.writeFileAtomic( - path, - `${customFitXml({ name: names[index] as string })}`, - ); - } - outcomes.push( - (await part.listCatalog()).models.filter((model) => model.startsWith("From ")), - ); - } - - expect(outcomes).toEqual(names.map((name) => [name])); - }); - }); }); diff --git a/src/drivers/android/test-catalog.ts b/src/drivers/android/test-catalog.ts new file mode 100644 index 000000000..f6e830c02 --- /dev/null +++ b/src/drivers/android/test-catalog.ts @@ -0,0 +1,144 @@ +import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import type { ProcessHandle, ProcessRunner, ProcessRunOptions } from "../../ports/index.js"; + +import { Catalog } from "./catalog.js"; +import type { + DeviceProfileSource, + DeviceProfileSourceDiagnostic, +} from "./device-profile-source.js"; +import { SystemImages } from "./system-images.js"; +import { type LibraryDeviceProfile } from "./profiles/index.js"; +import { + ANDROID_HOME_DIRECTORY as home, + ANDROID_SDK_ROOT as sdk, + fitDevices, + type FitImage, + processResult, + type Refusal, + writeDevicesXml, +} from "./testing.js"; + +export const avdmanager = `${sdk}/cmdline-tools/latest/bin/avdmanager`; +export const binaries = { + avdmanager, + sdkmanager: `${sdk}/cmdline-tools/latest/bin/sdkmanager`, +} as const; + +export async function androidFilesystem( + options: { readonly images?: readonly (readonly [string, string, string])[] } = {}, + filesystem: MemoryFilesystem = new MemoryFilesystem(), +): Promise { + for (const [api, tag, abi] of options.images ?? [["34", "google_apis", "arm64-v8a"]]) { + await filesystem.mkdirp(`${sdk}/system-images/android-${api}/${tag}/${abi}`); + } + return filesystem; +} + +export function createPart( + filesystem: MemoryFilesystem, + processRunner: ProcessRunner, + options: { + readonly avdmanager?: string; + readonly deviceProfileSources?: readonly DeviceProfileSource[]; + readonly extraEnv?: Readonly>; + readonly hostAbi?: string; + readonly onDiagnostic?: (diagnostic: DeviceProfileSourceDiagnostic) => void; + } = {}, +): Catalog { + return new Catalog({ + avdmanager: options.avdmanager ?? avdmanager, + avdmanagerEnv: () => ({ ANDROID_HOME: sdk }), + deviceProfileSources: options.deviceProfileSources, + env: { ANDROID_HOME: sdk, ...options.extraEnv }, + filesystem, + homeDirectory: home, + onDiagnostic: options.onDiagnostic, + processRunner, + sdkRoot: sdk, + systemImages: new SystemImages({ + acceptAndroidLicenses: false, + clock: new FakeClock(), + deviceRoot: `${home}/.simlock/devices/android`, + filesystem, + hostAbi: options.hostAbi ?? "arm64-v8a", + legacyAvdHome: `${home}/.android/avd`, + processRunner, + sdk: { + adb: `${sdk}/platform-tools/adb`, + emulator: `${sdk}/emulator/emulator`, + root: sdk, + sdkmanager: `${sdk}/cmdline-tools/latest/bin/sdkmanager`, + }, + }), + }); +} + +// why: `avdmanager list device` answers with the listing a test names however often it is asked, so a +// second read is counted rather than refused. +class ListingRunner extends ScriptedProcessRunner { + constructor(private readonly listing: string) { + super([]); + } + + override spawn( + command: string, + args: readonly string[], + options: ProcessRunOptions = {}, + ): ProcessHandle { + if (command === binaries.avdmanager && args[0] === "list") { + return new ScriptedProcessRunner([processResult(command, args, this.listing)]).spawn( + command, + args, + options, + ); + } + return super.spawn(command, args, options); + } +} + +export async function fitHarness( + images: readonly FitImage[], + options: { + readonly devicesXml?: string; + readonly extraEnv?: Readonly>; + readonly hostAbi?: string; + readonly listing?: string; + readonly profiles?: readonly LibraryDeviceProfile[]; + } = {}, +) { + const filesystem = await androidFilesystem({ + images: images.map(([api, tag, abi]) => [api, tag, abi]), + }); + for (const [api, tag, abi, properties] of images) { + if (properties !== undefined) { + await filesystem.writeFileAtomic( + `${sdk}/system-images/android-${api}/${tag}/${abi}/source.properties`, + properties, + ); + } + } + if (options.devicesXml !== undefined) await writeDevicesXml(filesystem, options.devicesXml); + const runner = new ListingRunner(options.listing ?? fitDevices); + const part = createPart(filesystem, runner, { + ...(options.extraEnv === undefined ? {} : { extraEnv: options.extraEnv }), + hostAbi: options.hostAbi ?? "arm64-v8a", + ...(options.profiles === undefined + ? {} + : { deviceProfileSources: [{ profiles: async () => options.profiles ?? [] }] }), + }); + const resolve = (model: string, osVersion?: string, imageTag?: string) => + part.resolveSpec({ + model, + platform: "android", + ...(osVersion === undefined ? {} : { osVersion }), + ...(imageTag === undefined ? {} : { imageTag }), + }); + const refusal = (model: string, osVersion?: string, imageTag?: string) => + resolve(model, osVersion, imageTag).then( + () => { + throw new Error(`${model} ${osVersion ?? ""} ${imageTag ?? ""} resolved`); + }, + (error: unknown) => error as Refusal, + ); + return { filesystem, part, refusal, resolve, runner }; +} From db4a5bbef099caa64bc9ae356c5fd03713ef17f4 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:32:00 +0100 Subject: [PATCH 60/83] test(drivers): split seeds.test.ts by topic --- src/drivers/android/seeds-copies.test.ts | 547 +++++++++ .../android/seeds-library-and-fit.test.ts | 336 ++++++ src/drivers/android/seeds.test.ts | 1041 +---------------- src/drivers/android/test-seeds.ts | 198 ++++ 4 files changed, 1083 insertions(+), 1039 deletions(-) create mode 100644 src/drivers/android/seeds-copies.test.ts create mode 100644 src/drivers/android/seeds-library-and-fit.test.ts create mode 100644 src/drivers/android/test-seeds.ts diff --git a/src/drivers/android/seeds-copies.test.ts b/src/drivers/android/seeds-copies.test.ts new file mode 100644 index 000000000..2c11ce0d6 --- /dev/null +++ b/src/drivers/android/seeds-copies.test.ts @@ -0,0 +1,547 @@ +import { describe, expect, it, vi } from "vitest"; + +import { FakeClock, MemoryFilesystem, type ScriptedProcessExpectation } from "../../ports/index.js"; + +import { createPart, sources, seedHome, seedHarness } from "./test-seeds.js"; +import { + ANDROID_SDK_ROOT as sdk, + avdDirectory, + binaries, + bootingAt, + bootProbes, + EmulatorExitingRunner, + makingASeedFrom, + makingTheSeed, + pixelDevices, + processResult, + seedEmulatorLine, + seedStages, + writeSeedFiles, +} from "./testing.js"; + +const copyHome = `${avdDirectory}/simlock_one`; + +/** why: the daemon after a restart: a new part over the same device folders and the same source table. */ +function reopened( + harness: Awaited>, + expectations: readonly ScriptedProcessExpectation[], + options: { readonly clock?: FakeClock; readonly seedLaunchInputs?: readonly string[] } = {}, +) { + const runner = new EmulatorExitingRunner([...expectations]); + const part = createPart(harness.filesystem, runner, { + ...options, + database: harness.database, + ids: ["four", "five", "six"], + }); + return { ...part, runner }; +} + +class SizeFailureFilesystem extends MemoryFilesystem { + override directorySize(): Promise { + return Promise.reject(new Error("EIO")); + } +} + +class CopyFailureFilesystem extends MemoryFilesystem { + readonly #message: string; + + constructor(message: string) { + super(); + this.#message = message; + } + + override copyTree(): Promise { + return Promise.reject(new Error(this.#message)); + } +} + +// why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build +// `Seeds` directly over a memory filesystem and a scripted runner, with the real parts it takes. +describe("Seeds", () => { + describe("seeds and copies", () => { + it("makes one seed when two plain leases of one kind are provisioned at once", async () => { + const madeHome = `${sources}/seed_three`; + const harness = await seedHarness([ + ...makingTheSeed(madeHome), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + await writeSeedFiles(harness.filesystem, madeHome); + + const [first, second] = await Promise.all([ + harness.seeds.provision(harness.spec), + harness.seeds.provision(harness.spec), + ]); + + expect(harness.runner.calls.filter((call) => call.args[0] === "create")).toHaveLength(1); + expect([first.deviceId, second.deviceId]).toEqual(["simlock_one", "simlock_two"]); + await expect( + harness.filesystem.exists(`${avdDirectory}/simlock_two/simlock.avd`), + ).resolves.toBe(true); + const listed = await harness.seeds.listSources(); + expect(listed.filter((source) => source.orphan !== true)).toHaveLength(1); + }); + + it("tells why a seed is stale, cut to 300 characters, and clears the reason when the image and emulator match again", async () => { + const imagePath = `${sdk}/system-images/android-34/google_apis/arm64-v8a`; + const packageName = "system-images;android-34;google_apis;arm64-v8a"; + const revision = (number: string) => `Pkg.Revision=${number}\n`; + const original = "Android emulator version 36.1.9"; + const cases = [ + { + change: async (filesystem: MemoryFilesystem) => + filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("2")), + emulatorVersion: original, + mend: async (filesystem: MemoryFilesystem) => + filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")), + reason: `system image ${packageName} changed from revision 1 to 2`, + }, + { + change: async (filesystem: MemoryFilesystem) => filesystem.rm(imagePath), + emulatorVersion: original, + mend: async (filesystem: MemoryFilesystem) => { + await filesystem.mkdirp(imagePath); + await filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")); + }, + reason: `system image ${packageName} is not installed`, + }, + { + emulatorVersion: "Android emulator version 36.2.0", + reason: `emulator changed from ${original} to Android emulator version 36.2.0`, + }, + { + emulatorVersion: "v".repeat(400), + reason: `emulator changed from ${original} to ${"v".repeat(400)}`.slice(0, 300), + }, + { + emulatorVersion: original, + reason: "android.emulator launch settings changed", + seedLaunchInputs: ["headless=true", "gpu=auto"], + }, + ] as const; + + for (const stale of cases) { + const harness = await seedHarness(makingTheSeed()); + await harness.filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")); + await harness.seeds.provision(harness.spec); + if ("change" in stale) await stale.change(harness.filesystem); + + const changed = await reopened( + harness, + [processResult(binaries.emulator, ["-version"], stale.emulatorVersion)], + "seedLaunchInputs" in stale ? { seedLaunchInputs: stale.seedLaunchInputs } : {}, + ); + await expect(changed.seeds.listSources()).resolves.toMatchObject([ + { id: "seed_two", staleReason: stale.reason }, + ]); + + if ("mend" in stale) await stale.mend(harness.filesystem); + const mended = await reopened(harness, [ + processResult(binaries.emulator, ["-version"], original), + ]); + const [source] = await mended.seeds.listSources(); + expect(source).toMatchObject({ id: "seed_two" }); + expect(source).not.toHaveProperty("staleReason"); + } + }); + + it("destroys a stale seed and makes it again at the next provision", async () => { + const harness = await seedHarness(makingTheSeed()); + await harness.seeds.provision(harness.spec); + const newHome = `${sources}/seed_five`; + await writeSeedFiles(harness.filesystem, newHome); + const newer = "Android emulator version 36.2.0"; + const later = await reopened(harness, [ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + processResult(binaries.emulator, ["-version"], newer), + ...makingTheSeed(newHome, newer), + processResult(binaries.emulator, ["-version"], newer), + ]); + + await later.seeds.provision(harness.spec); + + expect(await harness.filesystem.exists(seedHome)).toBe(false); + expect(later.runner.calls.filter((call) => call.args[0] === "create")).toHaveLength(1); + await expect(later.seeds.listSources()).resolves.toMatchObject([{ id: "seed_five" }]); + }); + + it("removes a seed's folder and row and throws when making it fails at any step", async () => { + const stage = seedStages(); + const serial = "emulator-5586"; + const neverReady = [ + stage.devices, + bootingAt(seedHome, ["-no-snapshot-load"], true)[0], + processResult( + binaries.adb, + ["-s", serial, "shell", "getprop", "sys.boot_completed"], + "0\n", + ), + ] as ScriptedProcessExpectation[]; + const stuckOff = [ + stage.devices, + ...bootingAt(seedHome, ["-no-snapshot-load"], true), + stage.sync, + stage.powerOff, + stage.probe, + stage.kill, + ]; + const cases = [ + { + expectations: [ + stage.version, + { ...stage.create, result: { code: 1, stderr: "no such package", stdout: "" } }, + ], + message: "no such package", + }, + { + advance: 300_000, + expectations: [stage.version, stage.create, ...neverReady], + message: "Timed out waiting for device to boot", + }, + { + advance: 60_000, + expectations: [stage.version, stage.create, ...stuckOff], + killsEmulator: true, + message: "Android seed seed_two did not power off within 60000ms", + }, + { + expectations: [ + stage.version, + stage.create, + stage.devices, + ...stage.firstBoot, + stage.sync, + stage.powerOff, + ...bootingAt(seedHome, ["-no-snapshot-load"], true), + { ...stage.save, result: { code: 1, stderr: "emulator gone", stdout: "" } }, + ], + message: "Android seed seed_two: boot snapshot simlock_boot was not saved", + }, + { + expectations: [ + stage.version, + stage.create, + stage.devices, + ...stage.firstBoot, + stage.sync, + stage.powerOff, + ...bootingAt(seedHome, ["-no-snapshot-load"], true), + stage.save, + processResult(binaries.adb, ["-s", serial, "emu", "avd", "snapshot", "list"], "\n"), + stage.probe, + stage.kill, + ], + message: "Android seed seed_two: boot snapshot simlock_boot was not saved", + }, + ] as const; + + for (const failing of cases) { + const clock = new FakeClock(); + const harness = await seedHarness(failing.expectations, { clock }); + // why: the stuck seed's emulator ignores `reboot -p`; only the 60 s deadline ends the wait. + harness.runner.ignoresEmuKill = "killsEmulator" in failing; + const provisioning = harness.seeds.provision(harness.spec); + const outcome = expect(provisioning).rejects.toThrow(failing.message); + if ("advance" in failing) { + await vi.waitFor(() => + expect( + harness.runner.calls.some( + (call) => call.args.includes("sys.boot_completed") || call.args.includes("reboot"), + ), + ).toBe(true), + ); + clock.advance(failing.advance); + } + await outcome; + + if ("killsEmulator" in failing) { + expect( + harness.runner.calls.some((call) => call.args.includes("kill")), + "the seed's emulator was asked to quit by its path", + ).toBe(true); + } + const emulators = harness.runner.calls.flatMap((call, index) => + call.args.includes("-avd") ? [index] : [], + ); + for (const emulator of emulators) { + let exited = false; + void harness.runner.handles[emulator]?.wait().then(() => { + exited = true; + }); + await vi.waitFor(() => expect(exited, "the seed's emulator was killed").toBe(true)); + } + expect(await harness.filesystem.exists(seedHome)).toBe(false); + await expect(harness.seeds.listSources()).resolves.toEqual([]); + expect(harness.database.prepare("SELECT 1 FROM android_source_data").all()).toEqual([]); + } + }); + + it("removes the device's folder and throws when copying the seed fails", async () => { + const harness = await seedHarness(makingTheSeed(), { + filesystem: new CopyFailureFilesystem("cp: no space left on device"), + }); + + await expect(harness.seeds.provision(harness.spec)).rejects.toThrow( + "copying Android seed seed_two to simlock_one failed: cp: no space left on device", + ); + + expect(await harness.filesystem.exists(copyHome)).toBe(false); + expect(await harness.filesystem.exists(seedHome)).toBe(true); + }); + + it("provisions a kind with a ready seed while another kind's seed is being made, and leaves that kind's making row and folder alone", async () => { + const makingHome = `${sources}/seed_device-4`; + const harness = await seedHarness([ + ...makingTheSeed(), + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + // why: the other kind's make never gets past its first boot: its emulator never says it booted. + ...makingASeedFrom(makingHome).slice(0, 3), + { + hangs: true, + match: { + args: ["-avd", "simlock", ...seedEmulatorLine, "-no-snapshot-load"], + command: binaries.emulator, + }, + }, + { + hangs: true, + match: { + args: ["-s", "emulator-5586", "shell", "getprop", "sys.boot_completed"], + command: binaries.adb, + }, + }, + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + await harness.seeds.provision(harness.spec); + await harness.filesystem.mkdirp( + `${sdk}/system-images/android-34/google_apis_playstore/arm64-v8a`, + ); + await writeSeedFiles(harness.filesystem, makingHome); + const otherKind = await harness.catalog.resolveSpec({ + imageTag: "google_apis_playstore", + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + void harness.seeds.provision(otherKind).catch(() => undefined); + await vi.waitFor(() => expect(bootProbes(harness.runner)).toBeGreaterThan(2)); + + let settled = false; + const provisioning = harness.seeds.provision(harness.spec).finally(() => { + settled = true; + }); + await vi.waitFor(() => + expect(settled, "the kind with a ready seed waited for the other kind's make").toBe(true), + ); + const second = await provisioning; + + expect(second.driverData).toMatchObject({ avdHome: expect.stringContaining("/simlock_") }); + expect( + harness.database.prepare("SELECT state FROM android_source_data ORDER BY state").all(), + ).toEqual([{ state: "making" }, { state: "ready" }]); + expect(await harness.filesystem.exists(makingHome)).toBe(true); + + // why: after a restart that row is another process's leftover, which only its own kind's make or a listing sweeps. + const restarted = await reopened(harness, [ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ]); + await restarted.seeds.provision( + await restarted.catalog.resolveSpec({ + model: "Pixel 5", + osVersion: "34", + platform: "android", + }), + ); + expect( + harness.database.prepare("SELECT state FROM android_source_data ORDER BY state").all(), + ).toEqual([{ state: "making" }, { state: "ready" }]); + expect(await harness.filesystem.exists(makingHome)).toBe(true); + }); + + it("lists each ready seed with its size, reports a sources folder no row names as an orphan, and destroys a source by id after stopping its emulator by path", async () => { + const serial = "emulator-5586"; + const orphanHome = `${sources}/seed_orphan`; + const harness = await seedHarness([ + ...makingTheSeed(), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + processResult(binaries.adb, ["devices"], `List of devices attached\n${serial}\tdevice\n`), + seedStages().probe, + seedStages().kill, + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ]); + await harness.seeds.provision(harness.spec); + await writeSeedFiles(harness.filesystem, orphanHome); + + await expect(harness.seeds.listSources()).resolves.toEqual([ + { + id: "seed_two", + imageTag: "google_apis", + kind: "seed", + model: "Pixel 5", + osVersion: "34", + sizeBytes: 16, + }, + { id: "seed_orphan", kind: "seed", model: "unknown", orphan: true, osVersion: "unknown" }, + ]); + expect(await harness.filesystem.exists(orphanHome)).toBe(true); + + const callsBefore = harness.runner.calls.length; + await harness.seeds.destroySource("seed_orphan"); + await harness.seeds.destroySource("seed_two"); + + expect(await harness.filesystem.exists(`${sources}/seed_two`)).toBe(false); + expect(await harness.filesystem.exists(orphanHome)).toBe(false); + expect( + harness.runner.calls + .slice(callsBefore) + .filter((call) => call.args.includes("kill")) + .map((call) => call.args), + "the running emulator of the source is found by its path and stopped before its folder goes", + ).toEqual([["-s", serial, "emu", "kill"]]); + await expect(harness.seeds.destroySource("seed_two")).rejects.toThrow( + "Android keeps no source seed_two", + ); + }); + + it("lists no size for a seed whose size cannot be counted", async () => { + const filesystem = new SizeFailureFilesystem(); + const harness = await seedHarness( + [ + ...makingTheSeed(), + processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), + ], + { filesystem }, + ); + await harness.seeds.provision(harness.spec); + + const [seed] = await harness.seeds.listSources(); + + expect(seed).toMatchObject({ id: "seed_two" }); + expect(seed).not.toHaveProperty("sizeBytes"); + }); + + it("sweeps a making row an earlier process left, stopping its emulator by path, and one this process has been making past the larger of 900 s and twice android.bootTimeoutMs plus 600 s, and leaves any other this process is making", async () => { + const serial = "emulator-5586"; + const clock = new FakeClock(); + const stage = seedStages(); + const harness = await seedHarness( + [ + stage.version, + stage.create, + stage.devices, + stage.firstBoot[0] as ScriptedProcessExpectation, + processResult( + binaries.adb, + ["-s", serial, "shell", "getprop", "sys.boot_completed"], + "0\n", + ), + ], + { clock }, + ); + const making = harness.seeds.provision(harness.spec); + const failed = expect(making).rejects.toThrow("Timed out waiting for device to boot"); + await vi.waitFor(() => + expect(harness.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( + true, + ), + ); + + await expect(harness.seeds.listSources()).resolves.toEqual([]); + expect(await harness.filesystem.exists(seedHome)).toBe(true); + + const restarted = await reopened(harness, [ + processResult(binaries.adb, ["devices"], `List of devices attached\n${serial}\tdevice\n`), + stage.probe, + stage.kill, + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ]); + await expect(restarted.seeds.listSources()).resolves.toEqual([]); + expect(await harness.filesystem.exists(seedHome)).toBe(false); + clock.advance(300_000); + await failed; + + const slowClock = new FakeClock(); + const slow = await seedHarness( + [ + stage.version, + stage.create, + stage.devices, + stage.firstBoot[0] as ScriptedProcessExpectation, + processResult( + binaries.adb, + ["-s", serial, "shell", "getprop", "sys.boot_completed"], + "0\n", + ), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ], + { bootTimeoutMs: 600_000, clock: slowClock }, + ); + const slowMaking = slow.seeds.provision(slow.spec); + const slowFailed = expect(slowMaking).rejects.toThrow(); + await vi.waitFor(() => + expect(slow.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( + true, + ), + ); + const ageTo = (ageMs: number) => + slow.database + .prepare("UPDATE android_source_data SET created_at = ?") + .run(slowClock.now() - ageMs); + + ageTo(1_000_000); + await slow.seeds.listSources(); + expect(await slow.filesystem.exists(seedHome)).toBe(true); + + ageTo(1_800_000); + await slow.seeds.listSources(); + expect(await slow.filesystem.exists(seedHome)).toBe(true); + + ageTo(1_800_001); + await slow.seeds.listSources(); + expect(await slow.filesystem.exists(seedHome)).toBe(false); + slowClock.advance(600_000); + await slowFailed; + + const quickClock = new FakeClock(); + const quick = await seedHarness( + [ + stage.version, + stage.create, + stage.devices, + stage.firstBoot[0] as ScriptedProcessExpectation, + processResult( + binaries.adb, + ["-s", serial, "shell", "getprop", "sys.boot_completed"], + "0\n", + ), + processResult(binaries.adb, ["devices"], "List of devices attached\n"), + ], + { bootTimeoutMs: 60_000, clock: quickClock }, + ); + const quickMaking = quick.seeds.provision(quick.spec); + const quickFailed = expect(quickMaking).rejects.toThrow(); + await vi.waitFor(() => + expect(quick.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( + true, + ), + ); + const quickAgeTo = (ageMs: number) => + quick.database + .prepare("UPDATE android_source_data SET created_at = ?") + .run(quickClock.now() - ageMs); + + // why: twice 60 s plus 600 s is 720 s, under the 900 s floor, which then decides. + quickAgeTo(900_000); + await quick.seeds.listSources(); + expect(await quick.filesystem.exists(seedHome)).toBe(true); + + quickAgeTo(900_001); + await quick.seeds.listSources(); + expect(await quick.filesystem.exists(seedHome)).toBe(false); + quickClock.advance(60_000); + await quickFailed; + }); + }); +}); diff --git a/src/drivers/android/seeds-library-and-fit.test.ts b/src/drivers/android/seeds-library-and-fit.test.ts new file mode 100644 index 000000000..3ea664225 --- /dev/null +++ b/src/drivers/android/seeds-library-and-fit.test.ts @@ -0,0 +1,336 @@ +import { describe, expect, it } from "vitest"; + +import { MemoryFilesystem } from "../../ports/index.js"; + +import type { DeviceProfileSource } from "./device-profile-source.js"; +import { LIBRARY_PROFILES, type LibraryDeviceProfile } from "./profiles/index.js"; +import { buildPart, libraryHarness } from "./test-seeds.js"; +import { + ANDROID_SDK_ROOT as sdk, + avdDirectory, + fitHarness as fitHarnessWith, + HOST_IMAGES, + pixelDevices, + seedConfig, + testLibraryProfile, + twoPixelDevices, +} from "./testing.js"; + +const fitHarness = ( + ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never +) => fitHarnessWith(buildPart, ...args); + +function libraryPixel9(): LibraryDeviceProfile { + const profile = LIBRARY_PROFILES.find((candidate) => candidate.id === "pixel_9"); + if (profile === undefined) throw new Error("the library has no profile with id pixel_9"); + return profile; +} + +class WriteFailureFilesystem extends MemoryFilesystem { + armed = false; + + constructor( + private readonly failingPath: string, + private readonly error: Error, + ) { + super(); + } + + override async writeFileAtomic( + ...args: Parameters + ): Promise { + if (this.armed && args[0] === this.failingPath) { + throw this.error; + } + return super.writeFileAtomic(...args); + } +} + +// why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build +// `Seeds` directly over a memory filesystem and a scripted runner, with the real parts it takes. +describe("Seeds", () => { + describe("library device profiles", () => { + it("provisions a library device with every `hardware` value of the profile in `config.ini`", async () => { + const harness = await libraryHarness({ listing: pixelDevices }); + + await harness.provision("Pixel 9"); + + const config = await harness.config(); + const hardware = Object.entries(libraryPixel9().hardware); + expect(hardware.length).toBeGreaterThan(100); + expect(hardware.filter(([key, value]) => config.get(key) !== value)).toEqual([]); + }); + + it("provisions identical `hw.*` and `vm.heapSize` lines whether `avdmanager list device` lists `pixel_9` or not", async () => { + const without = await libraryHarness({ listing: pixelDevices }); + const withListed = await libraryHarness({ + listing: twoPixelDevices, + seed: seedConfig(["hw.gpu.mode=host", "hw.lcd.density=480", "hw.ramSize=8G"]), + }); + + await without.provision("Pixel 9"); + await withListed.provision("Pixel 9"); + + const hardwareLines = async (harness: typeof without) => + (await harness.lines()).filter((line) => /^(hw\.|vm\.heapSize=)/.test(line)).sort(); + const expected = await hardwareLines(without); + expect(expected.length).toBeGreaterThan(100); + expect(await hardwareLines(withListed)).toEqual(expected); + }); + + it("removes every `hw.*` (except `hw.cpu.arch`), `vm.heapSize` and `skin.*` line the seed wrote that the profile does not set", async () => { + const harness = await libraryHarness({ + profiles: [testLibraryProfile({ hardware: { "hw.lcd.height": "100" } })], + seed: seedConfig(["hw.seedOnly=yes", "skin.dynamic=yes"]), + }); + + await harness.provision("Test Phone"); + + const lines = await harness.lines(); + expect(lines.filter((line) => line.startsWith("skin."))).toEqual([]); + expect(lines.filter((line) => line.startsWith("vm.heapSize"))).toEqual([]); + expect(lines.filter((line) => line.startsWith("hw.")).sort()).toEqual([ + "hw.cpu.arch=arm64", + "hw.lcd.height=100", + ]); + }); + + it("keeps `image.sysdir.1`, `tag.id`, `abi.type`, `target`, `hw.cpu.arch`, `disk.dataPartition.size` and `showDeviceFrame` as `avdmanager` wrote them", async () => { + const harness = await libraryHarness({ listing: pixelDevices }); + + await harness.provision("Pixel 9"); + + const config = await harness.config(); + expect( + [ + "image.sysdir.1", + "tag.id", + "abi.type", + "target", + "hw.cpu.arch", + "disk.dataPartition.size", + "showDeviceFrame", + ].map((key) => `${key}=${config.get(key)}`), + ).toEqual([ + "image.sysdir.1=system-images/android-35/google_apis/arm64-v8a/", + "tag.id=google_apis", + "abi.type=arm64-v8a", + "target=android-35", + "hw.cpu.arch=arm64", + "disk.dataPartition.size=6G", + "showDeviceFrame=yes", + ]); + }); + + it("sets `PlayStore.enabled=yes` on a Play Store image when the profile allows Play Store, and `no` on a `google_apis` image", async () => { + const playStore = { imageTag: "google_apis_playstore" }; + const allowed = await libraryHarness({ ...playStore, listing: pixelDevices }); + await allowed.provision("Pixel 9"); + expect((await allowed.config()).get("PlayStore.enabled")).toBe("yes"); + + const googleApis = await libraryHarness({ + listing: pixelDevices, + seed: seedConfig(["PlayStore.enabled=yes"]), + }); + await googleApis.provision("Pixel 9"); + expect((await googleApis.config()).get("PlayStore.enabled")).toBe("no"); + }); + + it("reads an image's tags from `SystemImage.TagId`, and from the folder name when it has none", async () => { + const playStoreOf = async (imageTag: string, imageProperties: string) => { + const harness = await libraryHarness({ imageProperties, imageTag, listing: pixelDevices }); + await harness.provision("Pixel 9"); + return (await harness.config()).get("PlayStore.enabled"); + }; + + expect(await playStoreOf("custom-folder", "SystemImage.TagId=google_apis_playstore\n")).toBe( + "yes", + ); + expect(await playStoreOf("google_apis_playstore", "")).toBe("yes"); + expect(await playStoreOf("google_apis_playstore", "Pkg.Revision=1\n")).toBe("yes"); + expect(await playStoreOf("google_apis_playstore", "SystemImage.TagId=google_apis\n")).toBe( + "no", + ); + }); + + it("writes the profile's boot properties to `boot.prop`, and removes a seed's `boot.prop` when the profile has none", async () => { + const withProperties = await libraryHarness({ + profiles: [ + testLibraryProfile({ bootProperties: { "persist.x": "y", "ro.simlock.test": "1" } }), + ], + seedBootProp: "ro.seed=1\n", + }); + await withProperties.provision("Test Phone"); + const written = await withProperties.filesystem.readFile(withProperties.bootPropPath); + expect( + written + .split("\n") + .filter((line) => line !== "") + .sort(), + ).toEqual(["persist.x=y", "ro.simlock.test=1"]); + + const without = await libraryHarness({ listing: pixelDevices, seedBootProp: "ro.seed=1\n" }); + await expect(without.filesystem.exists(without.bootPropPath)).resolves.toBe(true); + await without.provision("Pixel 9"); + await expect(without.filesystem.exists(without.bootPropPath)).resolves.toBe(false); + }); + + it.each([ + ["a hardware value", "\r"], + ["a hardware value", "\n"], + ["a hardware value", "\u0000"], + ["a hardware key", "\r"], + ["a hardware key", "\n"], + ["a hardware key", "\u0000"], + ["a boot property value", "\r"], + ["a boot property value", "\n"], + ["a boot property value", "\u0000"], + ] as const)( + "refuses %s with %j before `avdmanager create avd` runs", + async (where, character) => { + const profileWith = (inserted: string) => + testLibraryProfile( + where === "a hardware value" + ? { hardware: { "hw.ramSize": `2${inserted}G` } } + : where === "a hardware key" + ? { hardware: { [`hw.bad${inserted}key`]: "1" } } + : { bootProperties: { "ro.x": `a${inserted}b` } }, + ); + const accepted = await libraryHarness({ profiles: [profileWith("-")] }); + await accepted.provision("Test Phone"); + expect(accepted.createCalls()).toHaveLength(1); + + const refused = await libraryHarness({ profiles: [profileWith(character)] }); + + await expect(refused.provision("Test Phone")).rejects.toThrow(); + expect(refused.createCalls()).toEqual([]); + }, + ); + + it("refuses NUL in a `devices.xml` value, through the one shared check", async () => { + const profileNamed = (name: string): DeviceProfileSource => ({ + async profiles() { + return [ + { + hardwareProperties: { "hw.device.name": name }, + kind: "properties" as const, + name: "Custom A", + names: ["Custom A"], + }, + ]; + }, + }); + const accepted = await libraryHarness({ sources: [profileNamed("Custom A")] }); + await accepted.provision("Custom A"); + expect((await accepted.config()).get("hw.device.name")).toBe("Custom A"); + + const refused = await libraryHarness({ sources: [profileNamed("Custom\u0000A")] }); + + await expect(refused.provision("Custom A")).rejects.toThrow(/Refusing to merge/); + }); + + it("rethrows and runs no `avdmanager delete` when a write after `avdmanager create avd` fails", async () => { + const filesystem = new WriteFailureFilesystem( + `${avdDirectory}/sources/seed_two/simlock.avd/config.ini`, + new Error("disk full"), + ); + const harness = await libraryHarness({ filesystem, listing: pixelDevices }); + filesystem.armed = true; + + await expect(harness.provision("Pixel 9")).rejects.toThrow("disk full"); + + expect(harness.createCalls()).toHaveLength(1); + expect(harness.runner.calls.some((call) => call.args.includes("delete"))).toBe(false); + }); + + it("seeds `avdmanager create avd` with the first profile `avdmanager list device` prints, also when it lists `pixel_9`", async () => { + const listsPixel9 = await libraryHarness({ listing: twoPixelDevices }); + await listsPixel9.provision("Pixel 9"); + + const lacksPixel9 = await libraryHarness({ listing: pixelDevices }); + await lacksPixel9.provision("Pixel 9"); + + expect(listsPixel9.seedDeviceId()).toBe("pixel_5"); + expect(lacksPixel9.seedDeviceId()).toBe("pixel_5"); + }); + }); + + describe("image fit", () => { + it("creates a phone from the Play Store image at the newest level, and the TV from the android-tv image", async () => { + const phone = await fitHarness(HOST_IMAGES); + await phone.driver.provision(await phone.resolve("Pixel 5")); + const tv = await fitHarness(HOST_IMAGES); + await tv.driver.provision(await tv.resolve("Television (720p)")); + + expect(phone.createdPackage()).toBe( + "system-images;android-36;google_apis_playstore;arm64-v8a", + ); + expect(tv.createdPackage()).toBe("system-images;android-34;android-tv;arm64-v8a"); + }); + + it("creates the device from a fitting image, never a misfit one that sorts first", async () => { + const { createdPackage, driver, resolve } = await fitHarness( + [ + ["34", "google_apis", "arm64-v8a"], + ["34", "google-tv", "arm64-v8a"], + ["34", "android-tv", "x86_64"], + ], + { hostAbi: "arm64-v8a" }, + ); + + await driver.provision(await resolve("Television (720p)")); + + expect(createdPackage()).toBe("system-images;android-34;google-tv;arm64-v8a"); + }); + + it.each([ + [ + "host-native google_apis", + [ + ["35", "default", "arm64-v8a"], + ["35", "google_apis", "x86_64"], + ["35", "google_apis", "arm64-v8a"], + ] as const, + "system-images;android-35;google_apis;arm64-v8a", + ], + [ + "host-native", + [ + ["35", "google_apis", "x86_64"], + ["35", "default", "arm64-v8a"], + ] as const, + "system-images;android-35;default;arm64-v8a", + ], + [ + "google_apis", + [ + ["35", "default", "x86_64"], + ["35", "google_apis", "x86_64"], + ] as const, + "system-images;android-35;google_apis;x86_64", + ], + ["any", [["35", "default", "x86_64"]] as const, "system-images;android-35;default;x86_64"], + ])("follows today's order among fitting images: %s first", async (_name, images, expected) => { + const { createdPackage, driver, resolve } = await fitHarness(images); + + await driver.provision(await resolve("Pixel 5", "35")); + + expect(createdPackage()).toBe(expected); + }); + + it("throws RUNTIME_MISSING when no fitting image is left, and creates nothing from a misfit one", async () => { + const { driver, filesystem, resolve, runner } = await fitHarness([ + ["34", "google_apis", "arm64-v8a"], + ["34", "android-tv", "arm64-v8a"], + ]); + const spec = await resolve("Television (720p)"); + await filesystem.rm(`${sdk}/system-images/android-34/android-tv`); + + await expect(driver.provision(spec)).rejects.toMatchObject({ + downloadable: false, + name: "RuntimeMissingError", + }); + expect(runner.calls.some((call) => call.args[0] === "create")).toBe(false); + }); + }); +}); diff --git a/src/drivers/android/seeds.test.ts b/src/drivers/android/seeds.test.ts index f30317d71..5cd0462f2 100644 --- a/src/drivers/android/seeds.test.ts +++ b/src/drivers/android/seeds.test.ts @@ -1,284 +1,19 @@ -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it } from "vitest"; +import { buildPart, libraryHarness, seedHarness } from "./test-seeds.js"; import { - FakeClock, - FakeTcpProbe, - MemoryFilesystem, - type ProcessRunner, - type ScriptedProcessExpectation, -} from "../../ports/index.js"; -import { AdbTools } from "./adb-tools.js"; -import { AvdConfig } from "./avd-config.js"; -import { Catalog } from "./catalog.js"; -import { ConsolePorts, portAllocatorFor } from "./console-ports.js"; -import type { DeviceProfileSource } from "./device-profile-source.js"; -import { DeviceTable } from "./device-table.js"; -import { Emulator } from "./emulator.js"; -import { Ownership } from "./ownership.js"; -import { LIBRARY_PROFILES, type LibraryDeviceProfile } from "./profiles/index.js"; -import { Seeds } from "./seeds.js"; -import { SystemImages } from "./system-images.js"; -import { - ANDROID_HOME_DIRECTORY as home, ANDROID_SDK_ROOT as sdk, - androidFilesystem, - avdDirectory, - binaries, - bootingAt, - bootProbes, - type BuildProvisioner, createdFromPackage, customDeviceXml, customFitXml, - EmulatorExitingRunner, - fitHarness as fitHarnessWith, - HOST_IMAGES, - libraryHarness as libraryHarnessWith, - makingASeedFrom, makingTheSeed, - memoryAndroidSourceDatabase, - pixelDevices, - processResult, - seedConfig, - seedEmulatorLine, - seedStages, taggedHarness as taggedHarnessWith, - testLibraryProfile, - twoPixelDevices, - writeSeedFiles, } from "./testing.js"; -const adbServerPort = 5038; - -/** why: the part and the catalog that resolves its specs, built the way the driver builds them over one scripted runner. */ -function createPart( - filesystem: MemoryFilesystem, - processRunner: ProcessRunner, - options: { - readonly bootTimeoutMs?: number; - readonly clock?: FakeClock; - readonly database: Awaited>; - readonly deviceProfileSources?: readonly DeviceProfileSource[]; - readonly extraEnv?: Readonly>; - readonly hostAbi?: string; - readonly ids?: readonly string[]; - readonly seedLaunchInputs?: readonly string[]; - }, -) { - let nextId = 0; - const clock = options.clock ?? new FakeClock(); - const env = { ANDROID_HOME: sdk, ...options.extraEnv }; - const adbTools = new AdbTools({ - adb: binaries.adb, - adbServerPort, - baseEnv: env, - clock, - deviceRoot: avdDirectory, - emulator: binaries.emulator, - processRunner, - }); - const idGenerator = { generate: () => options.ids?.[nextId++] ?? `device-${nextId}` }; - const ownership = new Ownership({ adb: binaries.adb, adbTools, filesystem }); - const avdConfig = new AvdConfig({ - adb: binaries.adb, - adbTools, - filesystem, - idGenerator, - seedLaunchInputs: options.seedLaunchInputs ?? [], - }); - const emulator = new Emulator({ - adb: binaries.adb, - adbServerPort, - adbTools, - clock, - deviceRoot: avdDirectory, - emulator: binaries.emulator, - emulatorLaunch: undefined, - filesystem, - onDiagnostic: undefined, - ownership, - processRunner, - bootTimeoutMs: options.bootTimeoutMs ?? 300_000, - tcpProbe: new FakeTcpProbe([adbServerPort]), - }); - const systemImages = new SystemImages({ - acceptAndroidLicenses: false, - clock, - deviceRoot: avdDirectory, - filesystem, - hostAbi: options.hostAbi ?? "arm64-v8a", - legacyAvdHome: `${home}/.android/avd`, - processRunner, - sdk: { - adb: binaries.adb, - emulator: binaries.emulator, - root: sdk, - sdkmanager: binaries.sdkmanager, - }, - }); - const catalog = new Catalog({ - avdmanager: binaries.avdmanager, - avdmanagerEnv: () => adbTools.env(), - deviceProfileSources: options.deviceProfileSources, - env, - filesystem, - homeDirectory: home, - onDiagnostic: undefined, - processRunner, - sdkRoot: sdk, - systemImages, - }); - const seeds = new Seeds({ - adb: binaries.adb, - adbTools, - avdConfig, - avdmanager: binaries.avdmanager, - bootTimeoutMs: options.bootTimeoutMs ?? 300_000, - catalog, - clock, - consolePorts: new ConsolePorts({ - adbTools, - allocator: portAllocatorFor(processRunner, binaries.adb), - tcpProbe: new FakeTcpProbe([adbServerPort]), - }), - database: options.database, - deviceRoot: avdDirectory, - deviceTable: new DeviceTable({ - adbTools, - avdConfig, - avdmanager: binaries.avdmanager, - baseEnv: env, - deviceRoot: avdDirectory, - filesystem, - legacyAvdHome: `${home}/.android/avd`, - ownership, - register: async () => undefined, - }), - emulator, - filesystem, - idGenerator, - ownership, - processRunner, - systemImages, - }); - return { catalog, seeds }; -} - -const buildPart: BuildProvisioner = async (filesystem, processRunner, options) => { - const { catalog, seeds } = createPart(filesystem, processRunner, { - ...options, - database: await memoryAndroidSourceDatabase(), - }); - return { - listCatalog: () => catalog.listCatalog(), - provision: (spec) => seeds.provision(spec), - resolveSpec: (request) => catalog.resolveSpec(request), - }; -}; -const fitHarness = ( - ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never -) => fitHarnessWith(buildPart, ...args); -const libraryHarness = (options?: Parameters[1]) => - libraryHarnessWith(buildPart, options); const taggedHarness = ( ...args: Parameters extends [unknown, ...infer Rest] ? Rest : never ) => taggedHarnessWith(buildPart, ...args); -const sources = `${avdDirectory}/sources`; -const copyHome = `${avdDirectory}/simlock_one`; -const seedHome = `${sources}/seed_two`; - -async function seedHarness( - expectations: readonly ScriptedProcessExpectation[], - options: { - readonly bootTimeoutMs?: number; - readonly clock?: FakeClock; - readonly config?: string; - readonly filesystem?: MemoryFilesystem; - readonly ids?: readonly string[]; - } = {}, -) { - const filesystem = await androidFilesystem({}, options.filesystem); - await writeSeedFiles(filesystem, seedHome, options.config); - const database = await memoryAndroidSourceDatabase(); - const runner = new EmulatorExitingRunner([ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - ...expectations, - ]); - const part = createPart(filesystem, runner, { - ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), - ...(options.clock === undefined ? {} : { clock: options.clock }), - database, - ids: options.ids ?? ["one", "two", "three"], - }); - const spec = await part.catalog.resolveSpec({ - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - return { database, filesystem, runner, spec, ...part }; -} - -/** why: the daemon after a restart: a new part over the same device folders and the same source table. */ -function reopened( - harness: Awaited>, - expectations: readonly ScriptedProcessExpectation[], - options: { readonly clock?: FakeClock; readonly seedLaunchInputs?: readonly string[] } = {}, -) { - const runner = new EmulatorExitingRunner([...expectations]); - const part = createPart(harness.filesystem, runner, { - ...options, - database: harness.database, - ids: ["four", "five", "six"], - }); - return { ...part, runner }; -} - -function libraryPixel9(): LibraryDeviceProfile { - const profile = LIBRARY_PROFILES.find((candidate) => candidate.id === "pixel_9"); - if (profile === undefined) throw new Error("the library has no profile with id pixel_9"); - return profile; -} - -class SizeFailureFilesystem extends MemoryFilesystem { - override directorySize(): Promise { - return Promise.reject(new Error("EIO")); - } -} - -class CopyFailureFilesystem extends MemoryFilesystem { - readonly #message: string; - - constructor(message: string) { - super(); - this.#message = message; - } - - override copyTree(): Promise { - return Promise.reject(new Error(this.#message)); - } -} - -class WriteFailureFilesystem extends MemoryFilesystem { - armed = false; - - constructor( - private readonly failingPath: string, - private readonly error: Error, - ) { - super(); - } - - override async writeFileAtomic( - ...args: Parameters - ): Promise { - if (this.armed && args[0] === this.failingPath) { - throw this.error; - } - return super.writeFileAtomic(...args); - } -} - // why: a part of a decomposed module (ADR 0063) is tested through its own surface: these build // `Seeds` directly over a memory filesystem and a scripted runner, with the real parts it takes. describe("Seeds", () => { @@ -456,776 +191,4 @@ describe("Seeds", () => { expect(await keysOf(rich)).toEqual(plainKeys); }); }); - - describe("library device profiles", () => { - it("provisions a library device with every `hardware` value of the profile in `config.ini`", async () => { - const harness = await libraryHarness({ listing: pixelDevices }); - - await harness.provision("Pixel 9"); - - const config = await harness.config(); - const hardware = Object.entries(libraryPixel9().hardware); - expect(hardware.length).toBeGreaterThan(100); - expect(hardware.filter(([key, value]) => config.get(key) !== value)).toEqual([]); - }); - - it("provisions identical `hw.*` and `vm.heapSize` lines whether `avdmanager list device` lists `pixel_9` or not", async () => { - const without = await libraryHarness({ listing: pixelDevices }); - const withListed = await libraryHarness({ - listing: twoPixelDevices, - seed: seedConfig(["hw.gpu.mode=host", "hw.lcd.density=480", "hw.ramSize=8G"]), - }); - - await without.provision("Pixel 9"); - await withListed.provision("Pixel 9"); - - const hardwareLines = async (harness: typeof without) => - (await harness.lines()).filter((line) => /^(hw\.|vm\.heapSize=)/.test(line)).sort(); - const expected = await hardwareLines(without); - expect(expected.length).toBeGreaterThan(100); - expect(await hardwareLines(withListed)).toEqual(expected); - }); - - it("removes every `hw.*` (except `hw.cpu.arch`), `vm.heapSize` and `skin.*` line the seed wrote that the profile does not set", async () => { - const harness = await libraryHarness({ - profiles: [testLibraryProfile({ hardware: { "hw.lcd.height": "100" } })], - seed: seedConfig(["hw.seedOnly=yes", "skin.dynamic=yes"]), - }); - - await harness.provision("Test Phone"); - - const lines = await harness.lines(); - expect(lines.filter((line) => line.startsWith("skin."))).toEqual([]); - expect(lines.filter((line) => line.startsWith("vm.heapSize"))).toEqual([]); - expect(lines.filter((line) => line.startsWith("hw.")).sort()).toEqual([ - "hw.cpu.arch=arm64", - "hw.lcd.height=100", - ]); - }); - - it("keeps `image.sysdir.1`, `tag.id`, `abi.type`, `target`, `hw.cpu.arch`, `disk.dataPartition.size` and `showDeviceFrame` as `avdmanager` wrote them", async () => { - const harness = await libraryHarness({ listing: pixelDevices }); - - await harness.provision("Pixel 9"); - - const config = await harness.config(); - expect( - [ - "image.sysdir.1", - "tag.id", - "abi.type", - "target", - "hw.cpu.arch", - "disk.dataPartition.size", - "showDeviceFrame", - ].map((key) => `${key}=${config.get(key)}`), - ).toEqual([ - "image.sysdir.1=system-images/android-35/google_apis/arm64-v8a/", - "tag.id=google_apis", - "abi.type=arm64-v8a", - "target=android-35", - "hw.cpu.arch=arm64", - "disk.dataPartition.size=6G", - "showDeviceFrame=yes", - ]); - }); - - it("sets `PlayStore.enabled=yes` on a Play Store image when the profile allows Play Store, and `no` on a `google_apis` image", async () => { - const playStore = { imageTag: "google_apis_playstore" }; - const allowed = await libraryHarness({ ...playStore, listing: pixelDevices }); - await allowed.provision("Pixel 9"); - expect((await allowed.config()).get("PlayStore.enabled")).toBe("yes"); - - const googleApis = await libraryHarness({ - listing: pixelDevices, - seed: seedConfig(["PlayStore.enabled=yes"]), - }); - await googleApis.provision("Pixel 9"); - expect((await googleApis.config()).get("PlayStore.enabled")).toBe("no"); - }); - - it("reads an image's tags from `SystemImage.TagId`, and from the folder name when it has none", async () => { - const playStoreOf = async (imageTag: string, imageProperties: string) => { - const harness = await libraryHarness({ imageProperties, imageTag, listing: pixelDevices }); - await harness.provision("Pixel 9"); - return (await harness.config()).get("PlayStore.enabled"); - }; - - expect(await playStoreOf("custom-folder", "SystemImage.TagId=google_apis_playstore\n")).toBe( - "yes", - ); - expect(await playStoreOf("google_apis_playstore", "")).toBe("yes"); - expect(await playStoreOf("google_apis_playstore", "Pkg.Revision=1\n")).toBe("yes"); - expect(await playStoreOf("google_apis_playstore", "SystemImage.TagId=google_apis\n")).toBe( - "no", - ); - }); - - it("writes the profile's boot properties to `boot.prop`, and removes a seed's `boot.prop` when the profile has none", async () => { - const withProperties = await libraryHarness({ - profiles: [ - testLibraryProfile({ bootProperties: { "persist.x": "y", "ro.simlock.test": "1" } }), - ], - seedBootProp: "ro.seed=1\n", - }); - await withProperties.provision("Test Phone"); - const written = await withProperties.filesystem.readFile(withProperties.bootPropPath); - expect( - written - .split("\n") - .filter((line) => line !== "") - .sort(), - ).toEqual(["persist.x=y", "ro.simlock.test=1"]); - - const without = await libraryHarness({ listing: pixelDevices, seedBootProp: "ro.seed=1\n" }); - await expect(without.filesystem.exists(without.bootPropPath)).resolves.toBe(true); - await without.provision("Pixel 9"); - await expect(without.filesystem.exists(without.bootPropPath)).resolves.toBe(false); - }); - - it.each([ - ["a hardware value", "\r"], - ["a hardware value", "\n"], - ["a hardware value", "\u0000"], - ["a hardware key", "\r"], - ["a hardware key", "\n"], - ["a hardware key", "\u0000"], - ["a boot property value", "\r"], - ["a boot property value", "\n"], - ["a boot property value", "\u0000"], - ] as const)( - "refuses %s with %j before `avdmanager create avd` runs", - async (where, character) => { - const profileWith = (inserted: string) => - testLibraryProfile( - where === "a hardware value" - ? { hardware: { "hw.ramSize": `2${inserted}G` } } - : where === "a hardware key" - ? { hardware: { [`hw.bad${inserted}key`]: "1" } } - : { bootProperties: { "ro.x": `a${inserted}b` } }, - ); - const accepted = await libraryHarness({ profiles: [profileWith("-")] }); - await accepted.provision("Test Phone"); - expect(accepted.createCalls()).toHaveLength(1); - - const refused = await libraryHarness({ profiles: [profileWith(character)] }); - - await expect(refused.provision("Test Phone")).rejects.toThrow(); - expect(refused.createCalls()).toEqual([]); - }, - ); - - it("refuses NUL in a `devices.xml` value, through the one shared check", async () => { - const profileNamed = (name: string): DeviceProfileSource => ({ - async profiles() { - return [ - { - hardwareProperties: { "hw.device.name": name }, - kind: "properties" as const, - name: "Custom A", - names: ["Custom A"], - }, - ]; - }, - }); - const accepted = await libraryHarness({ sources: [profileNamed("Custom A")] }); - await accepted.provision("Custom A"); - expect((await accepted.config()).get("hw.device.name")).toBe("Custom A"); - - const refused = await libraryHarness({ sources: [profileNamed("Custom\u0000A")] }); - - await expect(refused.provision("Custom A")).rejects.toThrow(/Refusing to merge/); - }); - - it("rethrows and runs no `avdmanager delete` when a write after `avdmanager create avd` fails", async () => { - const filesystem = new WriteFailureFilesystem( - `${avdDirectory}/sources/seed_two/simlock.avd/config.ini`, - new Error("disk full"), - ); - const harness = await libraryHarness({ filesystem, listing: pixelDevices }); - filesystem.armed = true; - - await expect(harness.provision("Pixel 9")).rejects.toThrow("disk full"); - - expect(harness.createCalls()).toHaveLength(1); - expect(harness.runner.calls.some((call) => call.args.includes("delete"))).toBe(false); - }); - - it("seeds `avdmanager create avd` with the first profile `avdmanager list device` prints, also when it lists `pixel_9`", async () => { - const listsPixel9 = await libraryHarness({ listing: twoPixelDevices }); - await listsPixel9.provision("Pixel 9"); - - const lacksPixel9 = await libraryHarness({ listing: pixelDevices }); - await lacksPixel9.provision("Pixel 9"); - - expect(listsPixel9.seedDeviceId()).toBe("pixel_5"); - expect(lacksPixel9.seedDeviceId()).toBe("pixel_5"); - }); - }); - - describe("image fit", () => { - it("creates a phone from the Play Store image at the newest level, and the TV from the android-tv image", async () => { - const phone = await fitHarness(HOST_IMAGES); - await phone.driver.provision(await phone.resolve("Pixel 5")); - const tv = await fitHarness(HOST_IMAGES); - await tv.driver.provision(await tv.resolve("Television (720p)")); - - expect(phone.createdPackage()).toBe( - "system-images;android-36;google_apis_playstore;arm64-v8a", - ); - expect(tv.createdPackage()).toBe("system-images;android-34;android-tv;arm64-v8a"); - }); - - it("creates the device from a fitting image, never a misfit one that sorts first", async () => { - const { createdPackage, driver, resolve } = await fitHarness( - [ - ["34", "google_apis", "arm64-v8a"], - ["34", "google-tv", "arm64-v8a"], - ["34", "android-tv", "x86_64"], - ], - { hostAbi: "arm64-v8a" }, - ); - - await driver.provision(await resolve("Television (720p)")); - - expect(createdPackage()).toBe("system-images;android-34;google-tv;arm64-v8a"); - }); - - it.each([ - [ - "host-native google_apis", - [ - ["35", "default", "arm64-v8a"], - ["35", "google_apis", "x86_64"], - ["35", "google_apis", "arm64-v8a"], - ] as const, - "system-images;android-35;google_apis;arm64-v8a", - ], - [ - "host-native", - [ - ["35", "google_apis", "x86_64"], - ["35", "default", "arm64-v8a"], - ] as const, - "system-images;android-35;default;arm64-v8a", - ], - [ - "google_apis", - [ - ["35", "default", "x86_64"], - ["35", "google_apis", "x86_64"], - ] as const, - "system-images;android-35;google_apis;x86_64", - ], - ["any", [["35", "default", "x86_64"]] as const, "system-images;android-35;default;x86_64"], - ])("follows today's order among fitting images: %s first", async (_name, images, expected) => { - const { createdPackage, driver, resolve } = await fitHarness(images); - - await driver.provision(await resolve("Pixel 5", "35")); - - expect(createdPackage()).toBe(expected); - }); - - it("throws RUNTIME_MISSING when no fitting image is left, and creates nothing from a misfit one", async () => { - const { driver, filesystem, resolve, runner } = await fitHarness([ - ["34", "google_apis", "arm64-v8a"], - ["34", "android-tv", "arm64-v8a"], - ]); - const spec = await resolve("Television (720p)"); - await filesystem.rm(`${sdk}/system-images/android-34/android-tv`); - - await expect(driver.provision(spec)).rejects.toMatchObject({ - downloadable: false, - name: "RuntimeMissingError", - }); - expect(runner.calls.some((call) => call.args[0] === "create")).toBe(false); - }); - }); - - describe("seeds and copies", () => { - it("makes one seed when two plain leases of one kind are provisioned at once", async () => { - const madeHome = `${sources}/seed_three`; - const harness = await seedHarness([ - ...makingTheSeed(madeHome), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - await writeSeedFiles(harness.filesystem, madeHome); - - const [first, second] = await Promise.all([ - harness.seeds.provision(harness.spec), - harness.seeds.provision(harness.spec), - ]); - - expect(harness.runner.calls.filter((call) => call.args[0] === "create")).toHaveLength(1); - expect([first.deviceId, second.deviceId]).toEqual(["simlock_one", "simlock_two"]); - await expect( - harness.filesystem.exists(`${avdDirectory}/simlock_two/simlock.avd`), - ).resolves.toBe(true); - const listed = await harness.seeds.listSources(); - expect(listed.filter((source) => source.orphan !== true)).toHaveLength(1); - }); - - it("tells why a seed is stale, cut to 300 characters, and clears the reason when the image and emulator match again", async () => { - const imagePath = `${sdk}/system-images/android-34/google_apis/arm64-v8a`; - const packageName = "system-images;android-34;google_apis;arm64-v8a"; - const revision = (number: string) => `Pkg.Revision=${number}\n`; - const original = "Android emulator version 36.1.9"; - const cases = [ - { - change: async (filesystem: MemoryFilesystem) => - filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("2")), - emulatorVersion: original, - mend: async (filesystem: MemoryFilesystem) => - filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")), - reason: `system image ${packageName} changed from revision 1 to 2`, - }, - { - change: async (filesystem: MemoryFilesystem) => filesystem.rm(imagePath), - emulatorVersion: original, - mend: async (filesystem: MemoryFilesystem) => { - await filesystem.mkdirp(imagePath); - await filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")); - }, - reason: `system image ${packageName} is not installed`, - }, - { - emulatorVersion: "Android emulator version 36.2.0", - reason: `emulator changed from ${original} to Android emulator version 36.2.0`, - }, - { - emulatorVersion: "v".repeat(400), - reason: `emulator changed from ${original} to ${"v".repeat(400)}`.slice(0, 300), - }, - { - emulatorVersion: original, - reason: "android.emulator launch settings changed", - seedLaunchInputs: ["headless=true", "gpu=auto"], - }, - ] as const; - - for (const stale of cases) { - const harness = await seedHarness(makingTheSeed()); - await harness.filesystem.writeFileAtomic(`${imagePath}/source.properties`, revision("1")); - await harness.seeds.provision(harness.spec); - if ("change" in stale) await stale.change(harness.filesystem); - - const changed = await reopened( - harness, - [processResult(binaries.emulator, ["-version"], stale.emulatorVersion)], - "seedLaunchInputs" in stale ? { seedLaunchInputs: stale.seedLaunchInputs } : {}, - ); - await expect(changed.seeds.listSources()).resolves.toMatchObject([ - { id: "seed_two", staleReason: stale.reason }, - ]); - - if ("mend" in stale) await stale.mend(harness.filesystem); - const mended = await reopened(harness, [ - processResult(binaries.emulator, ["-version"], original), - ]); - const [source] = await mended.seeds.listSources(); - expect(source).toMatchObject({ id: "seed_two" }); - expect(source).not.toHaveProperty("staleReason"); - } - }); - - it("destroys a stale seed and makes it again at the next provision", async () => { - const harness = await seedHarness(makingTheSeed()); - await harness.seeds.provision(harness.spec); - const newHome = `${sources}/seed_five`; - await writeSeedFiles(harness.filesystem, newHome); - const newer = "Android emulator version 36.2.0"; - const later = await reopened(harness, [ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - processResult(binaries.emulator, ["-version"], newer), - ...makingTheSeed(newHome, newer), - processResult(binaries.emulator, ["-version"], newer), - ]); - - await later.seeds.provision(harness.spec); - - expect(await harness.filesystem.exists(seedHome)).toBe(false); - expect(later.runner.calls.filter((call) => call.args[0] === "create")).toHaveLength(1); - await expect(later.seeds.listSources()).resolves.toMatchObject([{ id: "seed_five" }]); - }); - - it("removes a seed's folder and row and throws when making it fails at any step", async () => { - const stage = seedStages(); - const serial = "emulator-5586"; - const neverReady = [ - stage.devices, - bootingAt(seedHome, ["-no-snapshot-load"], true)[0], - processResult( - binaries.adb, - ["-s", serial, "shell", "getprop", "sys.boot_completed"], - "0\n", - ), - ] as ScriptedProcessExpectation[]; - const stuckOff = [ - stage.devices, - ...bootingAt(seedHome, ["-no-snapshot-load"], true), - stage.sync, - stage.powerOff, - stage.probe, - stage.kill, - ]; - const cases = [ - { - expectations: [ - stage.version, - { ...stage.create, result: { code: 1, stderr: "no such package", stdout: "" } }, - ], - message: "no such package", - }, - { - advance: 300_000, - expectations: [stage.version, stage.create, ...neverReady], - message: "Timed out waiting for device to boot", - }, - { - advance: 60_000, - expectations: [stage.version, stage.create, ...stuckOff], - killsEmulator: true, - message: "Android seed seed_two did not power off within 60000ms", - }, - { - expectations: [ - stage.version, - stage.create, - stage.devices, - ...stage.firstBoot, - stage.sync, - stage.powerOff, - ...bootingAt(seedHome, ["-no-snapshot-load"], true), - { ...stage.save, result: { code: 1, stderr: "emulator gone", stdout: "" } }, - ], - message: "Android seed seed_two: boot snapshot simlock_boot was not saved", - }, - { - expectations: [ - stage.version, - stage.create, - stage.devices, - ...stage.firstBoot, - stage.sync, - stage.powerOff, - ...bootingAt(seedHome, ["-no-snapshot-load"], true), - stage.save, - processResult(binaries.adb, ["-s", serial, "emu", "avd", "snapshot", "list"], "\n"), - stage.probe, - stage.kill, - ], - message: "Android seed seed_two: boot snapshot simlock_boot was not saved", - }, - ] as const; - - for (const failing of cases) { - const clock = new FakeClock(); - const harness = await seedHarness(failing.expectations, { clock }); - // why: the stuck seed's emulator ignores `reboot -p`; only the 60 s deadline ends the wait. - harness.runner.ignoresEmuKill = "killsEmulator" in failing; - const provisioning = harness.seeds.provision(harness.spec); - const outcome = expect(provisioning).rejects.toThrow(failing.message); - if ("advance" in failing) { - await vi.waitFor(() => - expect( - harness.runner.calls.some( - (call) => call.args.includes("sys.boot_completed") || call.args.includes("reboot"), - ), - ).toBe(true), - ); - clock.advance(failing.advance); - } - await outcome; - - if ("killsEmulator" in failing) { - expect( - harness.runner.calls.some((call) => call.args.includes("kill")), - "the seed's emulator was asked to quit by its path", - ).toBe(true); - } - const emulators = harness.runner.calls.flatMap((call, index) => - call.args.includes("-avd") ? [index] : [], - ); - for (const emulator of emulators) { - let exited = false; - void harness.runner.handles[emulator]?.wait().then(() => { - exited = true; - }); - await vi.waitFor(() => expect(exited, "the seed's emulator was killed").toBe(true)); - } - expect(await harness.filesystem.exists(seedHome)).toBe(false); - await expect(harness.seeds.listSources()).resolves.toEqual([]); - expect(harness.database.prepare("SELECT 1 FROM android_source_data").all()).toEqual([]); - } - }); - - it("removes the device's folder and throws when copying the seed fails", async () => { - const harness = await seedHarness(makingTheSeed(), { - filesystem: new CopyFailureFilesystem("cp: no space left on device"), - }); - - await expect(harness.seeds.provision(harness.spec)).rejects.toThrow( - "copying Android seed seed_two to simlock_one failed: cp: no space left on device", - ); - - expect(await harness.filesystem.exists(copyHome)).toBe(false); - expect(await harness.filesystem.exists(seedHome)).toBe(true); - }); - - it("provisions a kind with a ready seed while another kind's seed is being made, and leaves that kind's making row and folder alone", async () => { - const makingHome = `${sources}/seed_device-4`; - const harness = await seedHarness([ - ...makingTheSeed(), - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - // why: the other kind's make never gets past its first boot: its emulator never says it booted. - ...makingASeedFrom(makingHome).slice(0, 3), - { - hangs: true, - match: { - args: ["-avd", "simlock", ...seedEmulatorLine, "-no-snapshot-load"], - command: binaries.emulator, - }, - }, - { - hangs: true, - match: { - args: ["-s", "emulator-5586", "shell", "getprop", "sys.boot_completed"], - command: binaries.adb, - }, - }, - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - await harness.seeds.provision(harness.spec); - await harness.filesystem.mkdirp( - `${sdk}/system-images/android-34/google_apis_playstore/arm64-v8a`, - ); - await writeSeedFiles(harness.filesystem, makingHome); - const otherKind = await harness.catalog.resolveSpec({ - imageTag: "google_apis_playstore", - model: "Pixel 5", - osVersion: "34", - platform: "android", - }); - void harness.seeds.provision(otherKind).catch(() => undefined); - await vi.waitFor(() => expect(bootProbes(harness.runner)).toBeGreaterThan(2)); - - let settled = false; - const provisioning = harness.seeds.provision(harness.spec).finally(() => { - settled = true; - }); - await vi.waitFor(() => - expect(settled, "the kind with a ready seed waited for the other kind's make").toBe(true), - ); - const second = await provisioning; - - expect(second.driverData).toMatchObject({ avdHome: expect.stringContaining("/simlock_") }); - expect( - harness.database.prepare("SELECT state FROM android_source_data ORDER BY state").all(), - ).toEqual([{ state: "making" }, { state: "ready" }]); - expect(await harness.filesystem.exists(makingHome)).toBe(true); - - // why: after a restart that row is another process's leftover, which only its own kind's make or a listing sweeps. - const restarted = await reopened(harness, [ - processResult(binaries.avdmanager, ["list", "device"], pixelDevices), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ]); - await restarted.seeds.provision( - await restarted.catalog.resolveSpec({ - model: "Pixel 5", - osVersion: "34", - platform: "android", - }), - ); - expect( - harness.database.prepare("SELECT state FROM android_source_data ORDER BY state").all(), - ).toEqual([{ state: "making" }, { state: "ready" }]); - expect(await harness.filesystem.exists(makingHome)).toBe(true); - }); - - it("lists each ready seed with its size, reports a sources folder no row names as an orphan, and destroys a source by id after stopping its emulator by path", async () => { - const serial = "emulator-5586"; - const orphanHome = `${sources}/seed_orphan`; - const harness = await seedHarness([ - ...makingTheSeed(), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - processResult(binaries.adb, ["devices"], `List of devices attached\n${serial}\tdevice\n`), - seedStages().probe, - seedStages().kill, - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ]); - await harness.seeds.provision(harness.spec); - await writeSeedFiles(harness.filesystem, orphanHome); - - await expect(harness.seeds.listSources()).resolves.toEqual([ - { - id: "seed_two", - imageTag: "google_apis", - kind: "seed", - model: "Pixel 5", - osVersion: "34", - sizeBytes: 16, - }, - { id: "seed_orphan", kind: "seed", model: "unknown", orphan: true, osVersion: "unknown" }, - ]); - expect(await harness.filesystem.exists(orphanHome)).toBe(true); - - const callsBefore = harness.runner.calls.length; - await harness.seeds.destroySource("seed_orphan"); - await harness.seeds.destroySource("seed_two"); - - expect(await harness.filesystem.exists(`${sources}/seed_two`)).toBe(false); - expect(await harness.filesystem.exists(orphanHome)).toBe(false); - expect( - harness.runner.calls - .slice(callsBefore) - .filter((call) => call.args.includes("kill")) - .map((call) => call.args), - "the running emulator of the source is found by its path and stopped before its folder goes", - ).toEqual([["-s", serial, "emu", "kill"]]); - await expect(harness.seeds.destroySource("seed_two")).rejects.toThrow( - "Android keeps no source seed_two", - ); - }); - - it("lists no size for a seed whose size cannot be counted", async () => { - const filesystem = new SizeFailureFilesystem(); - const harness = await seedHarness( - [ - ...makingTheSeed(), - processResult(binaries.emulator, ["-version"], "Android emulator version 36.1.9"), - ], - { filesystem }, - ); - await harness.seeds.provision(harness.spec); - - const [seed] = await harness.seeds.listSources(); - - expect(seed).toMatchObject({ id: "seed_two" }); - expect(seed).not.toHaveProperty("sizeBytes"); - }); - - it("sweeps a making row an earlier process left, stopping its emulator by path, and one this process has been making past the larger of 900 s and twice android.bootTimeoutMs plus 600 s, and leaves any other this process is making", async () => { - const serial = "emulator-5586"; - const clock = new FakeClock(); - const stage = seedStages(); - const harness = await seedHarness( - [ - stage.version, - stage.create, - stage.devices, - stage.firstBoot[0] as ScriptedProcessExpectation, - processResult( - binaries.adb, - ["-s", serial, "shell", "getprop", "sys.boot_completed"], - "0\n", - ), - ], - { clock }, - ); - const making = harness.seeds.provision(harness.spec); - const failed = expect(making).rejects.toThrow("Timed out waiting for device to boot"); - await vi.waitFor(() => - expect(harness.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( - true, - ), - ); - - await expect(harness.seeds.listSources()).resolves.toEqual([]); - expect(await harness.filesystem.exists(seedHome)).toBe(true); - - const restarted = await reopened(harness, [ - processResult(binaries.adb, ["devices"], `List of devices attached\n${serial}\tdevice\n`), - stage.probe, - stage.kill, - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ]); - await expect(restarted.seeds.listSources()).resolves.toEqual([]); - expect(await harness.filesystem.exists(seedHome)).toBe(false); - clock.advance(300_000); - await failed; - - const slowClock = new FakeClock(); - const slow = await seedHarness( - [ - stage.version, - stage.create, - stage.devices, - stage.firstBoot[0] as ScriptedProcessExpectation, - processResult( - binaries.adb, - ["-s", serial, "shell", "getprop", "sys.boot_completed"], - "0\n", - ), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ], - { bootTimeoutMs: 600_000, clock: slowClock }, - ); - const slowMaking = slow.seeds.provision(slow.spec); - const slowFailed = expect(slowMaking).rejects.toThrow(); - await vi.waitFor(() => - expect(slow.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( - true, - ), - ); - const ageTo = (ageMs: number) => - slow.database - .prepare("UPDATE android_source_data SET created_at = ?") - .run(slowClock.now() - ageMs); - - ageTo(1_000_000); - await slow.seeds.listSources(); - expect(await slow.filesystem.exists(seedHome)).toBe(true); - - ageTo(1_800_000); - await slow.seeds.listSources(); - expect(await slow.filesystem.exists(seedHome)).toBe(true); - - ageTo(1_800_001); - await slow.seeds.listSources(); - expect(await slow.filesystem.exists(seedHome)).toBe(false); - slowClock.advance(600_000); - await slowFailed; - - const quickClock = new FakeClock(); - const quick = await seedHarness( - [ - stage.version, - stage.create, - stage.devices, - stage.firstBoot[0] as ScriptedProcessExpectation, - processResult( - binaries.adb, - ["-s", serial, "shell", "getprop", "sys.boot_completed"], - "0\n", - ), - processResult(binaries.adb, ["devices"], "List of devices attached\n"), - ], - { bootTimeoutMs: 60_000, clock: quickClock }, - ); - const quickMaking = quick.seeds.provision(quick.spec); - const quickFailed = expect(quickMaking).rejects.toThrow(); - await vi.waitFor(() => - expect(quick.runner.calls.some((call) => call.args.includes("sys.boot_completed"))).toBe( - true, - ), - ); - const quickAgeTo = (ageMs: number) => - quick.database - .prepare("UPDATE android_source_data SET created_at = ?") - .run(quickClock.now() - ageMs); - - // why: twice 60 s plus 600 s is 720 s, under the 900 s floor, which then decides. - quickAgeTo(900_000); - await quick.seeds.listSources(); - expect(await quick.filesystem.exists(seedHome)).toBe(true); - - quickAgeTo(900_001); - await quick.seeds.listSources(); - expect(await quick.filesystem.exists(seedHome)).toBe(false); - quickClock.advance(60_000); - await quickFailed; - }); - }); }); diff --git a/src/drivers/android/test-seeds.ts b/src/drivers/android/test-seeds.ts new file mode 100644 index 000000000..e6c2646a8 --- /dev/null +++ b/src/drivers/android/test-seeds.ts @@ -0,0 +1,198 @@ +import { + FakeClock, + FakeTcpProbe, + MemoryFilesystem, + type ProcessRunner, + type ScriptedProcessExpectation, +} from "../../ports/index.js"; + +import { AdbTools } from "./adb-tools.js"; +import { AvdConfig } from "./avd-config.js"; +import { Catalog } from "./catalog.js"; +import { ConsolePorts, portAllocatorFor } from "./console-ports.js"; +import type { DeviceProfileSource } from "./device-profile-source.js"; +import { DeviceTable } from "./device-table.js"; +import { Emulator } from "./emulator.js"; +import { Ownership } from "./ownership.js"; +import { Seeds } from "./seeds.js"; +import { SystemImages } from "./system-images.js"; +import { + ANDROID_HOME_DIRECTORY as home, + ANDROID_SDK_ROOT as sdk, + androidFilesystem, + avdDirectory, + binaries, + type BuildProvisioner, + EmulatorExitingRunner, + libraryHarness as libraryHarnessWith, + memoryAndroidSourceDatabase, + pixelDevices, + processResult, + writeSeedFiles, +} from "./testing.js"; + +const adbServerPort = 5038; + +/** why: the part and the catalog that resolves its specs, built the way the driver builds them over one scripted runner. */ +export function createPart( + filesystem: MemoryFilesystem, + processRunner: ProcessRunner, + options: { + readonly bootTimeoutMs?: number; + readonly clock?: FakeClock; + readonly database: Awaited>; + readonly deviceProfileSources?: readonly DeviceProfileSource[]; + readonly extraEnv?: Readonly>; + readonly hostAbi?: string; + readonly ids?: readonly string[]; + readonly seedLaunchInputs?: readonly string[]; + }, +) { + let nextId = 0; + const clock = options.clock ?? new FakeClock(); + const env = { ANDROID_HOME: sdk, ...options.extraEnv }; + const adbTools = new AdbTools({ + adb: binaries.adb, + adbServerPort, + baseEnv: env, + clock, + deviceRoot: avdDirectory, + emulator: binaries.emulator, + processRunner, + }); + const idGenerator = { generate: () => options.ids?.[nextId++] ?? `device-${nextId}` }; + const ownership = new Ownership({ adb: binaries.adb, adbTools, filesystem }); + const avdConfig = new AvdConfig({ + adb: binaries.adb, + adbTools, + filesystem, + idGenerator, + seedLaunchInputs: options.seedLaunchInputs ?? [], + }); + const emulator = new Emulator({ + adb: binaries.adb, + adbServerPort, + adbTools, + clock, + deviceRoot: avdDirectory, + emulator: binaries.emulator, + emulatorLaunch: undefined, + filesystem, + onDiagnostic: undefined, + ownership, + processRunner, + bootTimeoutMs: options.bootTimeoutMs ?? 300_000, + tcpProbe: new FakeTcpProbe([adbServerPort]), + }); + const systemImages = new SystemImages({ + acceptAndroidLicenses: false, + clock, + deviceRoot: avdDirectory, + filesystem, + hostAbi: options.hostAbi ?? "arm64-v8a", + legacyAvdHome: `${home}/.android/avd`, + processRunner, + sdk: { + adb: binaries.adb, + emulator: binaries.emulator, + root: sdk, + sdkmanager: binaries.sdkmanager, + }, + }); + const catalog = new Catalog({ + avdmanager: binaries.avdmanager, + avdmanagerEnv: () => adbTools.env(), + deviceProfileSources: options.deviceProfileSources, + env, + filesystem, + homeDirectory: home, + onDiagnostic: undefined, + processRunner, + sdkRoot: sdk, + systemImages, + }); + const seeds = new Seeds({ + adb: binaries.adb, + adbTools, + avdConfig, + avdmanager: binaries.avdmanager, + bootTimeoutMs: options.bootTimeoutMs ?? 300_000, + catalog, + clock, + consolePorts: new ConsolePorts({ + adbTools, + allocator: portAllocatorFor(processRunner, binaries.adb), + tcpProbe: new FakeTcpProbe([adbServerPort]), + }), + database: options.database, + deviceRoot: avdDirectory, + deviceTable: new DeviceTable({ + adbTools, + avdConfig, + avdmanager: binaries.avdmanager, + baseEnv: env, + deviceRoot: avdDirectory, + filesystem, + legacyAvdHome: `${home}/.android/avd`, + ownership, + register: async () => undefined, + }), + emulator, + filesystem, + idGenerator, + ownership, + processRunner, + systemImages, + }); + return { catalog, seeds }; +} + +export const buildPart: BuildProvisioner = async (filesystem, processRunner, options) => { + const { catalog, seeds } = createPart(filesystem, processRunner, { + ...options, + database: await memoryAndroidSourceDatabase(), + }); + return { + listCatalog: () => catalog.listCatalog(), + provision: (spec) => seeds.provision(spec), + resolveSpec: (request) => catalog.resolveSpec(request), + }; +}; + +export const libraryHarness = (options?: Parameters[1]) => + libraryHarnessWith(buildPart, options); + +export const sources = `${avdDirectory}/sources`; + +export const seedHome = `${sources}/seed_two`; + +export async function seedHarness( + expectations: readonly ScriptedProcessExpectation[], + options: { + readonly bootTimeoutMs?: number; + readonly clock?: FakeClock; + readonly config?: string; + readonly filesystem?: MemoryFilesystem; + readonly ids?: readonly string[]; + } = {}, +) { + const filesystem = await androidFilesystem({}, options.filesystem); + await writeSeedFiles(filesystem, seedHome, options.config); + const database = await memoryAndroidSourceDatabase(); + const runner = new EmulatorExitingRunner([ + processResult(binaries.avdmanager, ["list", "device"], pixelDevices), + ...expectations, + ]); + const part = createPart(filesystem, runner, { + ...(options.bootTimeoutMs === undefined ? {} : { bootTimeoutMs: options.bootTimeoutMs }), + ...(options.clock === undefined ? {} : { clock: options.clock }), + database, + ids: options.ids ?? ["one", "two", "three"], + }); + const spec = await part.catalog.resolveSpec({ + model: "Pixel 5", + osVersion: "34", + platform: "android", + }); + return { database, filesystem, runner, spec, ...part }; +} From 932c11d80d8fb4f6ce7c291636540bfe6ecce524 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:36:39 +0100 Subject: [PATCH 61/83] refactor(drivers): split the iOS driver into a root over simctl, marks, boot, DeviceHub, slim, catalog, components and seeds parts --- src/drivers/ios/catalog.ts | 242 ++++ src/drivers/ios/command-run.ts | 2 +- src/drivers/ios/components.ts | 410 ++++++ src/drivers/ios/devicehub.ts | 97 ++ src/drivers/ios/driver-options.ts | 161 +++ src/drivers/ios/index.ts | 1935 +++-------------------------- src/drivers/ios/marks.ts | 123 ++ src/drivers/ios/seeds.ts | 470 +++++++ src/drivers/ios/simctl.ts | 194 +++ src/drivers/ios/simulator-boot.ts | 94 ++ src/drivers/ios/slim-boot.ts | 338 +++++ src/source-hygiene.test.ts | 1 - 12 files changed, 2280 insertions(+), 1787 deletions(-) create mode 100644 src/drivers/ios/catalog.ts create mode 100644 src/drivers/ios/components.ts create mode 100644 src/drivers/ios/devicehub.ts create mode 100644 src/drivers/ios/driver-options.ts create mode 100644 src/drivers/ios/marks.ts create mode 100644 src/drivers/ios/seeds.ts create mode 100644 src/drivers/ios/simctl.ts create mode 100644 src/drivers/ios/simulator-boot.ts create mode 100644 src/drivers/ios/slim-boot.ts diff --git a/src/drivers/ios/catalog.ts b/src/drivers/ios/catalog.ts new file mode 100644 index 000000000..874e6c97c --- /dev/null +++ b/src/drivers/ios/catalog.ts @@ -0,0 +1,242 @@ +import { + type ExactDeviceRequest, + type DriverAdvisory, + type DriverCatalogEntry, + DriverCrashError, + UnsupportedRequestOptionError, + type DeviceClass, + type VirtualSpec, +} from "../../core/index.js"; + +import { Simctl } from "./simctl.js"; + +import { supportsPersistentSlim, type SlimOptions } from "./slim-boot.js"; + +import { iosRuntimeVersionFromId } from "./runtime-version.js"; + +import { resolveSlimCategories } from "./slim-labels.js"; +import { checkExcept } from "./slim-except.js"; +import { type DeviceType, type Runtime, type SimctlCatalog } from "./simctl-parse.js"; +import { requireIosPlatform } from "./simulator-data.js"; + +import { + LATEST_COMPONENT, + missingRuntime, + IosUnknownModelError, + IosVersionOutOfRangeError, + IosRuntimeUnpairedError, + findInstalledRuntime, + findDeviceType, + pairedInstalledRuntimes, + findPairedRuntime, + pairedVersions, + isUnboundedMax, + isVersionInRange, + majorVersionString, + newestRuntime, + compareVersions, +} from "./runtime-versions.js"; + +const PRODUCT_FAMILY_CLASSES: ReadonlyMap = new Map([ + ["iPhone", "phone"], + ["iPad", "tablet"], + ["Apple Watch", "watch"], + ["Apple TV", "tv"], + ["Apple Vision", "vision"], +]); + +export interface ResolvedIosSpec { + readonly deviceType: DeviceType; + readonly runtime: Runtime; + readonly spec: VirtualSpec; +} + +function specKey(spec: VirtualSpec): string { + return `${spec.model.toLocaleLowerCase()}\u0000${spec.osVersion}`; +} + +interface SimulatorCatalogOptions { + readonly simctl: Simctl; + readonly slim: SlimOptions | undefined; + readonly slimByDefault: boolean; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that resolves a requested model and OS version to an installed runtime and device type, lists the catalog, and reports the slim advisories. */ +export class SimulatorCatalog { + readonly #resolvedSpecs = new Map(); + readonly #simctl: Simctl; + readonly #slim: SlimOptions | undefined; + readonly #slimByDefault: boolean; + + constructor(options: SimulatorCatalogOptions) { + this.#simctl = options.simctl; + this.#slim = options.slim; + this.#slimByDefault = options.slimByDefault; + } + + async resolveSpec(request: ExactDeviceRequest): Promise { + requireIosPlatform(request.platform); + if (request.imageTag !== undefined) { + throw new UnsupportedRequestOptionError("ios", "imageTag"); + } + checkExcept(request.except ?? [], resolveSlimCategories(this.#slim?.categories)); + const catalog = await this.#simctl.loadCatalog(); + const deviceType = findDeviceType(catalog, request.model); + + if (deviceType === undefined) { + throw new IosUnknownModelError(request.model); + } + + const spec = + request.osVersion === undefined + ? this.#resolveDefaultRuntime(deviceType, catalog) + : this.#resolveExactRuntime(deviceType, request.osVersion, catalog); + // why: The resolution cache (`specKey`) ignores the mode, so the mode is applied here, outside it. + return request.mode === "slim" && this.#canSlim(spec) ? { ...spec, mode: "slim" } : spec; + } + + #canSlim(spec: VirtualSpec): boolean { + const runtime = this.#resolvedSpecs.get(specKey(spec))?.runtime; + return ( + this.#slim !== undefined && + runtime !== undefined && + supportsPersistentSlim(iosRuntimeVersionFromId(runtime.identifier)) + ); + } + + /** + * why: Version requested explicitly: validated against the model's `[min, max]` pairing range + * *before* anything else -- an out-of-range request can never be fixed by downloading, so it + * is never reported as downloadable. + */ + #resolveExactRuntime( + deviceType: DeviceType, + osVersion: string, + catalog: SimctlCatalog, + ): VirtualSpec { + if (!isVersionInRange(osVersion, deviceType)) { + throw new IosVersionOutOfRangeError(deviceType.name, osVersion, deviceType); + } + + // why: Looked up among the runtimes the catalog lists for this model, so a listed pair always + // resolves and an unlisted one never does (ADR 0008 §3). Any installed build of the version + // that pairs will do: two builds can share a marketing version and pair differently. + const paired = findPairedRuntime(catalog, deviceType, osVersion); + if (paired !== undefined) { + return this.#commitResolution(deviceType, paired); + } + // why: Installed but not paired: in the model's declared `[min, max]` range is necessary but not + // sufficient, since `supportedDeviceTypeIds` is the authoritative source once a runtime is + // on disk. Refused before committing, so a mismatch never reaches `simctl create`. + if (findInstalledRuntime(catalog, osVersion) !== undefined) { + throw new IosRuntimeUnpairedError(deviceType.name, osVersion); + } + + throw missingRuntime( + osVersion, + `iOS ${osVersion} is not installed; pass --allow-download (or set downloads.policy) ` + + `to download it`, + osVersion, + ); + } + + #resolveDefaultRuntime(deviceType: DeviceType, catalog: SimctlCatalog): VirtualSpec { + const paired = newestRuntime(pairedInstalledRuntimes(catalog, deviceType)); + if (paired !== undefined) { + return this.#commitResolution(deviceType, paired); + } + + throw missingRuntime( + "default", + `No installed iOS runtime pairs with ${deviceType.name}; pass --allow-download (or ` + + `set downloads.policy) to download a compatible runtime`, + isUnboundedMax(deviceType.maxRuntimeVersion) + ? LATEST_COMPONENT + : majorVersionString(deviceType.maxRuntimeVersion), + ); + } + + #commitResolution(deviceType: DeviceType, runtime: Runtime): VirtualSpec { + const spec: VirtualSpec = { + model: deviceType.name, + osVersion: runtime.version, + physical: false, + platform: "ios", + }; + this.#resolvedSpecs.set(specKey(spec), { deviceType, runtime, spec }); + return spec; + } + + async resolvedSpec(spec: VirtualSpec): Promise { + const existing = this.#resolvedSpecs.get(specKey(spec)); + if (existing !== undefined) { + return existing; + } + + await this.resolveSpec(spec); + const resolved = this.#resolvedSpecs.get(specKey(spec)); + if (resolved === undefined) { + throw new DriverCrashError("simctl did not resolve the requested device specification"); + } + + return resolved; + } + + async listCatalog(): Promise { + const catalog = await this.#simctl.loadCatalog(); + const installedRuntimes = catalog.runtimes.filter((runtime) => runtime.isAvailable); + const models = catalog.deviceTypes.map((deviceType) => deviceType.name); + return { + defaultRuntime: newestRuntime(installedRuntimes)?.version, + // why: Keyed by name and paired through the device type `resolveSpec` would pick for that name, + // so two device types that differ only in letter case both list the first one's runtimes. + // A device type answers to its name only, in any letter case (`findDeviceType`). + modelAliases: {}, + modelClasses: Object.fromEntries( + catalog.deviceTypes.flatMap((deviceType) => { + const deviceClass = PRODUCT_FAMILY_CLASSES.get(deviceType.productFamily); + return deviceClass === undefined ? [] : [[deviceType.name, deviceClass]]; + }), + ), + modelRuntimes: Object.fromEntries( + models.map((model) => [model, pairedVersions(catalog, findDeviceType(catalog, model))]), + ), + models, + runtimes: installedRuntimes.map((runtime) => runtime.version), + slimCategories: resolveSlimCategories(this.#slim?.categories).categories.map( + (category) => category.name, + ), + }; + } + + async slimAdvisories(): Promise { + if (this.#slim === undefined || !this.#slimByDefault) { + return []; + } + + const catalog = await this.#simctl.loadCatalog(); + const unsupportedVersions = [ + ...new Set( + catalog.runtimes + .filter((runtime) => runtime.isAvailable) + .filter((runtime) => !supportsPersistentSlim(iosRuntimeVersionFromId(runtime.identifier))) + .map((runtime) => runtime.version), + ), + ].sort(compareVersions); + + if (unsupportedVersions.length === 0) { + return []; + } + + const plural = unsupportedVersions.length > 1; + return [ + { + code: "slim-runtime-unsupported", + message: + `The default device mode is slim, but iOS ${unsupportedVersions.join(", ")} ${plural ? "are" : "is"} ` + + `below the 18.5 persistent-override floor; devices on ${plural ? "those runtimes" : "that runtime"} ` + + `get full devices -- \`launchctl disable\` overrides do not survive a reboot below iOS 18.5`, + }, + ]; + } +} diff --git a/src/drivers/ios/command-run.ts b/src/drivers/ios/command-run.ts index 79fd47fc0..54f07aeb6 100644 --- a/src/drivers/ios/command-run.ts +++ b/src/drivers/ios/command-run.ts @@ -86,6 +86,6 @@ export async function deleteLateSimulator( } } -function errorMessage(error: unknown): string { +export function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/src/drivers/ios/components.ts b/src/drivers/ios/components.ts new file mode 100644 index 000000000..29a56dc88 --- /dev/null +++ b/src/drivers/ios/components.ts @@ -0,0 +1,410 @@ +import { join } from "node:path"; +import { + type ComponentInstallProgress, + type ComponentInstallResult, + type ComponentReceipt, + COMPONENT_REMOVAL_TIMEOUT_MS, + type DriverAdvisory, + type DriverToolVersion, + type DriverComponent, + DriverCrashError, + type InstalledComponent, + sameReceipt, +} from "../../core/index.js"; +import type { Clock, Filesystem, ProcessRunner } from "../../ports/index.js"; + +import { errorMessage } from "./command-run.js"; + +import { COMMAND_TIMEOUT_MS, Simctl } from "./simctl.js"; +import { DeviceMarks } from "./marks.js"; +import { runBoundedProcess, runInstallerProcess } from "../installer-process.js"; + +import { + IOS_RUNTIME_ASSET_ROOT, + type RuntimeAsset, + runtimeMountsAsset, + parseRuntimeAsset, + type RuntimeImage, + isBeingDeleted, + runtimeReceipt, + imageReceipt, + isIosImage, + type ListedImage, + leftoverDownloadsSentence, + unavailableDevicesResidue, + type DefaultSetDevices, + devicesPerRuntime, + installedComponent, + withBareMajorHint, +} from "./runtime-images.js"; +import { + LATEST_COMPONENT, + belowDownloadFloor, + IosDownloadFloorError, + IosRuntimeUnavailableForDownloadError, + runtimeForComponent, + findInstalledRuntime, + compareVersions, +} from "./runtime-versions.js"; + +// why: `xcodebuild -version` answers in about a second; a first run after an Xcode update can take +// longer while it checks its components, and a hang must not hold host facts forever. +const XCODE_VERSION_TIMEOUT_MS = 15_000; + +// why: How often a removal lists again while `simctl runtime list` shows the deleted image as still +// being deleted, which it does for a few seconds after `simctl runtime delete` answers (#259). +const DELETING_POLL_MS = 1_000; + +interface RuntimeComponentsOptions { + readonly clock: Clock; + readonly filesystem: Filesystem; + readonly marks: DeviceMarks; + readonly processRunner: ProcessRunner; + readonly simctl: Simctl; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that finds, installs, lists and removes the iOS runtimes, and reports the download cache they leave behind. */ +export class RuntimeComponents { + readonly #clock: Clock; + readonly #filesystem: Filesystem; + readonly #marks: DeviceMarks; + readonly #processRunner: ProcessRunner; + readonly #simctl: Simctl; + + constructor(options: RuntimeComponentsOptions) { + this.#clock = options.clock; + this.#filesystem = options.filesystem; + this.#marks = options.marks; + this.#processRunner = options.processRunner; + this.#simctl = options.simctl; + } + + async findComponent(component: string): Promise { + const [catalog, images] = await Promise.all([ + this.#simctl.loadCatalog(), + this.#simctl.loadRuntimeImages(), + ]); + const runtime = findInstalledRuntime(catalog, component); + return runtime === undefined ? undefined : installedComponent(runtime, images); + } + + async installComponent( + component: string, + options: { + readonly onProgress: (progress: ComponentInstallProgress) => void; + readonly signal: AbortSignal; + }, + ): Promise { + if (belowDownloadFloor(component)) { + throw new IosDownloadFloorError(component); + } + const before = await this.#installedReceipts(); + const args = + component === LATEST_COMPONENT + ? ["-downloadPlatform", "iOS"] + : ["-downloadPlatform", "iOS", "-buildVersion", component]; + try { + await this.#xcodebuildOrThrow(args, options); + } catch (error: unknown) { + throw withBareMajorHint(component, error); + } + + const [catalog, images] = await Promise.all([ + this.#simctl.loadCatalog(), + this.#simctl.loadRuntimeImages(), + ]); + const runtime = runtimeForComponent(catalog, component); + if (runtime === undefined) { + throw new DriverCrashError( + `xcodebuild reported success but iOS ${component} is still not installed`, + ); + } + const installed = installedComponent(runtime, images); + const wasThere = before.some((receipt) => sameReceipt(receipt, installed.receipt)); + return { ...installed, outcome: wasThere ? "already-installed" : "installed" }; + } + + async listComponents(): Promise { + return (await this.listedImages()).map( + ({ image: _image, unusedDevices: _unused, ...component }) => component, + ); + } + + async #deleteRuntimeImage(identifier: string, signal: AbortSignal): Promise { + const args = ["runtime", "delete", identifier]; + const command = `simctl ${args.join(" ")}`; + let outcome; + try { + outcome = await runBoundedProcess( + this.#processRunner, + this.#clock, + "xcrun", + this.#simctl.scopedArgv(args), + { signal, timeoutMs: COMPONENT_REMOVAL_TIMEOUT_MS }, + ); + } catch (error: unknown) { + throw new DriverCrashError(`${command} failed: ${errorMessage(error)}`); + } + if (outcome.timedOut) { + throw new DriverCrashError( + `${command} timed out after ${String(COMPONENT_REMOVAL_TIMEOUT_MS)}ms`, + ); + } + if (outcome.stopped) throw new DriverCrashError(`${command} was ended before it finished`); + this.#simctl.assertSuccessful(args, outcome.result); + } + + async #waitWhileDeleting( + identifier: string, + version: string, + deadline: number, + signal: AbortSignal, + ): Promise { + for (;;) { + const listed = (await this.#simctl.loadRuntimeImages()).find( + (image) => image.identifier === identifier, + ); + if (listed === undefined || !isBeingDeleted(listed)) return; + const remainingMs = deadline - this.#clock.now(); + if (remainingMs <= 0) { + throw new DriverCrashError( + `ios ${version} was still being deleted ` + + `${String(COMPONENT_REMOVAL_TIMEOUT_MS)}ms after its removal started`, + ); + } + if (!(await this.#pause(Math.min(DELETING_POLL_MS, remainingMs), signal))) { + throw new DriverCrashError( + `The removal of ios ${version} was ended while it was still being deleted`, + ); + } + } + } + + #pause(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.resolve(false); + return new Promise((resolve) => { + const onAbort = (): void => { + this.#clock.cancel(timer); + resolve(false); + }; + const timer = this.#clock.setTimer(delayMs, () => { + signal.removeEventListener("abort", onAbort); + resolve(true); + }); + signal.addEventListener("abort", onAbort, { once: true }); + }); + } + + async #downloadResidue(image: RuntimeImage, version: string): Promise<{ residue?: string }> { + const left = (await this.#downloadedRuntimeAssets()).filter((asset) => + image.build === undefined ? asset.version === version : asset.build === image.build, + ); + if (left.length === 0) return {}; + const label = image.build === undefined ? version : `${version} (${image.build})`; + return { + residue: leftoverDownloadsSentence([label], left.length, await this.#assetsSize(left)), + }; + } + + async #assetsSize(assets: readonly RuntimeAsset[]): Promise { + let total = 0; + for (const asset of assets) { + try { + total += await this.#filesystem.directorySize(asset.path); + } catch { + return undefined; + } + } + return total; + } + + async listedImages(): Promise { + const [images, defaultSet] = await Promise.all([ + this.#simctl.loadRuntimeImages(), + this.#simctl.runLegacy(["list", "-j", "devices"], COMMAND_TIMEOUT_MS), + ]); + let foreign: ReadonlyMap; + try { + const parsed = JSON.parse(defaultSet.stdout) as unknown; + foreign = devicesPerRuntime(parsed, await this.#marks.ownedUdids(parsed)); + } catch (error: unknown) { + if (error instanceof DriverCrashError) throw error; + throw new DriverCrashError(`Could not parse simctl device list: ${errorMessage(error)}`); + } + return images.filter(isIosImage).flatMap((image): ListedImage[] => + image.version === undefined + ? [] + : [ + { + foreignDevices: foreign.get(image.runtimeIdentifier)?.used ?? 0, + image, + unusedDevices: images.some( + (other) => + other !== image && + isIosImage(other) && + other.runtimeIdentifier === image.runtimeIdentifier, + ) + ? 0 + : (foreign.get(image.runtimeIdentifier)?.unused ?? 0), + receipt: imageReceipt(image), + version: image.version, + ...(image.build === undefined ? {} : { variant: image.build }), + ...(image.sizeBytes === undefined ? {} : { sizeBytes: image.sizeBytes }), + }, + ], + ); + } + + async #installedReceipts(): Promise { + const [catalog, images] = await Promise.all([ + this.#simctl.loadCatalog(), + this.#simctl.loadRuntimeImages(), + ]); + return catalog.runtimes + .filter((runtime) => runtime.isAvailable) + .map((runtime) => runtimeReceipt(runtime, images)); + } + + async #xcodebuildOrThrow( + args: readonly string[], + options: { + readonly onProgress: (progress: ComponentInstallProgress) => void; + readonly signal: AbortSignal; + }, + ): Promise { + let outcome; + try { + outcome = await runInstallerProcess( + this.#processRunner, + this.#clock, + "xcodebuild", + args, + options, + ); + } catch (error: unknown) { + throw new DriverCrashError(`xcodebuild ${args.join(" ")} failed: ${errorMessage(error)}`); + } + if (outcome.stopped) { + throw new DriverCrashError(`xcodebuild ${args.join(" ")} was ended before it finished`); + } + const { result } = outcome; + if (result.code === 0) { + return; + } + const message = result.stderr || result.stdout; + // why: Apple's downloader accepts `-buildVersion ` only for whatever it + // currently offers -- a pin naming a release Apple has since retired from its live catalog + // (superseded by a newer point release), or one still ambiguous across several concurrent + // beta/seed builds sharing the same marketing version, is rejected outright with "is not + // available for download", distinct from every other `xcodebuild` failure this method + // otherwise reports as a `DriverCrashError`. Falling back to an unpinned `-downloadPlatform + // iOS` here would silently substitute whatever Apple currently offers instead -- a different, + // unrequested, and possibly multi-gigabyte runtime -- for a request that named an exact + // version, so this fails the request instead of guessing; see + // `IosRuntimeUnavailableForDownloadError`. + const buildVersionIndex = args.indexOf("-buildVersion"); + if (buildVersionIndex !== -1 && /is not available for download/.test(message)) { + throw new IosRuntimeUnavailableForDownloadError( + args[buildVersionIndex + 1] ?? "requested version", + ); + } + throw new DriverCrashError(`xcodebuild ${args.join(" ")} failed: ${message}`); + } + + async toolVersions(): Promise { + const result = await this.#processRunner.run("xcodebuild", ["-version"], { + timeoutMs: XCODE_VERSION_TIMEOUT_MS, + }); + if (result.code !== 0) { + if (/requires Xcode/.test(result.stderr)) return []; + throw new Error(`xcodebuild -version exited with ${String(result.code)}`); + } + const version = /^Xcode (\S+)$/m.exec(result.stdout)?.[1]; + const build = /^Build version (\S+)$/m.exec(result.stdout)?.[1]; + if (version === undefined) throw new Error("xcodebuild -version printed no Xcode version"); + return [{ name: "xcode", version, ...(build === undefined ? {} : { build }) }]; + } + + async cacheAdvisories(): Promise { + const assets = await this.#downloadedRuntimeAssets(); + + if (assets.length === 0) { + return []; + } + + const installed = (await this.#simctl.loadCatalog()).runtimes.filter( + (runtime) => runtime.isAvailable, + ); + const orphans = assets.filter( + (asset) => !installed.some((runtime) => runtimeMountsAsset(runtime, asset)), + ); + + if (orphans.length === 0) { + return []; + } + + // why: Oldest first, by version and then by build, so a store that has been collecting for + // months reads in the order the downloads happened rather than as a lexicographic jumble. + const described = [ + ...new Map( + orphans.map((asset) => [`${asset.version} (${asset.build})`, asset] as const), + ).entries(), + ] + .sort( + ([, left], [, right]) => + compareVersions(left.version, right.version) || left.build.localeCompare(right.build), + ) + .map(([label]) => label); + return [ + { + code: "runtime-cache-unreclaimable", + message: leftoverDownloadsSentence( + described, + orphans.length, + await this.#assetsSize(orphans), + ), + }, + ]; + } + + async #downloadedRuntimeAssets(): Promise { + let bundles: readonly string[]; + try { + bundles = await this.#filesystem.readdir(IOS_RUNTIME_ASSET_ROOT); + } catch { + return []; + } + + const assets: RuntimeAsset[] = []; + for (const bundle of bundles.filter((name) => name.endsWith(".asset"))) { + let contents: string; + try { + contents = await this.#filesystem.readFile( + join(IOS_RUNTIME_ASSET_ROOT, bundle, "Info.plist"), + ); + } catch { + continue; + } + + const asset = parseRuntimeAsset(join(IOS_RUNTIME_ASSET_ROOT, bundle), contents); + if (asset !== undefined) { + assets.push(asset); + } + } + + return assets; + } + + /** why: the part of the runtime removal that runs after its seeds are gone: the runtime's own deletion, the wait while it is still being deleted, and the residue left behind. */ + async removeImage({ image, unusedDevices, version }: ListedImage, signal: AbortSignal) { + // why: the budget is for the runtime's own deletion; a seed held by a late clone waits outside it. + const deadline = this.#clock.now() + COMPONENT_REMOVAL_TIMEOUT_MS; + await this.#deleteRuntimeImage(image.identifier, signal); + await this.#waitWhileDeleting(image.identifier, version, deadline, signal); + const sentences = [ + unavailableDevicesResidue(unusedDevices), + (await this.#downloadResidue(image, version)).residue, + ].filter((sentence) => sentence !== undefined); + return sentences.length === 0 ? {} : { residue: sentences.join(". ") }; + } +} diff --git a/src/drivers/ios/devicehub.ts b/src/drivers/ios/devicehub.ts new file mode 100644 index 000000000..049ba8122 --- /dev/null +++ b/src/drivers/ios/devicehub.ts @@ -0,0 +1,97 @@ +import { join } from "node:path"; +import { DriverCrashError } from "../../core/index.js"; +import type { Clock, Filesystem, Logger, ProcessRunner } from "../../ports/index.js"; + +import { runCommand } from "./command-run.js"; + +import { COMMAND_TIMEOUT_MS } from "./simctl.js"; + +const DEVICEHUB_DOMAIN = "com.apple.dt.Devices"; +const DEVICEHUB_STAGING_FILE = "devicehub-preferences.plist"; + +let pasteboardWrites: Promise = Promise.resolve(); + +async function withPasteboardWrite(run: () => Promise): Promise { + const turn = pasteboardWrites.then(run, run); + pasteboardWrites = turn.then( + () => undefined, + () => undefined, + ); + return await turn; +} + +interface DeviceHubPreferencesOptions { + readonly clock: Clock; + readonly filesystem: Filesystem; + readonly logger: Logger | undefined; + readonly overridesRoot: string; + readonly processRunner: ProcessRunner; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that turns DeviceHub's clipboard sync off for a simulator, and removes that setting when the simulator is destroyed. */ +export class DeviceHubPreferences { + readonly #clock: Clock; + readonly #filesystem: Filesystem; + readonly #logger: Logger | undefined; + readonly #overridesRoot: string; + readonly #processRunner: ProcessRunner; + + constructor(options: DeviceHubPreferencesOptions) { + this.#clock = options.clock; + this.#filesystem = options.filesystem; + this.#logger = options.logger; + this.#overridesRoot = options.overridesRoot; + this.#processRunner = options.processRunner; + } + + // why: DeviceHub's preferences belong to cfprefsd, which keeps its own copy and writes it back + // over the file, so an edit made to the file alone is lost or ignored. The domain is exported + // to a scratch file, edited there with PlistBuddy, and imported back through cfprefsd. + async #editDeviceHubPreferences( + edit: (buddy: (command: string) => Promise) => Promise, + ): Promise { + return await withPasteboardWrite(async () => { + await this.#filesystem.mkdirp(this.#overridesRoot, { mode: 0o700 }); + const staged = join(this.#overridesRoot, DEVICEHUB_STAGING_FILE); + const run = async (command: string, args: string[]): Promise => { + const outcome = await runCommand( + this.#processRunner, + this.#clock, + command, + args, + COMMAND_TIMEOUT_MS, + ); + return outcome.kind !== "timed-out" && outcome.result.code === 0; + }; + try { + if (!(await run("defaults", ["export", DEVICEHUB_DOMAIN, staged]))) return false; + const buddy = (command: string) => run("/usr/libexec/PlistBuddy", ["-c", command, staged]); + return (await edit(buddy)) && (await run("defaults", ["import", DEVICEHUB_DOMAIN, staged])); + } finally { + await this.#filesystem.rm(staged).catch(() => undefined); + } + }); + } + + async disablePasteboardSync(udid: string): Promise { + const path = `:DevicePreferences:${udid}`; + const done = await this.#editDeviceHubPreferences(async (buddy) => { + if (await buddy(`Set ${path}:pasteboardSyncEnabled false`)) return true; + await buddy("Add :DevicePreferences dict"); + await buddy(`Add ${path} dict`); + return await buddy(`Add ${path}:pasteboardSyncEnabled bool false`); + }); + if (!done) throw new DriverCrashError(`Could not disable DeviceHub clipboard sync for ${udid}`); + } + + async removePasteboardPreference(udid: string): Promise { + const done = await this.#editDeviceHubPreferences((buddy) => + buddy(`Delete :DevicePreferences:${udid}`), + ).catch(() => false); + if (!done) { + this.#logger?.warn( + `could not remove DeviceHub preference for ${udid}: preference deletion failed`, + ); + } + } +} diff --git a/src/drivers/ios/driver-options.ts b/src/drivers/ios/driver-options.ts new file mode 100644 index 000000000..0850a8fc0 --- /dev/null +++ b/src/drivers/ios/driver-options.ts @@ -0,0 +1,161 @@ +import { join } from "node:path"; +import { OwnedRootError, type ScreenCapture } from "../../core/index.js"; +import type { + BytePipeLauncher, + Clock, + Filesystem, + IdGenerator, + Logger, + ProcessRunner, + SqliteDatabase, + TcpListenerFactory, + TcpProbe, + TcpStreamConnector, +} from "../../ports/index.js"; +import type { DeviceInputFactory } from "../../core/index.js"; +import type { SilenceFailedFact, WindowFailedFact } from "./boot-courtesies.js"; +import type { SlimmedFact, SlimSkippedFact } from "./slim-boot.js"; +import { defaultIosDeviceRoot, type IosDriverConfig } from "./definition.js"; +import { headlessNeedsDefaultSet } from "./device-set-leftovers.js"; +import { createIosDeviceInput } from "./input/index.js"; +import { createIosScreenCapture } from "./screen-capture.js"; + +export interface IosSimctlDriverOptions { + readonly clock: Clock; + readonly database: SqliteDatabase; + readonly driverConfig: Readonly>; + readonly filesystem: Filesystem; + readonly idGenerator: IdGenerator; + readonly instanceId: string; + readonly processRunner: ProcessRunner; + readonly simlockHome: string; + readonly uid?: number; + /** why: the daemon's pid is named in the staging directory of a new root. */ + readonly pid?: number; + readonly coreSimulatorRoot?: string; + // why: where launchd_sim keeps each simulator's overrides folder; a test points it elsewhere. + readonly launchdOverridesRoot?: string; + readonly onSlimmed?: (fact: SlimmedFact) => void; + readonly onSlimSkipped?: (fact: SlimSkippedFact) => void; + /** why: the driver's own keys of the `ios` config section; with no `slim` the driver slims nothing, with no `simulator` it keeps audio on and opens no window. */ + readonly config?: Partial; + readonly onSilenceFailed?: (fact: SilenceFailedFact) => void; + readonly onWindowFailed?: (fact: WindowFailedFact) => void; + readonly slimByDefault?: boolean; + // why: the device input logs a Home press that did nothing; nothing else in the driver logs through it. + readonly logger?: Logger; + // why: a simulator reaches the worker's loopback, so its tunnels need these three ports from the daemon; with any one missing the driver offers no `deviceTcp`. + readonly tcpListener?: TcpListenerFactory; + readonly tcpProbe?: TcpProbe; + readonly tcpStreamConnector?: TcpStreamConnector; + // why: absent when this host has no way to run the capture program; the driver then offers no + // screen at all rather than one that fails on every open. + readonly screenCapture?: { + readonly command: string; + readonly deviceCommand?: string; + readonly launcher: BytePipeLauncher; + readonly baseEnvironment: NodeJS.ProcessEnv; + readonly onLog?: (line: string) => void; + }; +} + +// why: the input is offered exactly when the screen capture is, because both are the one program. +export function createInputFor( + options: IosSimctlDriverOptions, + deviceRoot: string, + ownedRoot: boolean, +): DeviceInputFactory | undefined { + if (options.screenCapture === undefined) return undefined; + return createIosDeviceInput({ + baseEnvironment: options.screenCapture.baseEnvironment, + clock: options.clock, + command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, + ...(ownedRoot ? { deviceSet: deviceRoot } : {}), + ...(options.logger === undefined ? {} : { logger: options.logger }), + launcher: options.screenCapture.launcher, + processRunner: options.processRunner, + }); +} + +function configuredScreenCaptureCommand(options: IosSimctlDriverOptions): string | undefined { + const configured = options.driverConfig["screenCaptureCommand"]; + return typeof configured === "string" && configured !== "" ? configured : undefined; +} + +function configuredDeviceCaptureCommand(options: IosSimctlDriverOptions): string | undefined { + const configured = options.driverConfig["deviceCaptureCommand"]; + return typeof configured === "string" && configured !== "" ? configured : undefined; +} + +export function configuredDeviceSet(options: IosSimctlDriverOptions): "simlock" | "default" { + const configured = options.driverConfig["deviceSet"]; + if (configured === undefined || configured === "simlock") { + if (options.config?.simulator?.headless === false) throw headlessNeedsDefaultSet("simlock"); + return "simlock"; + } + if (configured !== "default") { + throw new OwnedRootError( + `Refusing the ios device set: drivers.ios.deviceSet must be "simlock" or "default", but it is ${JSON.stringify(configured)}`, + "invalid-device-set", + typeof configured === "string" ? configured : JSON.stringify(configured), + "ios", + ); + } + if (options.driverConfig["deviceRoot"] !== undefined) { + throw new OwnedRootError( + `Refusing the ios device set: drivers.ios.deviceRoot names Simlock's own set, but drivers.ios.deviceSet is "default". Remove one of them.`, + "invalid-device-set", + configured, + "ios", + ); + } + return "default"; +} + +export function xcodeDeviceSetPath(options: IosSimctlDriverOptions): string { + return join(options.coreSimulatorRoot ?? ".", "Devices"); +} + +export function configuredDeviceRoot(options: IosSimctlDriverOptions): string { + const configured = options.driverConfig["deviceRoot"]; + + if (configured !== undefined && typeof configured !== "string") { + // why: A `deviceRoot` that is not a usable path refuses this platform's *configuration*, + // which costs iOS and nothing else -- the same as every other refusal here. It is not + // a daemon-fatal error: `"deviceRoot": true` and `"deviceRoot": "devices/ios"` are one + // keystroke apart, and killing the process over the first would take Android down with + // it and leave the reason unreachable, since `doctor` -- where the CLI reference promises + // it appears -- needs a daemon to answer. `not-absolute` is the vocabulary term the + // docs already publish for "this names no usable directory"; nothing new is invented. + throw new OwnedRootError( + `Refusing the ios device root: drivers.ios.deviceRoot must be an absolute path, but it is the ${typeof configured} ${JSON.stringify(configured)}`, + "not-absolute", + String(configured), + "ios", + ); + } + + return configured ?? defaultIosDeviceRoot(options.simlockHome); +} + +export function createScreenCaptureFor( + options: IosSimctlDriverOptions, + deviceRoot: string, + ownsDeviceSet: boolean, + deviceOsVersion: (udid: string) => Promise, +): ScreenCapture | undefined { + const capture = options.screenCapture; + if (capture === undefined) return undefined; + const deviceCommand = configuredDeviceCaptureCommand(options) ?? capture.deviceCommand; + return createIosScreenCapture({ + baseEnvironment: capture.baseEnvironment, + clock: options.clock, + command: configuredScreenCaptureCommand(options) ?? capture.command, + ...(deviceCommand === undefined ? {} : { deviceCommand }), + deviceOsVersion, + ...(ownsDeviceSet ? { deviceSet: deviceRoot } : {}), + filesystem: options.filesystem, + launcher: capture.launcher, + ...(capture.onLog === undefined ? {} : { onLog: capture.onLog }), + }); +} diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 17a28b575..906230f9d 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -1,14 +1,11 @@ import { dirname, join } from "node:path"; import { - BootTimeoutError, - KeptCategoriesError, type ComponentInstallProgress, type ComponentInstallResult, type ComponentReceipt, type ComponentRemoval, type DeviceInputFactory, type DeviceTcp, - COMPONENT_REMOVAL_TIMEOUT_MS, type ExactDeviceRequest, type Driver, type DriverAdvisory, @@ -31,131 +28,54 @@ import { type InstalledComponent, type LegacyDevice, type ObservedDevice, - OwnedRootError, type PassthroughCommand, type PassthroughContext, removeListedComponent, - sameReceipt, - UnsupportedRequestOptionError, - type ObservedMark, type DeviceClass, type DeviceMode, type DeviceSpec, type VirtualSpec, type ScreenCapture, } from "../../core/index.js"; -import type { - BytePipeLauncher, - Clock, - Filesystem, - IdGenerator, - Logger, - ProcessResult, - ProcessRunner, - SqliteDatabase, - TcpListenerFactory, - TcpProbe, - TcpStreamConnector, -} from "../../ports/index.js"; -import { - runAfterBoot, - type AfterBoot, - type ProcessOutcome, - type SilenceFailedFact, - type WindowFailedFact, -} from "./boot-courtesies.js"; -import { deleteLateSimulator, runCommand, type LateAnswer } from "./command-run.js"; +import type { Clock, ProcessRunner } from "../../ports/index.js"; +import { runAfterBoot, type AfterBoot } from "./boot-courtesies.js"; +import { runCommand } from "./command-run.js"; import { SeedTurns } from "./seed-turns.js"; -import { runBoundedProcess, runInstallerProcess } from "../installer-process.js"; +import { COMMAND_TIMEOUT_MS, Simctl } from "./simctl.js"; +import { DeviceMarks } from "./marks.js"; +import { SimulatorBoot } from "./simulator-boot.js"; +import { DeviceHubPreferences } from "./devicehub.js"; +import { DEFAULT_OVERRIDES_ROOT, SlimBoot, type SlimOptions } from "./slim-boot.js"; +import { RuntimeComponents } from "./components.js"; +import { SimulatorCatalog } from "./catalog.js"; +import { Seeds } from "./seeds.js"; + import { - headlessNeedsDefaultSet, refuseDefaultSourcesLeftovers, refuseOwnSetLeftovers, refuseXcodeSetLeftovers, - UDID_PATTERN, } from "./device-set-leftovers.js"; import { createIosDeviceTcpFor } from "./device-tcp.js"; import { IosPhysicalHandler } from "./physical.js"; -import { createIosDeviceInput } from "./input/index.js"; import { iosRuntimeVersionFromId } from "./runtime-version.js"; -import { IosSourceTable, sourcesSetFor, type ReadySeed, type SourceRow } from "./source-data.js"; -import { createIosScreenCapture } from "./screen-capture.js"; -import { defaultIosDeviceRoot, type IosDriverConfig } from "./definition.js"; +import { sourcesSetFor } from "./source-data.js"; import { - overridesFor, - resolveSlimCategories, - slimSignature, - type SlimCategory, -} from "./slim-labels.js"; -import { assertKeptEnabled, checkExcept, splitByExcept } from "./slim-except.js"; -import { - isRecord, - parseCatalog, - parseManagedDevices, - type DeviceType, - type Runtime, - type SimctlCatalog, -} from "./simctl-parse.js"; -import { parseOverridesPlist, parsePrintDisabled, renderOverridesPlist } from "./slim-overrides.js"; + configuredDeviceRoot, + configuredDeviceSet, + createInputFor, + createScreenCaptureFor, + type IosSimctlDriverOptions, + xcodeDeviceSetPath, +} from "./driver-options.js"; + +import { parseManagedDevices } from "./simctl-parse.js"; + import { type IosDriverData, iosDriverData, requireIosPlatform } from "./simulator-data.js"; import { IOS_PASSTHROUGH_TOOLS, passthroughCommand } from "./passthrough-rules.js"; -import { - IOS_RUNTIME_MIN_FREE_BYTES, - IOS_RUNTIME_ASSET_ROOT, - type RuntimeAsset, - runtimeMountsAsset, - parseRuntimeAsset, - type RuntimeImage, - isBeingDeleted, - parseRuntimeImages, - runtimeReceipt, - imageReceipt, - isIosImage, - type ListedImage, - leftoverDownloadsSentence, - unavailableDevicesResidue, - type DefaultSetDevices, - devicesPerRuntime, - installedComponent, - withBareMajorHint, -} from "./runtime-images.js"; -import { - LATEST_COMPONENT, - missingRuntime, - belowDownloadFloor, - IosUnknownModelError, - IosVersionOutOfRangeError, - IosRuntimeUnpairedError, - IosDownloadFloorError, - IosRuntimeUnavailableForDownloadError, - runtimeForComponent, - findInstalledRuntime, - findDeviceType, - pairedInstalledRuntimes, - findPairedRuntime, - pairedVersions, - isUnboundedMax, - isVersionInRange, - majorVersionString, - newestRuntime, - compareVersions, -} from "./runtime-versions.js"; +import { IOS_RUNTIME_MIN_FREE_BYTES } from "./runtime-images.js"; -/** why: ADR 0061 §1: a `making` row of this process older than this is a make that never ended. */ -const STALE_MAKING_MS = 300_000; -const COMMAND_TIMEOUT_MS = 30_000; -/** why: a clone or create past its limit is left running; one still silent this long after is killed. */ -const LATE_CREATE_KILL_MS = 300_000; - -// why: `xcodebuild -version` answers in about a second; a first run after an Xcode update can take -// longer while it checks its components, and a hang must not hold host facts forever. -const XCODE_VERSION_TIMEOUT_MS = 15_000; const DEFAULT_FULL_BOOT_TIMEOUT_MS = 300_000; -const INSTALLER_POLL_MS = 1_000; const PROVISION_ESTIMATE_MS = 500; -// why: How often a removal lists again while `simctl runtime list` shows the deleted image as still -// being deleted, which it does for a few seconds after `simctl runtime delete` answers (#259). -const DELETING_POLL_MS = 1_000; // why: A cold `simctl boot` to `bootstatus` measures roughly 30s on a fast, idle machine and up to // a minute on a loaded or slower one. The upper end is the estimate, deliberately: this number // is what a waiting requester is quoted, and quoting 30s to someone who then waits 60s is the @@ -168,20 +88,6 @@ const COLD_BOOT_ESTIMATE_MS = 60_000; // A physical device reclaim uninstalls apps instead and has its own estimate below. const ERASE_ESTIMATE_MS = 34_000; const PHYSICAL_RECLAIM_ESTIMATE_MS = 90_000; -const MARK_FILE_NAME = "simlock-mark.json"; -const DEVICEHUB_DOMAIN = "com.apple.dt.Devices"; -const DEVICEHUB_STAGING_FILE = "devicehub-preferences.plist"; - -let pasteboardWrites: Promise = Promise.resolve(); - -async function withPasteboardWrite(run: () => Promise): Promise { - const turn = pasteboardWrites.then(run, run); - pasteboardWrites = turn.then( - () => undefined, - () => undefined, - ); - return await turn; -} const IOS_DEFAULT_MODELS: Readonly>> = { phone: ["iPhone 17", "iPhone 16", "iPhone 15", "iPhone 14", "iPhone 13"], @@ -201,123 +107,28 @@ const IOS_DEFAULT_MODELS: Readonly>; - readonly filesystem: Filesystem; - readonly idGenerator: IdGenerator; - readonly instanceId: string; - readonly processRunner: ProcessRunner; - readonly simlockHome: string; - readonly uid?: number; - /** why: the daemon's pid is named in the staging directory of a new root. */ - readonly pid?: number; - readonly coreSimulatorRoot?: string; - // why: where launchd_sim keeps each simulator's overrides folder; a test points it elsewhere. - readonly launchdOverridesRoot?: string; - readonly onSlimmed?: (fact: SlimmedFact) => void; - readonly onSlimSkipped?: (fact: SlimSkippedFact) => void; - /** why: the driver's own keys of the `ios` config section; with no `slim` the driver slims nothing, with no `simulator` it keeps audio on and opens no window. */ - readonly config?: Partial; - readonly onSilenceFailed?: (fact: SilenceFailedFact) => void; - readonly onWindowFailed?: (fact: WindowFailedFact) => void; - readonly slimByDefault?: boolean; - // why: the device input logs a Home press that did nothing; nothing else in the driver logs through it. - readonly logger?: Logger; - // why: a simulator reaches the worker's loopback, so its tunnels need these three ports from the daemon; with any one missing the driver offers no `deviceTcp`. - readonly tcpListener?: TcpListenerFactory; - readonly tcpProbe?: TcpProbe; - readonly tcpStreamConnector?: TcpStreamConnector; - // why: absent when this host has no way to run the capture program; the driver then offers no - // screen at all rather than one that fails on every open. - readonly screenCapture?: { - readonly command: string; - readonly deviceCommand?: string; - readonly launcher: BytePipeLauncher; - readonly baseEnvironment: NodeJS.ProcessEnv; - readonly onLog?: (line: string) => void; - }; -} - -type SlimOptions = IosDriverConfig["slim"]; - -type SlimWrite = - | { readonly kind: "no-categories"; readonly detail: string } - | { - readonly kind: "written"; - readonly resolved: ReturnType; - readonly wanted: ReadonlyMap; - readonly kept: readonly SlimCategory[]; - readonly startedAt: number; - readonly writeError: string | undefined; - }; - -const PRODUCT_FAMILY_CLASSES: ReadonlyMap = new Map([ - ["iPhone", "phone"], - ["iPad", "tablet"], - ["Apple Watch", "watch"], - ["Apple TV", "tv"], - ["Apple Vision", "vision"], -]); - -interface ResolvedIosSpec { - readonly deviceType: DeviceType; - readonly runtime: Runtime; - readonly spec: VirtualSpec; -} - export class IosSimctlDriver implements Driver { readonly platform = "ios" as const; readonly rebootReadyBeforeGrant: boolean; readonly defaultModels = IOS_DEFAULT_MODELS; readonly #clock: Clock; readonly componentFootprint: { readonly path: string; readonly bytes: number }; - readonly #filesystem: Filesystem; - readonly #idGenerator: IdGenerator; - readonly #onSlimmed: ((fact: SlimmedFact) => void) | undefined; - readonly #onSlimSkipped: ((fact: SlimSkippedFact) => void) | undefined; readonly #processRunner: ProcessRunner; readonly #physical: IosPhysicalHandler; - readonly #resolvedSpecs = new Map(); + readonly #simctl: Simctl; + readonly #boot: SimulatorBoot; + readonly #deviceHub: DeviceHubPreferences; + readonly #marks: DeviceMarks; + readonly #slimBoot: SlimBoot; + readonly #catalog: SimulatorCatalog; + readonly #components: RuntimeComponents; + readonly #seeds: Seeds; readonly #deviceRoot: string; - readonly #instanceId: string; readonly #rootOptions: EnsureOwnedRootOptions | undefined; - readonly #logger: Logger | undefined; - readonly #overridesRoot: string; readonly #slim: SlimOptions | undefined; - readonly #createTimeoutMs: number; readonly #fullBootTimeoutMs: number; readonly #afterBoot: AfterBoot; readonly #slimByDefault: boolean; - readonly #sources: IosSourceTable; - readonly #sourcesSet: string; - #sourceMaker: string | undefined; - #swept = false; - readonly #seedMakes = new Map>(); readonly #seedTurns = new SeedTurns(); private constructor( @@ -331,29 +142,88 @@ export class IosSimctlDriver implements Driver { bytes: IOS_RUNTIME_MIN_FREE_BYTES, path: options.coreSimulatorRoot ?? ".", }; - this.#filesystem = options.filesystem; - this.#idGenerator = options.idGenerator; - this.#onSlimmed = options.onSlimmed; - this.#onSlimSkipped = options.onSlimSkipped; this.#processRunner = options.processRunner; + this.#simctl = new Simctl({ + clock: options.clock, + createTimeoutMs: options.config?.simulator?.createTimeoutMs ?? COMMAND_TIMEOUT_MS, + deviceRoot, + logger: options.logger, + ownsDeviceSet: rootOptions !== undefined, + processRunner: options.processRunner, + seedTurns: this.#seedTurns, + }); + this.#boot = new SimulatorBoot({ + clock: options.clock, + ownsDeviceSet: rootOptions !== undefined, + simctl: this.#simctl, + }); + this.#marks = new DeviceMarks({ + clock: options.clock, + deviceRoot, + filesystem: options.filesystem, + idGenerator: options.idGenerator, + instanceId: options.instanceId, + ownsDeviceSet: rootOptions !== undefined, + }); + this.#components = new RuntimeComponents({ + clock: options.clock, + filesystem: options.filesystem, + marks: this.#marks, + processRunner: options.processRunner, + simctl: this.#simctl, + }); + this.#deviceHub = new DeviceHubPreferences({ + clock: options.clock, + filesystem: options.filesystem, + logger: options.logger, + overridesRoot: options.launchdOverridesRoot ?? DEFAULT_OVERRIDES_ROOT, + processRunner: options.processRunner, + }); this.#physical = new IosPhysicalHandler({ filesystem: options.filesystem, idGenerator: options.idGenerator, - invoke: (argv, timeoutMs) => this.#invokeXcrun(argv, timeoutMs), + invoke: (argv, timeoutMs) => this.#simctl.invokeXcrun(argv, timeoutMs), simlockHome: options.simlockHome, }); this.#deviceRoot = deviceRoot; - this.#instanceId = options.instanceId; this.#rootOptions = rootOptions; - this.#logger = options.logger; - this.#overridesRoot = options.launchdOverridesRoot ?? DEFAULT_OVERRIDES_ROOT; this.#slim = options.config?.slim; - this.#createTimeoutMs = options.config?.simulator?.createTimeoutMs ?? COMMAND_TIMEOUT_MS; this.#fullBootTimeoutMs = options.config?.full?.bootTimeoutMs ?? DEFAULT_FULL_BOOT_TIMEOUT_MS; this.#afterBoot = this.#afterBootFor(options); this.#slimByDefault = options.slimByDefault === true; - this.#sources = new IosSourceTable(options.database); - this.#sourcesSet = sourcesSetFor(options.simlockHome, deviceRoot, rootOptions !== undefined); + this.#catalog = new SimulatorCatalog({ + simctl: this.#simctl, + slim: this.#slim, + slimByDefault: this.#slimByDefault, + }); + this.#slimBoot = new SlimBoot({ + clock: options.clock, + filesystem: options.filesystem, + fullBootTimeoutMs: this.#fullBootTimeoutMs, + logger: options.logger, + onSlimmed: options.onSlimmed, + onSlimSkipped: options.onSlimSkipped, + overridesRoot: options.launchdOverridesRoot ?? DEFAULT_OVERRIDES_ROOT, + simctl: this.#simctl, + slim: this.#slim, + }); + this.#seeds = new Seeds({ + boot: this.#boot, + clock: options.clock, + slimBoot: this.#slimBoot, + database: options.database, + deviceRoot, + filesystem: options.filesystem, + fullBootTimeoutMs: this.#fullBootTimeoutMs, + idGenerator: options.idGenerator, + instanceId: options.instanceId, + logger: options.logger, + marks: this.#marks, + ownsDeviceSet: rootOptions !== undefined, + seedTurns: this.#seedTurns, + simctl: this.#simctl, + sourcesSet: sourcesSetFor(options.simlockHome, deviceRoot, rootOptions !== undefined), + }); this.screenCapture = createScreenCaptureFor( options, deviceRoot, @@ -369,7 +239,7 @@ export class IosSimctlDriver implements Driver { silent: options.config?.simulator?.audio === false, headless: options.config?.simulator?.headless !== false, commands: { - simctl: (args, timeoutMs) => this.#invokeSimctl(args, timeoutMs), + simctl: (args, timeoutMs) => this.#simctl.invoke(args, timeoutMs), open: (args, timeoutMs) => runCommand(this.#processRunner, this.#clock, "open", args, timeoutMs), }, @@ -437,142 +307,26 @@ export class IosSimctlDriver implements Driver { await ensureOwnedRoot(this.#rootOptions); } - async resolveSpec(request: ExactDeviceRequest): Promise { - requireIosPlatform(request.platform); - if (request.imageTag !== undefined) { - throw new UnsupportedRequestOptionError("ios", "imageTag"); - } - checkExcept(request.except ?? [], resolveSlimCategories(this.#slim?.categories)); - const catalog = await this.#loadCatalog(); - const deviceType = findDeviceType(catalog, request.model); - - if (deviceType === undefined) { - throw new IosUnknownModelError(request.model); - } - - const spec = - request.osVersion === undefined - ? this.#resolveDefaultRuntime(deviceType, catalog) - : this.#resolveExactRuntime(deviceType, request.osVersion, catalog); - // why: The resolution cache (`specKey`) ignores the mode, so the mode is applied here, outside it. - return request.mode === "slim" && this.#canSlim(spec) ? { ...spec, mode: "slim" } : spec; + resolveSpec(request: ExactDeviceRequest): Promise { + return this.#catalog.resolveSpec(request); } - #canSlim(spec: VirtualSpec): boolean { - const runtime = this.#resolvedSpecs.get(specKey(spec))?.runtime; - return ( - this.#slim !== undefined && - runtime !== undefined && - supportsPersistentSlim(iosRuntimeVersionFromId(runtime.identifier)) - ); + findComponent(component: string): Promise { + return this.#components.findComponent(component); } - /** - * why: Version requested explicitly: validated against the model's `[min, max]` pairing range - * *before* anything else -- an out-of-range request can never be fixed by downloading, so it - * is never reported as downloadable. - */ - #resolveExactRuntime( - deviceType: DeviceType, - osVersion: string, - catalog: SimctlCatalog, - ): VirtualSpec { - if (!isVersionInRange(osVersion, deviceType)) { - throw new IosVersionOutOfRangeError(deviceType.name, osVersion, deviceType); - } - - // why: Looked up among the runtimes the catalog lists for this model, so a listed pair always - // resolves and an unlisted one never does (ADR 0008 §3). Any installed build of the version - // that pairs will do: two builds can share a marketing version and pair differently. - const paired = findPairedRuntime(catalog, deviceType, osVersion); - if (paired !== undefined) { - return this.#commitResolution(deviceType, paired); - } - // why: Installed but not paired: in the model's declared `[min, max]` range is necessary but not - // sufficient, since `supportedDeviceTypeIds` is the authoritative source once a runtime is - // on disk. Refused before committing, so a mismatch never reaches `simctl create`. - if (findInstalledRuntime(catalog, osVersion) !== undefined) { - throw new IosRuntimeUnpairedError(deviceType.name, osVersion); - } - - throw missingRuntime( - osVersion, - `iOS ${osVersion} is not installed; pass --allow-download (or set downloads.policy) ` + - `to download it`, - osVersion, - ); - } - - #resolveDefaultRuntime(deviceType: DeviceType, catalog: SimctlCatalog): VirtualSpec { - const paired = newestRuntime(pairedInstalledRuntimes(catalog, deviceType)); - if (paired !== undefined) { - return this.#commitResolution(deviceType, paired); - } - - throw missingRuntime( - "default", - `No installed iOS runtime pairs with ${deviceType.name}; pass --allow-download (or ` + - `set downloads.policy) to download a compatible runtime`, - isUnboundedMax(deviceType.maxRuntimeVersion) - ? LATEST_COMPONENT - : majorVersionString(deviceType.maxRuntimeVersion), - ); - } - - #commitResolution(deviceType: DeviceType, runtime: Runtime): VirtualSpec { - const spec: VirtualSpec = { - model: deviceType.name, - osVersion: runtime.version, - physical: false, - platform: this.platform, - }; - this.#resolvedSpecs.set(specKey(spec), { deviceType, runtime, spec }); - return spec; - } - - async findComponent(component: string): Promise { - const [catalog, images] = await Promise.all([this.#loadCatalog(), this.#loadRuntimeImages()]); - const runtime = findInstalledRuntime(catalog, component); - return runtime === undefined ? undefined : installedComponent(runtime, images); - } - - async installComponent( + installComponent( component: string, options: { readonly onProgress: (progress: ComponentInstallProgress) => void; readonly signal: AbortSignal; }, ): Promise { - if (belowDownloadFloor(component)) { - throw new IosDownloadFloorError(component); - } - const before = await this.#installedReceipts(); - const args = - component === LATEST_COMPONENT - ? ["-downloadPlatform", "iOS"] - : ["-downloadPlatform", "iOS", "-buildVersion", component]; - try { - await this.#xcodebuildOrThrow(args, options); - } catch (error: unknown) { - throw withBareMajorHint(component, error); - } - - const [catalog, images] = await Promise.all([this.#loadCatalog(), this.#loadRuntimeImages()]); - const runtime = runtimeForComponent(catalog, component); - if (runtime === undefined) { - throw new DriverCrashError( - `xcodebuild reported success but iOS ${component} is still not installed`, - ); - } - const installed = installedComponent(runtime, images); - const wasThere = before.some((receipt) => sameReceipt(receipt, installed.receipt)); - return { ...installed, outcome: wasThere ? "already-installed" : "installed" }; + return this.#components.installComponent(component, options); } - async listComponents(): Promise { - return (await this.#listedImages()).map( - ({ image: _image, unusedDevices: _unused, ...component }) => component, - ); + listComponents(): Promise { + return this.#components.listComponents(); } async removeComponent( @@ -580,333 +334,26 @@ export class IosSimctlDriver implements Driver { options: { readonly signal: AbortSignal }, ): Promise { return removeListedComponent({ - list: () => this.#listedImages(), + list: () => this.#components.listedImages(), platform: this.platform, receipt, - remove: async ({ image, unusedDevices, version }) => { - await this.#destroySeedsOf(image.runtimeIdentifier); - // why: the budget is for the runtime's own deletion; a seed held by a late clone waits outside it. - const deadline = this.#clock.now() + COMPONENT_REMOVAL_TIMEOUT_MS; - await this.#deleteRuntimeImage(image.identifier, options.signal); - await this.#waitWhileDeleting(image.identifier, version, deadline, options.signal); - const sentences = [ - unavailableDevicesResidue(unusedDevices), - (await this.#downloadResidue(image, version)).residue, - ].filter((sentence) => sentence !== undefined); - return sentences.length === 0 ? {} : { residue: sentences.join(". ") }; + remove: async (listed) => { + await this.#seeds.destroySeedsOf(listed.image.runtimeIdentifier); + return this.#components.removeImage(listed, options.signal); }, }); } - /** why: ADR 0061 §3: a seed of a removed runtime can never be copied again; a stuck one is logged and the removal goes on. */ - async #destroySeedsOf(runtimeId: string): Promise { - for (const row of this.#sources.readyRows()) { - if (row.kind !== "seed" || row.runtimeId !== runtimeId || row.udid === undefined) continue; - const udid = row.udid; - try { - await this.#deleteSeedInTurn(udid); - this.#sources.deleteRow(row.sourceId); - } catch (error: unknown) { - this.#logger?.warn( - `could not remove seed ${udid} of removed runtime ${runtimeId}: ${errorMessage(error)}`, - ); - } - } - } - - async #deleteRuntimeImage(identifier: string, signal: AbortSignal): Promise { - const args = ["runtime", "delete", identifier]; - const command = `simctl ${args.join(" ")}`; - let outcome; - try { - outcome = await runBoundedProcess( - this.#processRunner, - this.#clock, - "xcrun", - this.#scopedSimctlArgv(args), - { signal, timeoutMs: COMPONENT_REMOVAL_TIMEOUT_MS }, - ); - } catch (error: unknown) { - throw new DriverCrashError(`${command} failed: ${errorMessage(error)}`); - } - if (outcome.timedOut) { - throw new DriverCrashError( - `${command} timed out after ${String(COMPONENT_REMOVAL_TIMEOUT_MS)}ms`, - ); - } - if (outcome.stopped) throw new DriverCrashError(`${command} was ended before it finished`); - this.#assertSuccessful(args, outcome.result); - } - - async #waitWhileDeleting( - identifier: string, - version: string, - deadline: number, - signal: AbortSignal, - ): Promise { - for (;;) { - const listed = (await this.#loadRuntimeImages()).find( - (image) => image.identifier === identifier, - ); - if (listed === undefined || !isBeingDeleted(listed)) return; - const remainingMs = deadline - this.#clock.now(); - if (remainingMs <= 0) { - throw new DriverCrashError( - `${this.platform} ${version} was still being deleted ` + - `${String(COMPONENT_REMOVAL_TIMEOUT_MS)}ms after its removal started`, - ); - } - if (!(await this.#pause(Math.min(DELETING_POLL_MS, remainingMs), signal))) { - throw new DriverCrashError( - `The removal of ${this.platform} ${version} was ended while it was still being deleted`, - ); - } - } - } - - #pause(delayMs: number, signal: AbortSignal): Promise { - if (signal.aborted) return Promise.resolve(false); - return new Promise((resolve) => { - const onAbort = (): void => { - this.#clock.cancel(timer); - resolve(false); - }; - const timer = this.#clock.setTimer(delayMs, () => { - signal.removeEventListener("abort", onAbort); - resolve(true); - }); - signal.addEventListener("abort", onAbort, { once: true }); - }); - } - - async #downloadResidue(image: RuntimeImage, version: string): Promise<{ residue?: string }> { - const left = (await this.#downloadedRuntimeAssets()).filter((asset) => - image.build === undefined ? asset.version === version : asset.build === image.build, - ); - if (left.length === 0) return {}; - const label = image.build === undefined ? version : `${version} (${image.build})`; - return { - residue: leftoverDownloadsSentence([label], left.length, await this.#assetsSize(left)), - }; - } - - async #assetsSize(assets: readonly RuntimeAsset[]): Promise { - let total = 0; - for (const asset of assets) { - try { - total += await this.#filesystem.directorySize(asset.path); - } catch { - return undefined; - } - } - return total; - } - - async #listedImages(): Promise { - const [images, defaultSet] = await Promise.all([ - this.#loadRuntimeImages(), - this.#legacySimctl(["list", "-j", "devices"], COMMAND_TIMEOUT_MS), - ]); - let foreign: ReadonlyMap; - try { - const parsed = JSON.parse(defaultSet.stdout) as unknown; - foreign = devicesPerRuntime(parsed, await this.#ownedUdids(parsed)); - } catch (error: unknown) { - if (error instanceof DriverCrashError) throw error; - throw new DriverCrashError(`Could not parse simctl device list: ${errorMessage(error)}`); - } - return images.filter(isIosImage).flatMap((image): ListedImage[] => - image.version === undefined - ? [] - : [ - { - foreignDevices: foreign.get(image.runtimeIdentifier)?.used ?? 0, - image, - unusedDevices: images.some( - (other) => - other !== image && - isIosImage(other) && - other.runtimeIdentifier === image.runtimeIdentifier, - ) - ? 0 - : (foreign.get(image.runtimeIdentifier)?.unused ?? 0), - receipt: imageReceipt(image), - version: image.version, - ...(image.build === undefined ? {} : { variant: image.build }), - ...(image.sizeBytes === undefined ? {} : { sizeBytes: image.sizeBytes }), - }, - ], - ); - } - - async #installedReceipts(): Promise { - const [catalog, images] = await Promise.all([this.#loadCatalog(), this.#loadRuntimeImages()]); - return catalog.runtimes - .filter((runtime) => runtime.isAvailable) - .map((runtime) => runtimeReceipt(runtime, images)); - } - - async #xcodebuildOrThrow( - args: readonly string[], - options: { - readonly onProgress: (progress: ComponentInstallProgress) => void; - readonly signal: AbortSignal; - }, - ): Promise { - let outcome; - try { - outcome = await runInstallerProcess( - this.#processRunner, - this.#clock, - "xcodebuild", - args, - options, - ); - } catch (error: unknown) { - throw new DriverCrashError(`xcodebuild ${args.join(" ")} failed: ${errorMessage(error)}`); - } - if (outcome.stopped) { - throw new DriverCrashError(`xcodebuild ${args.join(" ")} was ended before it finished`); - } - const { result } = outcome; - if (result.code === 0) { - return; - } - const message = result.stderr || result.stdout; - // why: Apple's downloader accepts `-buildVersion ` only for whatever it - // currently offers -- a pin naming a release Apple has since retired from its live catalog - // (superseded by a newer point release), or one still ambiguous across several concurrent - // beta/seed builds sharing the same marketing version, is rejected outright with "is not - // available for download", distinct from every other `xcodebuild` failure this method - // otherwise reports as a `DriverCrashError`. Falling back to an unpinned `-downloadPlatform - // iOS` here would silently substitute whatever Apple currently offers instead -- a different, - // unrequested, and possibly multi-gigabyte runtime -- for a request that named an exact - // version, so this fails the request instead of guessing; see - // `IosRuntimeUnavailableForDownloadError`. - const buildVersionIndex = args.indexOf("-buildVersion"); - if (buildVersionIndex !== -1 && /is not available for download/.test(message)) { - throw new IosRuntimeUnavailableForDownloadError( - args[buildVersionIndex + 1] ?? "requested version", - ); - } - throw new DriverCrashError(`xcodebuild ${args.join(" ")} failed: ${message}`); - } - - async listSources(): Promise { - await this.#sweepLeftovers(); - const rows = this.#sources.readyRows(); - const orphans = await this.#orphanSources(); - if (rows.length === 0) return orphans; - const catalog = await this.#loadCatalog(); - return [ - ...rows.map((row) => { - const staleReason = sourceStaleReason(row, catalog); - return { - id: row.sourceId, - kind: row.kind, - model: row.model, - osVersion: row.osVersion, - ...(row.sizeBytes === undefined ? {} : { sizeBytes: row.sizeBytes }), - ...(staleReason === undefined ? {} : { staleReason }), - }; - }), - ...orphans, - ]; - } - - async saveSource(device: VirtualDriverDevice): Promise<{ sourceId: string; sizeBytes?: number }> { - const data = iosDriverData(device); - const catalog = await this.#loadCatalog(); - const runtime = catalog.runtimes.find((entry) => entry.identifier === data.runtimeId); - const deviceType = catalog.deviceTypes.find((entry) => entry.identifier === data.deviceTypeId); - if (runtime === undefined || deviceType === undefined) { - throw new DriverCrashError( - `device ${data.udid} was made with a runtime or device type that is gone`, - ); - } - const sourceId = this.#idGenerator.generate(); - this.#sources.insertMaking({ - createdAt: this.#clock.now(), - deviceTypeId: data.deviceTypeId, - kind: "baseline", - maker: this.#maker(), - model: deviceType.name, - osVersion: runtime.version, - runtimeBuild: runtime.build, - runtimeId: data.runtimeId, - setPath: this.#sourcesSet, - sourceId, - specKey: `baseline-${sourceId}`, - }); - let cloneUdid: string | undefined; - try { - await this.#ensureSourcesSet(); - await this.#shutdown(data.udid); - const cloned = await this.#createSimulator( - ["clone", data.udid, `simlock-baseline-${sourceId}`, this.#sourcesSet], - { destination: this.#sourcesSet }, - ); - cloneUdid = cloned.stdout.trim(); - if (cloneUdid === "") throw new DriverCrashError("simctl clone returned no device UDID"); - this.#sources.setUdid(sourceId, cloneUdid); - await this.#copyOverrides(data.udid, cloneUdid); - await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); - const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes); - return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; - } catch (error: unknown) { - if (cloneUdid !== undefined && cloneUdid !== "") { - await this.#deleteSeedSimulator(cloneUdid).catch(() => undefined); - } - this.#sources.deleteRow(sourceId); - throw error; - } - } - - async destroySource(sourceId: string): Promise { - const row = this.#sources.findRow(sourceId); - if (row === undefined) { - // why: safety rule 1 (ADR 0061 §1): a UDID no row names is deleted only when it is a simulator folder of the sources set. - if (!(await this.#isOrphanSource(sourceId))) { - throw new DriverCrashError(`iOS keeps no source ${sourceId}`); - } - await this.#deleteSeedInTurn(sourceId); - return; - } - if (row.udid !== undefined) { - const udid = row.udid; - await this.#deleteSeedInTurn(udid); - } - this.#sources.deleteRow(sourceId); + listSources(): Promise { + return this.#seeds.listSources(); } - /** why: a simulator in the sources set that no row names is a leftover of a crash; `doctor` reports it and only `--purge-orphans` deletes it. */ - async #orphanSources(): Promise { - const named = new Set(this.#sources.allUdids()); - return (await this.#sourceFolders()) - .filter((udid) => !named.has(udid)) - .map((udid) => ({ - id: udid, - kind: "seed", - model: "unknown", - orphan: true, - osVersion: "unknown", - })); + saveSource(device: VirtualDriverDevice): Promise<{ sourceId: string; sizeBytes?: number }> { + return this.#seeds.saveSource(device); } - async #isOrphanSource(udid: string): Promise { - return (await this.#sourceFolders()).includes(udid) && !this.#sources.allUdids().includes(udid); - } - - async #sourceFolders(): Promise { - try { - return (await this.#filesystem.readdir(this.#sourcesSet)).filter((entry) => - UDID_PATTERN.test(entry), - ); - } catch (error: unknown) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } + destroySource(sourceId: string): Promise { + return this.#seeds.destroySource(sourceId); } async provision( @@ -914,237 +361,8 @@ export class IosSimctlDriver implements Driver { options?: { readonly source?: string }, ): Promise { requireIosPlatform(spec.platform); - const resolved = await this.#resolvedSpec(spec); - const seed = - options?.source === undefined - ? await this.#seedFor(resolved) - : await this.#namedSource(options.source); - // why: Cosmetic, and deliberately so: what proves this device is Simlock's is that it - // lives in the device set every call below is scoped to, never what it is called - // (safety rule 8). The name exists because it is what a human reads in `simctl list` - // and in the simulator's window title. - const name = `simlock-${this.#idGenerator.generate()}`; - const result = await this.#seedTurns.take(seed.udid, () => - this.#createSimulator(["clone", seed.udid, name, this.#deviceRoot], { - seedUdid: seed.udid, - source: this.#sourcesSet, - }), - ); - const udid = result.stdout.trim(); - - if (udid === "") { - throw new DriverCrashError("simctl clone returned no device UDID"); - } - - try { - await this.#writeMark(udid); - await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); - } catch (error: unknown) { - // why: a clone whose mark or database drop failed is not a device Simlock hands out, and - // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. - await this.#invokeSimctl(["delete", udid], COMMAND_TIMEOUT_MS).catch(() => undefined); - throw error; - } - await this.#copyOverrides(seed.udid, udid); - - return { - address: udid, - deviceId: udid, - driverData: { - deviceTypeId: resolved.deviceType.identifier, - name, - runtimeId: resolved.runtime.identifier, - udid, - } satisfies IosDriverData, - }; - } - - async #namedSource(sourceId: string): Promise<{ readonly udid: string }> { - const row = this.#sources.findRow(sourceId); - if (row?.state !== "ready" || row.udid === undefined) { - throw new DriverCrashError(`iOS keeps no ready source ${sourceId}`); - } - const reason = sourceStaleReason(row, await this.#loadCatalog()); - if (reason !== undefined) throw new DriverCrashError(`source ${sourceId} is stale: ${reason}`); - return { udid: row.udid }; - } - - /** why: two provisions of one kind at once make one seed, the second waiting for the first's make. */ - #seedFor(resolved: ResolvedIosSpec): Promise { - const specKey = seedSpecKey(resolved); - const running = this.#seedMakes.get(specKey); - if (running !== undefined) return running; - const made = this.#findOrMakeSeed(resolved, specKey).finally(() => { - this.#seedMakes.delete(specKey); - }); - this.#seedMakes.set(specKey, made); - return made; - } - - async #deleteSeedInTurn(udid: string): Promise { - await this.#seedTurns.takeToDelete(udid, () => this.#deleteSeedSimulator(udid)); - } - - async #findOrMakeSeed(resolved: ResolvedIosSpec, specKey: string): Promise { - const seed = this.#sources.findReadySeed(specKey); - if (seed !== undefined && !seedIsStale(seed, resolved)) return seed; - if (seed !== undefined) await this.#destroySeed(seed); - return this.#makeSeed(resolved, specKey); - } - - async #makeSeed(resolved: ResolvedIosSpec, specKey: string): Promise { - // why: a `making` row of this process older than the longest a make can take belongs to a make that never ended. - for (const row of this.#sources.makingRowsOfKindBefore( - this.#maker(), - specKey, - this.#clock.now() - STALE_MAKING_MS, - )) { - await this.#removeLeftover(row); - } - const sourceId = this.#idGenerator.generate(); - this.#sources.insertMaking({ - createdAt: this.#clock.now(), - deviceTypeId: resolved.deviceType.identifier, - kind: "seed", - maker: this.#maker(), - model: resolved.spec.model, - osVersion: resolved.spec.osVersion, - runtimeBuild: resolved.runtime.build, - runtimeId: resolved.runtime.identifier, - setPath: this.#sourcesSet, - sourceId, - specKey, - }); - try { - await this.#ensureSourcesSet(); - const created = await this.#createSimulator( - [ - "create", - `simlock-seed-${sourceId}`, - resolved.deviceType.identifier, - resolved.runtime.identifier, - ], - { destination: this.#sourcesSet, source: this.#sourcesSet }, - ); - const udid = created.stdout.trim(); - this.#sources.setUdid(sourceId, udid); - try { - await this.#bootAndWait(udid, sourceId, this.#fullBootTimeoutMs, this.#sourcesSet); - } catch (error: unknown) { - await this.#deleteSeedSimulator(udid); - // why: a seed that ran out of boot time is a boot timeout, as an Android seed's is. - if (error instanceof BootTimeoutError) throw error; - throw new DriverCrashError(`seed ${udid} did not boot: ${errorMessage(error)}`); - } - try { - await this.#shutdown(udid, this.#sourcesSet); - } catch (error: unknown) { - await this.#deleteSeedSimulator(udid); - throw error; - } - this.#sources.markReady(sourceId, await this.#sizeOf(udid)); - return { sourceId, udid, runtimeBuild: resolved.runtime.build }; - } catch (error: unknown) { - this.#sources.deleteRow(sourceId); - throw error; - } - } - - /** - * why: ADR 0061 §1: a `making` row another process left is a seed that process never finished, so - * its simulator (if `create` answered) and row go at the first source read of this process. A - * simulator no row names is left in place: `doctor` reports it. - */ - async #sweepLeftovers(): Promise { - if (this.#swept) return; - this.#swept = true; - for (const row of this.#sources.makingRowsNotMadeBy(this.#maker())) { - await this.#removeLeftover(row); - } - } - - async #removeLeftover(row: SourceRow): Promise { - try { - if (row.udid !== undefined) await this.#deleteSeedSimulator(row.udid); - this.#sources.deleteRow(row.sourceId); - } catch (error: unknown) { - this.#logger?.warn( - `could not remove leftover source ${row.udid ?? row.sourceId}: ${errorMessage(error)}`, - ); - } - } - - #maker(): string { - this.#sourceMaker ??= this.#idGenerator.generate(); - return this.#sourceMaker; - } - - /** why: ADR 0060 §1: the slim file is in place before the copy's first boot, so the copy boots with what the seed was given. */ - async #copyOverrides(seedUdid: string, copyUdid: string): Promise { - const from = join(this.#overridesFolder(seedUdid), OVERRIDES_FILE_NAME); - try { - if (!(await this.#filesystem.exists(from))) return; - await this.#filesystem.mkdirp(this.#overridesFolder(copyUdid), { mode: 0o700 }); - await this.#filesystem.copyFile( - from, - join(this.#overridesFolder(copyUdid), OVERRIDES_FILE_NAME), - ); - } catch (error: unknown) { - this.#logger?.warn( - `could not copy the launchd overrides of seed ${seedUdid} to ${copyUdid}: ${errorMessage(error)}`, - ); - } - } - - /** - * why: `simctl clone` keeps the LaunchServices database, which records each app (system apps - * included) by its absolute path under the original device's folder; a copy then uses the - * original's data, or fails once that folder is gone. Dropped from every clone before its first - * boot, LaunchServices rebuilds it from the clone's own container folders (ADR 0061 §3). - */ - async #dropLaunchServicesDatabase(setPath: string, udid: string): Promise { - const folder = join(setPath, udid, "data", "var", "db", "lsd"); - if (!(await this.#filesystem.exists(folder))) return; - for (const entry of await this.#filesystem.readdir(folder)) { - if (LAUNCH_SERVICES_DATABASE.test(entry)) await this.#filesystem.rm(join(folder, entry)); - } - } - - async #destroySeed(seed: ReadySeed): Promise { - await this.#deleteSeedInTurn(seed.udid); - this.#sources.deleteRow(seed.sourceId); - } - - async #deleteSeedSimulator(udid: string): Promise { - await this.#bestEffortShutdown(udid, this.#sourcesSet); - const args = ["delete", udid]; - const outcome = await this.#invokeSimctl(args, COMMAND_TIMEOUT_MS, this.#sourcesSet); - // why: a seed deleted by hand is already gone, and its row must still be dropped. - if (outcome.kind === "finished" && unknownDevice(outcome.result)) return; - this.#checked(args, COMMAND_TIMEOUT_MS, outcome); - } - - async #ensureSourcesSet(): Promise { - if (this.#rootOptions !== undefined) { - await this.#filesystem.mkdirp(this.#sourcesSet, { mode: 0o700 }); - return; - } - await ensureOwnedRoot({ - filesystem: this.#filesystem, - idGenerator: this.#idGenerator, - instanceId: this.#instanceId, - path: this.#sourcesSet, - platform: "ios", - }); - } - - async #sizeOf(udid: string): Promise { - try { - return await this.#filesystem.directorySize(join(this.#sourcesSet, udid)); - } catch { - // why: a size that cannot be counted leaves the seed `ready` with no size; `doctor` then leaves it out. - return undefined; - } + const resolved = await this.#catalog.resolvedSpec(spec); + return this.#seeds.provision(resolved, options); } async makeReady( @@ -1156,19 +374,14 @@ export class IosSimctlDriver implements Driver { }, ): Promise { const data = iosDriverData(device); - if (this.#rootOptions === undefined) await this.#disablePasteboardSync(data.udid); - const { plan, bootstatusTimeoutMs } = planSlimBoot( - data, - this.#slim, - this.#fullBootTimeoutMs, - options, - ); + if (this.#rootOptions === undefined) await this.#deviceHub.disablePasteboardSync(data.udid); + const { plan, bootstatusTimeoutMs } = this.#slimBoot.plan(data, options); const write = plan.kind === "apply" - ? await this.#writeOverrides(data.udid, plan.slim, options.except) + ? await this.#slimBoot.writeOverrides(data.udid, plan.slim, options.except) : undefined; - await this.#bootAndWait( + await this.#boot.bootAndWait( data.udid, device.deviceId, bootstatusTimeoutMs, @@ -1176,218 +389,7 @@ export class IosSimctlDriver implements Driver { options.purpose === "prepare", ); await runAfterBoot(this.#afterBoot, data.udid, device.deviceId); - - if (plan.kind === "skip") { - this.#onSlimSkipped?.({ - deviceId: device.deviceId, - detail: plan.detail, - reason: plan.reason, - }); - } - - if (write === undefined) { - return this.#asIs(device, data, "full"); - } - if (write.kind === "no-categories") { - return this.#skipApply(device, data, write.detail, "full"); - } - - return this.#readBack(device, data, write); - } - - #asIs(device: DriverDevice, data: IosDriverData, mode: DeviceMode): DriverDevice { - return { - address: data.udid, - deviceId: device.deviceId, - driverData: device.driverData, - mode, - }; - } - - async #bootAndWait( - udid: string, - deviceId: string, - bootstatusTimeoutMs: number, - set?: string, - refreshBooted = false, - ): Promise { - const deadline = this.#clock.now() + bootstatusTimeoutMs; - const boot = await this.#boot(udid, deadline, set, refreshBooted); - if (boot.kind === "timed-out") { - throw new BootTimeoutError(deviceId); - } - if (boot.result.code !== 0 && !alreadyBooted(boot.result.stderr)) { - this.#assertSuccessful(["boot", udid], boot.result); - } - const outcome = await this.#invokeSimctl(["bootstatus", udid, "-b"], this.#left(deadline), set); - if (outcome.kind === "finished") { - this.#assertSuccessful(["bootstatus", udid, "-b"], outcome.result); - if (await this.#waitForInstaller(udid, deadline, set)) return; - } - - await this.#bestEffortShutdown(udid, set); - throw new BootTimeoutError(deviceId); - } - - // why: a booted simulator keeps the clipboard sync it booted with, so a lease's device that is - // already up is shut down and booted again once DeviceHub's preference is set. - async #boot(udid: string, deadline: number, set: string | undefined, refreshBooted: boolean) { - const boot = await this.#invokeSimctl(["boot", udid], this.#left(deadline), set); - const refresh = - refreshBooted && - this.#rootOptions === undefined && - set === undefined && - boot.kind === "finished" && - boot.result.code !== 0 && - alreadyBooted(boot.result.stderr); - if (!refresh) return boot; - await this.#shutdown(udid); - return await this.#invokeSimctl(["boot", udid], this.#left(deadline)); - } - - #left(deadline: number): number { - return Math.max(0, deadline - this.#clock.now()); - } - - async #waitForInstaller(udid: string, deadline: number, set?: string): Promise { - // why: bootstatus can finish before the installer accepts requests after an erase (#547). - while (this.#clock.now() < deadline) { - const outcome = await this.#invokeSimctl( - ["listapps", udid], - Math.min(COMMAND_TIMEOUT_MS, deadline - this.#clock.now()), - set, - ); - if (outcome.kind === "finished" && outcome.result.code === 0) { - return this.#clock.now() < deadline; - } - const remainingMs = deadline - this.#clock.now(); - if (remainingMs <= 0) break; - await new Promise((resolve) => { - this.#clock.setTimer(Math.min(INSTALLER_POLL_MS, remainingMs), resolve); - }); - } - return false; - } - - async #writeOverrides( - udid: string, - slim: SlimOptions, - except: readonly string[] = [], - ): Promise { - const configured = resolveSlimCategories(slim.categories); - const { kept, resolved } = splitByExcept(configured, except); - if (configured.categories.length === 0) { - return { - detail: `none of the configured slim categories (${(slim.categories ?? []).join(", ")}) are known`, - kind: "no-categories", - }; - } - - const startedAt = this.#clock.now(); - const wanted = overridesFor(resolved.categories, kept); - let writeError: string | undefined; - try { - const folder = this.#overridesFolder(udid); - const file = join(folder, OVERRIDES_FILE_NAME); - const existing = await this.#readOverrides(file); - const merged = new Map([...existing, ...wanted]); - if (!sameEntries(existing, merged)) { - await this.#filesystem.mkdirp(folder, { mode: 0o700 }); - await this.#filesystem.writeFileAtomic(file, renderOverridesPlist(merged), { mode: 0o644 }); - } - } catch (error: unknown) { - writeError = `could not write the launchd overrides file: ${errorMessage(error)}`; - } - - return { kept, kind: "written", resolved, startedAt, wanted, writeError }; - } - - async #readOverrides(file: string): Promise> { - if (!(await this.#filesystem.exists(file))) return new Map(); - const parsed = parseOverridesPlist(await this.#filesystem.readFile(file)); - if (parsed === undefined) - throw new Error("the existing file is not a property list dictionary"); - return parsed; - } - - #overridesFolder(udid: string): string { - return join(this.#overridesRoot, `${OVERRIDES_FOLDER_PREFIX}${udid}`); - } - - async #readBack( - device: DriverDevice, - data: IosDriverData, - write: Extract, - ): Promise { - if (write.kept.length > 0 && write.writeError !== undefined) { - throw new KeptCategoriesError(write.writeError); - } - const outcome = await this.#invokeSimctl( - ["spawn", data.udid, "launchctl", "print-disabled", "system"], - READBACK_TIMEOUT_MS, - ); - const details = write.writeError === undefined ? [] : [write.writeError]; - - if (outcome.kind === "timed-out" || outcome.result.code !== 0) { - details.push(`could not read back the launchd overrides: ${readBackFailure(outcome)}`); - if (write.kept.length > 0) throw new KeptCategoriesError(details.join("; ")); - return this.#skipApply(device, data, details.join("; "), "slim"); - } - return this.#judgeReadBack(device, data, write, details, outcome.result.stdout); - } - - #judgeReadBack( - device: DriverDevice, - data: IosDriverData, - write: Extract, - details: string[], - stdout: string, - ): DriverDevice { - const read = parsePrintDisabled(stdout); - assertKeptEnabled(write.kept, read); - const mismatched = [...write.wanted.keys()] - .sort() - .filter((label) => (read.get(label) ?? false) !== write.wanted.get(label)); - if (mismatched.length > 0) { - details.push( - `${String(mismatched.length)} of ${String(write.wanted.size)} managed labels did not read back as written at boot; first: ${mismatched[0] ?? ""}`, - ); - return this.#skipApply(device, data, details.join("; "), "full"); - } - if (details.length > 0) { - return this.#skipApply(device, data, details.join("; "), "slim"); - } - - // why: an except list naming the whole slim set slims nothing: full, no `device.slimmed`. - if (write.resolved.categories.length === 0) return this.#asIs(device, data, "full"); - this.#reportSlimmed(device, data, write); - return this.#asIs(device, data, "slim"); - } - - #reportSlimmed( - device: DriverDevice, - data: IosDriverData, - write: Extract, - ): void { - this.#onSlimmed?.({ - address: data.udid, - categories: write.resolved.categories.map((category) => category.name), - deviceId: device.deviceId, - durationMs: this.#clock.now() - write.startedAt, - labelCount: [...write.wanted.values()].filter(Boolean).length, - signature: slimSignature(write.resolved.categories), - unknownLabels: [...write.resolved.unknown], - }); - } - - #skipApply( - device: DriverDevice, - data: IosDriverData, - detail: string, - mode: DeviceMode, - ): DriverDevice { - this.#onSlimSkipped?.({ deviceId: device.deviceId, detail, reason: "apply-failed" }); - return this.#asIs(device, data, mode); + return this.#slimBoot.settle(device, data, plan, write); } async reclaim( @@ -1407,85 +409,23 @@ export class IosSimctlDriver implements Driver { } async shutdown(device: DriverDevice): Promise { - await this.#shutdown(iosDriverData(device).udid); + await this.#simctl.shutdown(iosDriverData(device).udid); } async destroy(device: DriverDevice): Promise { const data = iosDriverData(device); - await this.#shutdown(data.udid); - await this.#simctl(["delete", data.udid], COMMAND_TIMEOUT_MS); - if (this.#rootOptions === undefined) await this.#removePasteboardPreference(data.udid); - await this.#removeOverrides(data.udid); - } - - // why: DeviceHub's preferences belong to cfprefsd, which keeps its own copy and writes it back - // over the file, so an edit made to the file alone is lost or ignored. The domain is exported - // to a scratch file, edited there with PlistBuddy, and imported back through cfprefsd. - async #editDeviceHubPreferences( - edit: (buddy: (command: string) => Promise) => Promise, - ): Promise { - return await withPasteboardWrite(async () => { - await this.#filesystem.mkdirp(this.#overridesRoot, { mode: 0o700 }); - const staged = join(this.#overridesRoot, DEVICEHUB_STAGING_FILE); - const run = async (command: string, args: string[]): Promise => { - const outcome = await runCommand( - this.#processRunner, - this.#clock, - command, - args, - COMMAND_TIMEOUT_MS, - ); - return outcome.kind !== "timed-out" && outcome.result.code === 0; - }; - try { - if (!(await run("defaults", ["export", DEVICEHUB_DOMAIN, staged]))) return false; - const buddy = (command: string) => run("/usr/libexec/PlistBuddy", ["-c", command, staged]); - return (await edit(buddy)) && (await run("defaults", ["import", DEVICEHUB_DOMAIN, staged])); - } finally { - await this.#filesystem.rm(staged).catch(() => undefined); - } - }); - } - - async #disablePasteboardSync(udid: string): Promise { - const path = `:DevicePreferences:${udid}`; - const done = await this.#editDeviceHubPreferences(async (buddy) => { - if (await buddy(`Set ${path}:pasteboardSyncEnabled false`)) return true; - await buddy("Add :DevicePreferences dict"); - await buddy(`Add ${path} dict`); - return await buddy(`Add ${path}:pasteboardSyncEnabled bool false`); - }); - if (!done) throw new DriverCrashError(`Could not disable DeviceHub clipboard sync for ${udid}`); - } - - async #removePasteboardPreference(udid: string): Promise { - const done = await this.#editDeviceHubPreferences((buddy) => - buddy(`Delete :DevicePreferences:${udid}`), - ).catch(() => false); - if (!done) { - this.#logger?.warn( - `could not remove DeviceHub preference for ${udid}: preference deletion failed`, - ); - } - } - - // why: simctl delete leaves the folder behind; a failed removal must not fail a destroy that - // already deleted the simulator. - async #removeOverrides(udid: string): Promise { - try { - await this.#filesystem.rm(this.#overridesFolder(udid)); - } catch (error: unknown) { - this.#logger?.warn( - `could not remove the launchd overrides folder of ${udid}: ${errorMessage(error)}`, - ); - } + await this.#simctl.shutdown(data.udid); + await this.#simctl.run(["delete", data.udid], COMMAND_TIMEOUT_MS); + if (this.#rootOptions === undefined) + await this.#deviceHub.removePasteboardPreference(data.udid); + await this.#slimBoot.removeOverrides(data.udid); } async findLegacy(driverDeviceId: string): Promise { // why: Under Xcode's set the default set is the driver's own, so no device is "pre-root" // there, and a lost mark stays a report-only finding instead of a deletable legacy device. if (this.#rootOptions === undefined) return undefined; - const result = await this.#legacySimctl(["list", "-j", "devices"], COMMAND_TIMEOUT_MS); + const result = await this.#simctl.runLegacy(["list", "-j", "devices"], COMMAND_TIMEOUT_MS); const found = parseManagedDevices(JSON.parse(result.stdout) as unknown).find( (device) => device.udid === driverDeviceId, ); @@ -1511,9 +451,9 @@ export class IosSimctlDriver implements Driver { async destroyLegacy(device: DriverDevice): Promise { const { udid } = iosDriverData(device); // why: Best effort: a device that is already shut down reports a failure unrelated to the delete. - await this.#invokeLegacySimctl(["shutdown", udid], COMMAND_TIMEOUT_MS); - await this.#legacySimctl(["delete", udid], COMMAND_TIMEOUT_MS); - await this.#removeOverrides(udid); + await this.#simctl.invokeLegacy(["shutdown", udid], COMMAND_TIMEOUT_MS); + await this.#simctl.runLegacy(["delete", udid], COMMAND_TIMEOUT_MS); + await this.#slimBoot.removeOverrides(udid); } async listPresentPhysical(enrolledIds: readonly string[]): Promise { @@ -1529,7 +469,7 @@ export class IosSimctlDriver implements Driver { } async listManaged(): Promise { - const result = await this.#simctl(["list", "-j", "devices"], COMMAND_TIMEOUT_MS); + const result = await this.#simctl.run(["list", "-j", "devices"], COMMAND_TIMEOUT_MS); const listed = parseManagedDevices(JSON.parse(result.stdout) as unknown); // why: Xcode's set also holds the user's own simulators and other Simlocks'; only a // durable mark naming this instance proves a simulator here is ours. @@ -1539,13 +479,13 @@ export class IosSimctlDriver implements Driver { : ( await Promise.all( listed.map(async (device) => - (await this.#isMarkedByThisInstance(device.udid)) ? [device] : [], + (await this.#marks.isMarkedByThisInstance(device.udid)) ? [device] : [], ), ) ).flat(); const devices: ObservedDevice[] = await Promise.all( parsed.map(async (device) => { - const mark = await this.#readMark(device.udid); + const mark = await this.#marks.readMark(device.udid); return { address: device.udid, deviceId: device.udid, @@ -1564,31 +504,8 @@ export class IosSimctlDriver implements Driver { return { devices, processes }; } - async listCatalog(): Promise { - const catalog = await this.#loadCatalog(); - const installedRuntimes = catalog.runtimes.filter((runtime) => runtime.isAvailable); - const models = catalog.deviceTypes.map((deviceType) => deviceType.name); - return { - defaultRuntime: newestRuntime(installedRuntimes)?.version, - // why: Keyed by name and paired through the device type `resolveSpec` would pick for that name, - // so two device types that differ only in letter case both list the first one's runtimes. - // A device type answers to its name only, in any letter case (`findDeviceType`). - modelAliases: {}, - modelClasses: Object.fromEntries( - catalog.deviceTypes.flatMap((deviceType) => { - const deviceClass = PRODUCT_FAMILY_CLASSES.get(deviceType.productFamily); - return deviceClass === undefined ? [] : [[deviceType.name, deviceClass]]; - }), - ), - modelRuntimes: Object.fromEntries( - models.map((model) => [model, pairedVersions(catalog, findDeviceType(catalog, model))]), - ), - models, - runtimes: installedRuntimes.map((runtime) => runtime.version), - slimCategories: resolveSlimCategories(this.#slim?.categories).categories.map( - (category) => category.name, - ), - }; + listCatalog(): Promise { + return this.#catalog.listCatalog(); } estimate(estimate: DriverEstimate, spec: DeviceSpec): number { @@ -1604,120 +521,14 @@ export class IosSimctlDriver implements Driver { } async advisories(): Promise { - return [...(await this.#unreclaimableCacheAdvisories()), ...(await this.#slimAdvisories())]; - } - - async toolVersions(): Promise { - const result = await this.#processRunner.run("xcodebuild", ["-version"], { - timeoutMs: XCODE_VERSION_TIMEOUT_MS, - }); - if (result.code !== 0) { - if (/requires Xcode/.test(result.stderr)) return []; - throw new Error(`xcodebuild -version exited with ${String(result.code)}`); - } - const version = /^Xcode (\S+)$/m.exec(result.stdout)?.[1]; - const build = /^Build version (\S+)$/m.exec(result.stdout)?.[1]; - if (version === undefined) throw new Error("xcodebuild -version printed no Xcode version"); - return [{ name: "xcode", version, ...(build === undefined ? {} : { build }) }]; - } - - async #unreclaimableCacheAdvisories(): Promise { - const assets = await this.#downloadedRuntimeAssets(); - - if (assets.length === 0) { - return []; - } - - const installed = (await this.#loadCatalog()).runtimes.filter((runtime) => runtime.isAvailable); - const orphans = assets.filter( - (asset) => !installed.some((runtime) => runtimeMountsAsset(runtime, asset)), - ); - - if (orphans.length === 0) { - return []; - } - - // why: Oldest first, by version and then by build, so a store that has been collecting for - // months reads in the order the downloads happened rather than as a lexicographic jumble. - const described = [ - ...new Map( - orphans.map((asset) => [`${asset.version} (${asset.build})`, asset] as const), - ).entries(), - ] - .sort( - ([, left], [, right]) => - compareVersions(left.version, right.version) || left.build.localeCompare(right.build), - ) - .map(([label]) => label); return [ - { - code: "runtime-cache-unreclaimable", - message: leftoverDownloadsSentence( - described, - orphans.length, - await this.#assetsSize(orphans), - ), - }, + ...(await this.#components.cacheAdvisories()), + ...(await this.#catalog.slimAdvisories()), ]; } - async #downloadedRuntimeAssets(): Promise { - let bundles: readonly string[]; - try { - bundles = await this.#filesystem.readdir(IOS_RUNTIME_ASSET_ROOT); - } catch { - return []; - } - - const assets: RuntimeAsset[] = []; - for (const bundle of bundles.filter((name) => name.endsWith(".asset"))) { - let contents: string; - try { - contents = await this.#filesystem.readFile( - join(IOS_RUNTIME_ASSET_ROOT, bundle, "Info.plist"), - ); - } catch { - continue; - } - - const asset = parseRuntimeAsset(join(IOS_RUNTIME_ASSET_ROOT, bundle), contents); - if (asset !== undefined) { - assets.push(asset); - } - } - - return assets; - } - - async #slimAdvisories(): Promise { - if (this.#slim === undefined || !this.#slimByDefault) { - return []; - } - - const catalog = await this.#loadCatalog(); - const unsupportedVersions = [ - ...new Set( - catalog.runtimes - .filter((runtime) => runtime.isAvailable) - .filter((runtime) => !supportsPersistentSlim(iosRuntimeVersionFromId(runtime.identifier))) - .map((runtime) => runtime.version), - ), - ].sort(compareVersions); - - if (unsupportedVersions.length === 0) { - return []; - } - - const plural = unsupportedVersions.length > 1; - return [ - { - code: "slim-runtime-unsupported", - message: - `The default device mode is slim, but iOS ${unsupportedVersions.join(", ")} ${plural ? "are" : "is"} ` + - `below the 18.5 persistent-override floor; devices on ${plural ? "those runtimes" : "that runtime"} ` + - `get full devices -- \`launchctl disable\` overrides do not survive a reboot below iOS 18.5`, - }, - ]; + toolVersions(): Promise { + return this.#components.toolVersions(); } virtualLeaseEnvironment(): Readonly> { @@ -1737,461 +548,15 @@ export class IosSimctlDriver implements Driver { args: readonly string[], context?: PassthroughContext, ): Promise { - return passthroughCommand(tool, device, args, context, (a) => this.#scopedSimctlArgv(a)); - } - - #deviceDirectory(udid: string): string { - return join(this.#deviceRoot, udid); - } - - #dataPathFor(udid: string): string { - return join(this.#deviceDirectory(udid), "data"); + return passthroughCommand(tool, device, args, context, (a) => this.#simctl.scopedArgv(a)); } - - /** - * why: Writes the same provenance token into both regions of a device: the - * device root (durable -- survives `simctl erase`) and the data container - * (erasable -- destroyed by it). - * - * The erasable half goes first, and the two are not written concurrently, because a - * half-written pair is read by `Doctor` as tampering: durable-without-erasable is - * exactly the signature of a foreign erase. Writing the fragile half first means a - * failure (`//data` is not there, so `writeFileAtomic` -- which creates no - * parents -- cannot land) leaves *neither* mark, which reads as "never marked" and - * produces no finding at all. The write still throws, so the caller learns something - * was wrong with the device; what it must never do is accuse the user of erasing a - * device Simlock itself just erased. - */ - async #writeMark(udid: string): Promise { - const token = this.#idGenerator.generate(); - const contents = JSON.stringify({ - instanceId: this.#instanceId, - token, - udid, - writtenAt: new Date(this.#clock.now()).toISOString(), - }); - - await this.#filesystem.writeFileAtomic(join(this.#dataPathFor(udid), MARK_FILE_NAME), contents); - await this.#filesystem.writeFileAtomic( - join(this.#deviceDirectory(udid), MARK_FILE_NAME), - contents, - ); - } - - async #readMark(udid: string): Promise { - const durable = await this.#readToken(join(this.#deviceDirectory(udid), MARK_FILE_NAME)); - const erasable = await this.#readToken(join(this.#dataPathFor(udid), MARK_FILE_NAME)); - - if (durable === undefined && erasable === undefined) { - return undefined; - } - - return { durable, erasable, erasableReadable: true }; - } - - async #readToken(path: string): Promise { - return (await this.#readMarkFile(path))?.token; - } - - async #readMarkFile( - path: string, - ): Promise<{ readonly token: string; readonly instanceId: string | undefined } | undefined> { - try { - const contents = await this.#filesystem.readFile(path); - const parsed = JSON.parse(contents) as unknown; - if (!isRecord(parsed) || typeof parsed.token !== "string" || parsed.token === "") { - return undefined; - } - return { - instanceId: - typeof parsed.instanceId === "string" && parsed.instanceId !== "" - ? parsed.instanceId - : undefined, - token: parsed.token, - }; - } catch { - return undefined; - } - } - - /** why: Under Xcode's set the default set holds this instance's simulators too, and they are not foreign. */ - async #ownedUdids(listed: unknown): Promise> { - if (this.#rootOptions !== undefined || !isRecord(listed) || !isRecord(listed.devices)) { - return new Set(); - } - const udids = parseManagedDevices(listed).map((device) => device.udid); - const owned = await Promise.all(udids.map((udid) => this.#isMarkedByThisInstance(udid))); - return new Set(udids.filter((_udid, index) => owned[index])); - } - - async #isMarkedByThisInstance(udid: string): Promise { - const durable = await this.#readMarkFile(join(this.#deviceDirectory(udid), MARK_FILE_NAME)); - return durable?.instanceId === this.#instanceId; - } - - async #resolvedSpec(spec: VirtualSpec): Promise { - const existing = this.#resolvedSpecs.get(specKey(spec)); - if (existing !== undefined) { - return existing; - } - - await this.resolveSpec(spec); - const resolved = this.#resolvedSpecs.get(specKey(spec)); - if (resolved === undefined) { - throw new DriverCrashError("simctl did not resolve the requested device specification"); - } - - return resolved; - } - - async #loadCatalog(): Promise { - const result = await this.#simctl( - ["list", "-j", "devicetypes", "runtimes"], - COMMAND_TIMEOUT_MS, - ); - - try { - return parseCatalog(JSON.parse(result.stdout) as unknown); - } catch (error: unknown) { - if (error instanceof DriverCrashError) { - throw error; - } - - throw new DriverCrashError(`Could not parse simctl device catalog: ${errorMessage(error)}`); - } - } - - async #loadRuntimeImages(): Promise { - const result = await this.#simctl(["runtime", "list", "-j"], COMMAND_TIMEOUT_MS); - try { - return parseRuntimeImages(JSON.parse(result.stdout) as unknown); - } catch (error: unknown) { - if (error instanceof DriverCrashError) throw error; - throw new DriverCrashError(`Could not parse simctl runtime list: ${errorMessage(error)}`); - } - } - - async #shutdown(udid: string, set?: string): Promise { - const outcome = await this.#invokeSimctl(["shutdown", udid], COMMAND_TIMEOUT_MS, set); - - if (outcome.kind === "timed-out") { - throw new DriverCrashError(`simctl shutdown ${udid} timed out after ${COMMAND_TIMEOUT_MS}ms`); - } - if (outcome.result.code !== 0 && !alreadyShutdown(outcome.result.stderr)) { - this.#assertSuccessful(["shutdown", udid], outcome.result); - } - } - - async #bestEffortShutdown(udid: string, set?: string): Promise { - try { - await this.#shutdown(udid, set); - } catch { - // why: Boot failure cleanup must not obscure the timeout that triggered it. - } - } - - /** - * why: `simctl clone` and `simctl create` ask CoreSimulator's service for a whole simulator, and - * the service finishes it even when `simctl` is killed. At the limit this stops waiting but lets - * the command run on: the UDID it later prints is the driver's own, so ownership is proven by - * the call, never by the name (safety rule 8), and that simulator is deleted. A deletion of - * the seed waits for that copy (`SeedTurns.hold`); another clone of the seed does not. - */ - async #createSimulator( - args: readonly string[], - sets: { readonly destination?: string; readonly seedUdid?: string; readonly source?: string }, - ): Promise { - let ended!: () => void; - const lateEnded = new Promise((resolve) => { - ended = resolve; - }); - const outcome = await this.#invokeSimctl(args, this.#createTimeoutMs, sets.source, { - killAfterMs: LATE_CREATE_KILL_MS, - onSettled: (result) => { - void deleteLateSimulator(args, result, this.#logger, async (udid) => { - this.#checked( - ["delete", udid], - COMMAND_TIMEOUT_MS, - await this.#invokeSimctl(["delete", udid], COMMAND_TIMEOUT_MS, sets.destination), - ); - }).finally(ended); - }, - }); - if (outcome.kind === "timed-out" && sets.seedUdid !== undefined) { - this.#seedTurns.hold(sets.seedUdid, lateEnded); - } - return this.#checked(args, this.#createTimeoutMs, outcome); - } - - async #simctl(args: readonly string[], timeoutMs: number, set?: string): Promise { - return this.#checked(args, timeoutMs, await this.#invokeSimctl(args, timeoutMs, set)); - } - - async #legacySimctl(args: readonly string[], timeoutMs: number): Promise { - return this.#checked(args, timeoutMs, await this.#invokeLegacySimctl(args, timeoutMs)); - } - - #checked(args: readonly string[], timeoutMs: number, outcome: ProcessOutcome): ProcessResult { - if (outcome.kind === "timed-out") { - throw new DriverCrashError(`simctl ${args.join(" ")} timed out after ${timeoutMs}ms`); - } - - this.#assertSuccessful(args, outcome.result); - return outcome.result; - } - - #scopedSimctlArgv(args: readonly string[], set?: string): string[] { - if (set !== undefined) return ["simctl", "--set", set, ...args]; - // why: Xcode's own set is simctl's default, so a call there carries no `--set`. - if (this.#rootOptions === undefined) return ["simctl", ...args]; - return ["simctl", "--set", this.#deviceRoot, ...args]; - } - - async #invokeSimctl( - args: readonly string[], - timeoutMs: number, - set?: string, - late?: LateAnswer, - ): Promise { - return this.#invokeXcrun(this.#scopedSimctlArgv(args, set), timeoutMs, late); - } - - /** - * why: Deliberately unscoped, and the only thing in Simlock that is: the pre-root devices - * `findLegacy` / `destroyLegacy` deal with are in the machine's default set, which is - * where a `--set` would stop reaching them. Only those two may call it, and only for a - * UDID a registry record names -- registry-only destruction (safety rule 1) is satisfied - * by that record, not by the root. The one other caller is `listComponents`, and only for - * `simctl list`, which reads the default set and mutates nothing (ADR 0010 §8). - */ - async #invokeLegacySimctl(args: readonly string[], timeoutMs: number): Promise { - return this.#invokeXcrun(["simctl", ...args], timeoutMs); - } - - async #invokeXcrun( - argv: readonly string[], - timeoutMs: number, - late?: LateAnswer, - ): Promise { - return await runCommand(this.#processRunner, this.#clock, "xcrun", argv, timeoutMs, late); - } - - #assertSuccessful(args: readonly string[], result: ProcessResult): void { - if (result.code !== 0) { - const stderr = result.stderr.trim(); - const suffix = stderr === "" ? "" : `: ${stderr}`; - throw new DriverCrashError( - `simctl ${args.join(" ")} exited with code ${String(result.code)}${suffix}`, - ); - } - } -} - -// why: the input is offered exactly when the screen capture is, because both are the one program. -function createInputFor( - options: IosSimctlDriverOptions, - deviceRoot: string, - ownedRoot: boolean, -): DeviceInputFactory | undefined { - if (options.screenCapture === undefined) return undefined; - return createIosDeviceInput({ - baseEnvironment: options.screenCapture.baseEnvironment, - clock: options.clock, - command: configuredScreenCaptureCommand(options) ?? options.screenCapture.command, - ...(ownedRoot ? { deviceSet: deviceRoot } : {}), - ...(options.logger === undefined ? {} : { logger: options.logger }), - launcher: options.screenCapture.launcher, - processRunner: options.processRunner, - }); -} - -function configuredScreenCaptureCommand(options: IosSimctlDriverOptions): string | undefined { - const configured = options.driverConfig["screenCaptureCommand"]; - return typeof configured === "string" && configured !== "" ? configured : undefined; -} - -function configuredDeviceCaptureCommand(options: IosSimctlDriverOptions): string | undefined { - const configured = options.driverConfig["deviceCaptureCommand"]; - return typeof configured === "string" && configured !== "" ? configured : undefined; -} - -function configuredDeviceSet(options: IosSimctlDriverOptions): "simlock" | "default" { - const configured = options.driverConfig["deviceSet"]; - if (configured === undefined || configured === "simlock") { - if (options.config?.simulator?.headless === false) throw headlessNeedsDefaultSet("simlock"); - return "simlock"; - } - if (configured !== "default") { - throw new OwnedRootError( - `Refusing the ios device set: drivers.ios.deviceSet must be "simlock" or "default", but it is ${JSON.stringify(configured)}`, - "invalid-device-set", - typeof configured === "string" ? configured : JSON.stringify(configured), - "ios", - ); - } - if (options.driverConfig["deviceRoot"] !== undefined) { - throw new OwnedRootError( - `Refusing the ios device set: drivers.ios.deviceRoot names Simlock's own set, but drivers.ios.deviceSet is "default". Remove one of them.`, - "invalid-device-set", - configured, - "ios", - ); - } - return "default"; -} - -function xcodeDeviceSetPath(options: IosSimctlDriverOptions): string { - return join(options.coreSimulatorRoot ?? ".", "Devices"); -} - -function configuredDeviceRoot(options: IosSimctlDriverOptions): string { - const configured = options.driverConfig["deviceRoot"]; - - if (configured !== undefined && typeof configured !== "string") { - // why: A `deviceRoot` that is not a usable path refuses this platform's *configuration*, - // which costs iOS and nothing else -- the same as every other refusal here. It is not - // a daemon-fatal error: `"deviceRoot": true` and `"deviceRoot": "devices/ios"` are one - // keystroke apart, and killing the process over the first would take Android down with - // it and leave the reason unreachable, since `doctor` -- where the CLI reference promises - // it appears -- needs a daemon to answer. `not-absolute` is the vocabulary term the - // docs already publish for "this names no usable directory"; nothing new is invented. - throw new OwnedRootError( - `Refusing the ios device root: drivers.ios.deviceRoot must be an absolute path, but it is the ${typeof configured} ${JSON.stringify(configured)}`, - "not-absolute", - String(configured), - "ios", - ); - } - - return configured ?? defaultIosDeviceRoot(options.simlockHome); } export { iosRuntimeVersionFromId }; -export function supportsPersistentSlim(version: readonly [number, number] | undefined): boolean { - if (version === undefined) { - return false; - } - const [major, minor] = version; - return major > 18 || (major === 18 && minor >= 5); -} - -type SlimPlan = - | { readonly kind: "off" } - | { readonly kind: "skip"; readonly reason: SlimSkippedFact["reason"]; readonly detail: string } - | { readonly kind: "apply"; readonly slim: SlimOptions }; - -function planSlimBoot( - data: IosDriverData, - slim: SlimOptions | undefined, - fullBootTimeoutMs: number, - options: { readonly purpose: "prepare" | "recover"; readonly mode: DeviceMode }, -): { readonly plan: SlimPlan; readonly bootstatusTimeoutMs: number } { - if (options.purpose === "recover" || slim === undefined || options.mode !== "slim") { - return { bootstatusTimeoutMs: fullBootTimeoutMs, plan: { kind: "off" } }; - } - - const version = iosRuntimeVersionFromId(data.runtimeId); - if (!supportsPersistentSlim(version)) { - return { - bootstatusTimeoutMs: fullBootTimeoutMs, - plan: { - detail: - version === undefined - ? `runtimeId "${data.runtimeId}" does not parse to an iOS version` - : `iOS ${String(version[0])}.${String(version[1])} is below the 18.5 persistent-override floor`, - kind: "skip", - reason: version === undefined ? "unknown-runtime" : "runtime-too-old", - }, - }; - } - - return { bootstatusTimeoutMs: slim.bootTimeoutMs, plan: { kind: "apply", slim } }; -} - -function createScreenCaptureFor( - options: IosSimctlDriverOptions, - deviceRoot: string, - ownsDeviceSet: boolean, - deviceOsVersion: (udid: string) => Promise, -): ScreenCapture | undefined { - const capture = options.screenCapture; - if (capture === undefined) return undefined; - const deviceCommand = configuredDeviceCaptureCommand(options) ?? capture.deviceCommand; - return createIosScreenCapture({ - baseEnvironment: capture.baseEnvironment, - clock: options.clock, - command: configuredScreenCaptureCommand(options) ?? capture.command, - ...(deviceCommand === undefined ? {} : { deviceCommand }), - deviceOsVersion, - ...(ownsDeviceSet ? { deviceSet: deviceRoot } : {}), - filesystem: options.filesystem, - launcher: capture.launcher, - ...(capture.onLog === undefined ? {} : { onLog: capture.onLog }), - }); -} - -function sameEntries(a: ReadonlyMap, b: ReadonlyMap): boolean { - return a.size === b.size && [...a].every(([label, disabled]) => b.get(label) === disabled); -} - -function alreadyShutdown(stderr: string): boolean { - return /Unable to shutdown.*current state:\s*Shutdown/i.test(stderr); -} - -function unknownDevice(result: ProcessResult): boolean { - return /^Invalid device:/m.test(result.stderr); -} - -function readBackFailure( - outcome: - | { readonly kind: "timed-out" } - | { readonly kind: "finished"; readonly result: { code: number | null; stderr: string } }, -): string { - return outcome.kind === "timed-out" - ? `timed out after ${String(READBACK_TIMEOUT_MS)}ms` - : `exit ${String(outcome.result.code)}: ${outcome.result.stderr.trim()}`; -} - -function alreadyBooted(stderr: string): boolean { - return /current state:\s*Booted|already booted/i.test(stderr); -} - -/** why: a seed built on a runtime whose build changed no longer matches it; with no build on either side only the runtime's presence is checked, and a provision always has the runtime. */ -function seedIsStale(seed: ReadySeed, resolved: ResolvedIosSpec): boolean { - const current = resolved.runtime.build; - return seed.runtimeBuild !== undefined && current !== undefined && seed.runtimeBuild !== current; -} - -/** why: the phrase reads after "is stale:" in `doctor`, and is what a person acts on, so it names the runtime or device type by id. */ -function sourceStaleReason(row: SourceRow, catalog: SimctlCatalog): string | undefined { - const runtime = catalog.runtimes.find((entry) => entry.identifier === row.runtimeId); - if (runtime === undefined) return `runtime ${row.runtimeId} is not installed`; - if ( - row.runtimeBuild !== undefined && - runtime.build !== undefined && - row.runtimeBuild !== runtime.build - ) { - return `runtime ${row.runtimeId} changed from build ${row.runtimeBuild} to ${runtime.build}`; - } - if (!catalog.deviceTypes.some((entry) => entry.identifier === row.deviceTypeId)) { - return `device type ${row.deviceTypeId} is not available`; - } - return undefined; -} - -function seedSpecKey(resolved: ResolvedIosSpec): string { - return `${resolved.deviceType.identifier}\u0000${resolved.runtime.identifier}`; -} - -function specKey(spec: VirtualSpec): string { - return `${spec.model.toLocaleLowerCase()}\u0000${spec.osVersion}`; -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} export type { SilenceFailedFact, WindowFailedFact } from "./boot-courtesies.js"; +export type { IosSimctlDriverOptions } from "./driver-options.js"; +export { supportsPersistentSlim, type SlimmedFact, type SlimSkippedFact } from "./slim-boot.js"; export { type IosDriverConfig, iosDriverConfig } from "./definition.js"; export { iosDeviceDataTable } from "./device-data.js"; export { iosSourceDataOwner } from "./source-data.js"; diff --git a/src/drivers/ios/marks.ts b/src/drivers/ios/marks.ts new file mode 100644 index 000000000..358a51cc4 --- /dev/null +++ b/src/drivers/ios/marks.ts @@ -0,0 +1,123 @@ +import { join } from "node:path"; +import type { ObservedMark } from "../../core/index.js"; +import type { Clock, Filesystem, IdGenerator } from "../../ports/index.js"; +import { isRecord, parseManagedDevices } from "./simctl-parse.js"; + +interface DeviceMarksOptions { + readonly clock: Clock; + readonly deviceRoot: string; + readonly filesystem: Filesystem; + readonly idGenerator: IdGenerator; + readonly instanceId: string; + readonly ownsDeviceSet: boolean; +} + +const MARK_FILE_NAME = "simlock-mark.json"; + +/** why: the part of IosSimctlDriver (ADR 0063) that writes, reads and checks the durable and erasable marks Simlock leaves on each simulator it makes. */ +export class DeviceMarks { + readonly #clock: Clock; + readonly #deviceRoot: string; + readonly #filesystem: Filesystem; + readonly #idGenerator: IdGenerator; + readonly #instanceId: string; + readonly #ownsDeviceSet: boolean; + + constructor(options: DeviceMarksOptions) { + this.#clock = options.clock; + this.#deviceRoot = options.deviceRoot; + this.#filesystem = options.filesystem; + this.#idGenerator = options.idGenerator; + this.#instanceId = options.instanceId; + this.#ownsDeviceSet = options.ownsDeviceSet; + } + + #deviceDirectory(udid: string): string { + return join(this.#deviceRoot, udid); + } + + #dataPathFor(udid: string): string { + return join(this.#deviceDirectory(udid), "data"); + } + + /** + * why: Writes the same provenance token into both regions of a device: the + * device root (durable -- survives `simctl erase`) and the data container + * (erasable -- destroyed by it). + * + * The erasable half goes first, and the two are not written concurrently, because a + * half-written pair is read by `Doctor` as tampering: durable-without-erasable is + * exactly the signature of a foreign erase. Writing the fragile half first means a + * failure (`//data` is not there, so `writeFileAtomic` -- which creates no + * parents -- cannot land) leaves *neither* mark, which reads as "never marked" and + * produces no finding at all. The write still throws, so the caller learns something + * was wrong with the device; what it must never do is accuse the user of erasing a + * device Simlock itself just erased. + */ + async writeMark(udid: string): Promise { + const token = this.#idGenerator.generate(); + const contents = JSON.stringify({ + instanceId: this.#instanceId, + token, + udid, + writtenAt: new Date(this.#clock.now()).toISOString(), + }); + + await this.#filesystem.writeFileAtomic(join(this.#dataPathFor(udid), MARK_FILE_NAME), contents); + await this.#filesystem.writeFileAtomic( + join(this.#deviceDirectory(udid), MARK_FILE_NAME), + contents, + ); + } + + async readMark(udid: string): Promise { + const durable = await this.#readToken(join(this.#deviceDirectory(udid), MARK_FILE_NAME)); + const erasable = await this.#readToken(join(this.#dataPathFor(udid), MARK_FILE_NAME)); + + if (durable === undefined && erasable === undefined) { + return undefined; + } + + return { durable, erasable, erasableReadable: true }; + } + + async #readToken(path: string): Promise { + return (await this.#readMarkFile(path))?.token; + } + + async #readMarkFile( + path: string, + ): Promise<{ readonly token: string; readonly instanceId: string | undefined } | undefined> { + try { + const contents = await this.#filesystem.readFile(path); + const parsed = JSON.parse(contents) as unknown; + if (!isRecord(parsed) || typeof parsed.token !== "string" || parsed.token === "") { + return undefined; + } + return { + instanceId: + typeof parsed.instanceId === "string" && parsed.instanceId !== "" + ? parsed.instanceId + : undefined, + token: parsed.token, + }; + } catch { + return undefined; + } + } + + /** why: Under Xcode's set the default set holds this instance's simulators too, and they are not foreign. */ + async ownedUdids(listed: unknown): Promise> { + if (this.#ownsDeviceSet || !isRecord(listed) || !isRecord(listed.devices)) { + return new Set(); + } + const udids = parseManagedDevices(listed).map((device) => device.udid); + const owned = await Promise.all(udids.map((udid) => this.isMarkedByThisInstance(udid))); + return new Set(udids.filter((_udid, index) => owned[index])); + } + + async isMarkedByThisInstance(udid: string): Promise { + const durable = await this.#readMarkFile(join(this.#deviceDirectory(udid), MARK_FILE_NAME)); + return durable?.instanceId === this.#instanceId; + } +} diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts new file mode 100644 index 000000000..3d312130e --- /dev/null +++ b/src/drivers/ios/seeds.ts @@ -0,0 +1,470 @@ +import { join } from "node:path"; +import { + BootTimeoutError, + type DriverDevice, + type DriverSource, + DriverCrashError, + type VirtualDriverDevice, + ensureOwnedRoot, +} from "../../core/index.js"; +import type { Clock, Filesystem, IdGenerator, Logger } from "../../ports/index.js"; + +import { errorMessage } from "./command-run.js"; +import { SeedTurns } from "./seed-turns.js"; +import { COMMAND_TIMEOUT_MS, Simctl, unknownDevice } from "./simctl.js"; +import { DeviceMarks } from "./marks.js"; +import type { SlimBoot } from "./slim-boot.js"; +import type { SimulatorBoot } from "./simulator-boot.js"; +import type { ResolvedIosSpec } from "./catalog.js"; + +import { UDID_PATTERN } from "./device-set-leftovers.js"; + +import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; + +import { type SimctlCatalog } from "./simctl-parse.js"; + +import { type IosDriverData, iosDriverData } from "./simulator-data.js"; + +import type { SqliteDatabase } from "../../ports/index.js"; + +/** why: ADR 0061 §1: a `making` row of this process older than this is a make that never ended. */ +const STALE_MAKING_MS = 300_000; + +const LAUNCH_SERVICES_DATABASE = /^com\.apple\.LaunchServices-.*\.csstore$/; + +interface SeedsOptions { + readonly boot: SimulatorBoot; + readonly clock: Clock; + readonly slimBoot: SlimBoot; + readonly database: SqliteDatabase; + readonly deviceRoot: string; + readonly filesystem: Filesystem; + readonly fullBootTimeoutMs: number; + readonly idGenerator: IdGenerator; + readonly instanceId: string; + readonly logger: Logger | undefined; + readonly marks: DeviceMarks; + readonly ownsDeviceSet: boolean; + readonly seedTurns: SeedTurns; + readonly simctl: Simctl; + readonly sourcesSet: string; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that makes, keeps and deletes the seeds every provision copies (ADR 0061), and lists them as sources. */ +export class Seeds { + readonly #boot: SimulatorBoot; + readonly #clock: Clock; + readonly #slimBoot: SlimBoot; + readonly #deviceRoot: string; + readonly #filesystem: Filesystem; + readonly #fullBootTimeoutMs: number; + readonly #idGenerator: IdGenerator; + readonly #instanceId: string; + readonly #logger: Logger | undefined; + readonly #marks: DeviceMarks; + readonly #ownsDeviceSet: boolean; + readonly #seedMakes = new Map>(); + readonly #seedTurns: SeedTurns; + readonly #simctl: Simctl; + readonly #sources: IosSourceTable; + readonly #sourcesSet: string; + #sourceMaker: string | undefined; + #swept = false; + + constructor(options: SeedsOptions) { + this.#boot = options.boot; + this.#clock = options.clock; + this.#slimBoot = options.slimBoot; + this.#deviceRoot = options.deviceRoot; + this.#filesystem = options.filesystem; + this.#fullBootTimeoutMs = options.fullBootTimeoutMs; + this.#idGenerator = options.idGenerator; + this.#instanceId = options.instanceId; + this.#logger = options.logger; + this.#marks = options.marks; + this.#ownsDeviceSet = options.ownsDeviceSet; + this.#seedTurns = options.seedTurns; + this.#simctl = options.simctl; + this.#sources = new IosSourceTable(options.database); + this.#sourcesSet = options.sourcesSet; + } + + async listSources(): Promise { + await this.#sweepLeftovers(); + const rows = this.#sources.readyRows(); + const orphans = await this.#orphanSources(); + if (rows.length === 0) return orphans; + const catalog = await this.#simctl.loadCatalog(); + return [ + ...rows.map((row) => { + const staleReason = sourceStaleReason(row, catalog); + return { + id: row.sourceId, + kind: row.kind, + model: row.model, + osVersion: row.osVersion, + ...(row.sizeBytes === undefined ? {} : { sizeBytes: row.sizeBytes }), + ...(staleReason === undefined ? {} : { staleReason }), + }; + }), + ...orphans, + ]; + } + + async saveSource(device: VirtualDriverDevice): Promise<{ sourceId: string; sizeBytes?: number }> { + const data = iosDriverData(device); + const catalog = await this.#simctl.loadCatalog(); + const runtime = catalog.runtimes.find((entry) => entry.identifier === data.runtimeId); + const deviceType = catalog.deviceTypes.find((entry) => entry.identifier === data.deviceTypeId); + if (runtime === undefined || deviceType === undefined) { + throw new DriverCrashError( + `device ${data.udid} was made with a runtime or device type that is gone`, + ); + } + const sourceId = this.#idGenerator.generate(); + this.#sources.insertMaking({ + createdAt: this.#clock.now(), + deviceTypeId: data.deviceTypeId, + kind: "baseline", + maker: this.#maker(), + model: deviceType.name, + osVersion: runtime.version, + runtimeBuild: runtime.build, + runtimeId: data.runtimeId, + setPath: this.#sourcesSet, + sourceId, + specKey: `baseline-${sourceId}`, + }); + let cloneUdid: string | undefined; + try { + await this.#ensureSourcesSet(); + await this.#simctl.shutdown(data.udid); + const cloned = await this.#simctl.createSimulator( + ["clone", data.udid, `simlock-baseline-${sourceId}`, this.#sourcesSet], + { destination: this.#sourcesSet }, + ); + cloneUdid = cloned.stdout.trim(); + if (cloneUdid === "") throw new DriverCrashError("simctl clone returned no device UDID"); + this.#sources.setUdid(sourceId, cloneUdid); + await this.#slimBoot.copyOverrides(data.udid, cloneUdid); + await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); + const sizeBytes = await this.#sizeOf(cloneUdid); + this.#sources.markReady(sourceId, sizeBytes); + return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; + } catch (error: unknown) { + if (cloneUdid !== undefined && cloneUdid !== "") { + await this.#deleteSeedSimulator(cloneUdid).catch(() => undefined); + } + this.#sources.deleteRow(sourceId); + throw error; + } + } + + async destroySource(sourceId: string): Promise { + const row = this.#sources.findRow(sourceId); + if (row === undefined) { + // why: safety rule 1 (ADR 0061 §1): a UDID no row names is deleted only when it is a simulator folder of the sources set. + if (!(await this.#isOrphanSource(sourceId))) { + throw new DriverCrashError(`iOS keeps no source ${sourceId}`); + } + await this.#deleteSeedInTurn(sourceId); + return; + } + if (row.udid !== undefined) { + const udid = row.udid; + await this.#deleteSeedInTurn(udid); + } + this.#sources.deleteRow(sourceId); + } + + async provision( + resolved: ResolvedIosSpec, + options?: { readonly source?: string }, + ): Promise { + const seed = + options?.source === undefined + ? await this.#seedFor(resolved) + : await this.#namedSource(options.source); + // why: Cosmetic, and deliberately so: what proves this device is Simlock's is that it + // lives in the device set every call below is scoped to, never what it is called + // (safety rule 8). The name exists because it is what a human reads in `simctl list` + // and in the simulator's window title. + const name = `simlock-${this.#idGenerator.generate()}`; + const result = await this.#seedTurns.take(seed.udid, () => + this.#simctl.createSimulator(["clone", seed.udid, name, this.#deviceRoot], { + seedUdid: seed.udid, + source: this.#sourcesSet, + }), + ); + const udid = result.stdout.trim(); + + if (udid === "") { + throw new DriverCrashError("simctl clone returned no device UDID"); + } + + try { + await this.#marks.writeMark(udid); + await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); + } catch (error: unknown) { + // why: a clone whose mark or database drop failed is not a device Simlock hands out, and + // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. + await this.#simctl.invoke(["delete", udid], COMMAND_TIMEOUT_MS).catch(() => undefined); + throw error; + } + await this.#slimBoot.copyOverrides(seed.udid, udid); + + return { + address: udid, + deviceId: udid, + driverData: { + deviceTypeId: resolved.deviceType.identifier, + name, + runtimeId: resolved.runtime.identifier, + udid, + } satisfies IosDriverData, + }; + } + + /** why: a simulator in the sources set that no row names is a leftover of a crash; `doctor` reports it and only `--purge-orphans` deletes it. */ + async #orphanSources(): Promise { + const named = new Set(this.#sources.allUdids()); + return (await this.#sourceFolders()) + .filter((udid) => !named.has(udid)) + .map((udid) => ({ + id: udid, + kind: "seed", + model: "unknown", + orphan: true, + osVersion: "unknown", + })); + } + + async #isOrphanSource(udid: string): Promise { + return (await this.#sourceFolders()).includes(udid) && !this.#sources.allUdids().includes(udid); + } + + async #sourceFolders(): Promise { + try { + return (await this.#filesystem.readdir(this.#sourcesSet)).filter((entry) => + UDID_PATTERN.test(entry), + ); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + } + + async #namedSource(sourceId: string): Promise<{ readonly udid: string }> { + const row = this.#sources.findRow(sourceId); + if (row?.state !== "ready" || row.udid === undefined) { + throw new DriverCrashError(`iOS keeps no ready source ${sourceId}`); + } + const reason = sourceStaleReason(row, await this.#simctl.loadCatalog()); + if (reason !== undefined) throw new DriverCrashError(`source ${sourceId} is stale: ${reason}`); + return { udid: row.udid }; + } + + /** why: two provisions of one kind at once make one seed, the second waiting for the first's make. */ + #seedFor(resolved: ResolvedIosSpec): Promise { + const specKey = seedSpecKey(resolved); + const running = this.#seedMakes.get(specKey); + if (running !== undefined) return running; + const made = this.#findOrMakeSeed(resolved, specKey).finally(() => { + this.#seedMakes.delete(specKey); + }); + this.#seedMakes.set(specKey, made); + return made; + } + + async #deleteSeedInTurn(udid: string): Promise { + await this.#seedTurns.takeToDelete(udid, () => this.#deleteSeedSimulator(udid)); + } + + async #findOrMakeSeed(resolved: ResolvedIosSpec, specKey: string): Promise { + const seed = this.#sources.findReadySeed(specKey); + if (seed !== undefined && !seedIsStale(seed, resolved)) return seed; + if (seed !== undefined) await this.#destroySeed(seed); + return this.#makeSeed(resolved, specKey); + } + + async #makeSeed(resolved: ResolvedIosSpec, specKey: string): Promise { + // why: a `making` row of this process older than the longest a make can take belongs to a make that never ended. + for (const row of this.#sources.makingRowsOfKindBefore( + this.#maker(), + specKey, + this.#clock.now() - STALE_MAKING_MS, + )) { + await this.#removeLeftover(row); + } + const sourceId = this.#idGenerator.generate(); + this.#sources.insertMaking({ + createdAt: this.#clock.now(), + deviceTypeId: resolved.deviceType.identifier, + kind: "seed", + maker: this.#maker(), + model: resolved.spec.model, + osVersion: resolved.spec.osVersion, + runtimeBuild: resolved.runtime.build, + runtimeId: resolved.runtime.identifier, + setPath: this.#sourcesSet, + sourceId, + specKey, + }); + try { + await this.#ensureSourcesSet(); + const created = await this.#simctl.createSimulator( + [ + "create", + `simlock-seed-${sourceId}`, + resolved.deviceType.identifier, + resolved.runtime.identifier, + ], + { destination: this.#sourcesSet, source: this.#sourcesSet }, + ); + const udid = created.stdout.trim(); + this.#sources.setUdid(sourceId, udid); + try { + await this.#boot.bootAndWait(udid, sourceId, this.#fullBootTimeoutMs, this.#sourcesSet); + } catch (error: unknown) { + await this.#deleteSeedSimulator(udid); + // why: a seed that ran out of boot time is a boot timeout, as an Android seed's is. + if (error instanceof BootTimeoutError) throw error; + throw new DriverCrashError(`seed ${udid} did not boot: ${errorMessage(error)}`); + } + try { + await this.#simctl.shutdown(udid, this.#sourcesSet); + } catch (error: unknown) { + await this.#deleteSeedSimulator(udid); + throw error; + } + this.#sources.markReady(sourceId, await this.#sizeOf(udid)); + return { sourceId, udid, runtimeBuild: resolved.runtime.build }; + } catch (error: unknown) { + this.#sources.deleteRow(sourceId); + throw error; + } + } + + /** + * why: ADR 0061 §1: a `making` row another process left is a seed that process never finished, so + * its simulator (if `create` answered) and row go at the first source read of this process. A + * simulator no row names is left in place: `doctor` reports it. + */ + async #sweepLeftovers(): Promise { + if (this.#swept) return; + this.#swept = true; + for (const row of this.#sources.makingRowsNotMadeBy(this.#maker())) { + await this.#removeLeftover(row); + } + } + + async #removeLeftover(row: SourceRow): Promise { + try { + if (row.udid !== undefined) await this.#deleteSeedSimulator(row.udid); + this.#sources.deleteRow(row.sourceId); + } catch (error: unknown) { + this.#logger?.warn( + `could not remove leftover source ${row.udid ?? row.sourceId}: ${errorMessage(error)}`, + ); + } + } + + #maker(): string { + this.#sourceMaker ??= this.#idGenerator.generate(); + return this.#sourceMaker; + } + + async #destroySeed(seed: ReadySeed): Promise { + await this.#deleteSeedInTurn(seed.udid); + this.#sources.deleteRow(seed.sourceId); + } + + async #deleteSeedSimulator(udid: string): Promise { + await this.#simctl.bestEffortShutdown(udid, this.#sourcesSet); + const args = ["delete", udid]; + const outcome = await this.#simctl.invoke(args, COMMAND_TIMEOUT_MS, this.#sourcesSet); + // why: a seed deleted by hand is already gone, and its row must still be dropped. + if (outcome.kind === "finished" && unknownDevice(outcome.result)) return; + this.#simctl.checked(args, COMMAND_TIMEOUT_MS, outcome); + } + + async #ensureSourcesSet(): Promise { + if (this.#ownsDeviceSet) { + await this.#filesystem.mkdirp(this.#sourcesSet, { mode: 0o700 }); + return; + } + await ensureOwnedRoot({ + filesystem: this.#filesystem, + idGenerator: this.#idGenerator, + instanceId: this.#instanceId, + path: this.#sourcesSet, + platform: "ios", + }); + } + + async #sizeOf(udid: string): Promise { + try { + return await this.#filesystem.directorySize(join(this.#sourcesSet, udid)); + } catch { + // why: a size that cannot be counted leaves the seed `ready` with no size; `doctor` then leaves it out. + return undefined; + } + } + + /** + * why: `simctl clone` keeps the LaunchServices database, which records each app (system apps + * included) by its absolute path under the original device's folder; a copy then uses the + * original's data, or fails once that folder is gone. Dropped from every clone before its first + * boot, LaunchServices rebuilds it from the clone's own container folders (ADR 0061 §3). + */ + async #dropLaunchServicesDatabase(setPath: string, udid: string): Promise { + const folder = join(setPath, udid, "data", "var", "db", "lsd"); + if (!(await this.#filesystem.exists(folder))) return; + for (const entry of await this.#filesystem.readdir(folder)) { + if (LAUNCH_SERVICES_DATABASE.test(entry)) await this.#filesystem.rm(join(folder, entry)); + } + } + + /** why: ADR 0061 §3: a seed of a removed runtime can never be copied again; a stuck one is logged and the removal goes on. */ + async destroySeedsOf(runtimeId: string): Promise { + for (const row of this.#sources.readyRows()) { + if (row.kind !== "seed" || row.runtimeId !== runtimeId || row.udid === undefined) continue; + const udid = row.udid; + try { + await this.#deleteSeedInTurn(udid); + this.#sources.deleteRow(row.sourceId); + } catch (error: unknown) { + this.#logger?.warn( + `could not remove seed ${udid} of removed runtime ${runtimeId}: ${errorMessage(error)}`, + ); + } + } + } +} + +/** why: a seed built on a runtime whose build changed no longer matches it; with no build on either side only the runtime's presence is checked, and a provision always has the runtime. */ +function seedIsStale(seed: ReadySeed, resolved: ResolvedIosSpec): boolean { + const current = resolved.runtime.build; + return seed.runtimeBuild !== undefined && current !== undefined && seed.runtimeBuild !== current; +} + +/** why: the phrase reads after "is stale:" in `doctor`, and is what a person acts on, so it names the runtime or device type by id. */ +function sourceStaleReason(row: SourceRow, catalog: SimctlCatalog): string | undefined { + const runtime = catalog.runtimes.find((entry) => entry.identifier === row.runtimeId); + if (runtime === undefined) return `runtime ${row.runtimeId} is not installed`; + if ( + row.runtimeBuild !== undefined && + runtime.build !== undefined && + row.runtimeBuild !== runtime.build + ) { + return `runtime ${row.runtimeId} changed from build ${row.runtimeBuild} to ${runtime.build}`; + } + if (!catalog.deviceTypes.some((entry) => entry.identifier === row.deviceTypeId)) { + return `device type ${row.deviceTypeId} is not available`; + } + return undefined; +} + +function seedSpecKey(resolved: ResolvedIosSpec): string { + return `${resolved.deviceType.identifier}\u0000${resolved.runtime.identifier}`; +} diff --git a/src/drivers/ios/simctl.ts b/src/drivers/ios/simctl.ts new file mode 100644 index 000000000..2ceec9da8 --- /dev/null +++ b/src/drivers/ios/simctl.ts @@ -0,0 +1,194 @@ +import type { ProcessOutcome } from "./boot-courtesies.js"; +import { DriverCrashError } from "../../core/index.js"; +import type { Clock, Logger, ProcessResult, ProcessRunner } from "../../ports/index.js"; +import { deleteLateSimulator, errorMessage, runCommand, type LateAnswer } from "./command-run.js"; +import type { SeedTurns } from "./seed-turns.js"; +import { parseCatalog, type SimctlCatalog } from "./simctl-parse.js"; +import { parseRuntimeImages, type RuntimeImage } from "./runtime-images.js"; + +interface SimctlOptions { + readonly clock: Clock; + readonly createTimeoutMs: number; + readonly deviceRoot: string; + readonly logger: Logger | undefined; + readonly ownsDeviceSet: boolean; + readonly processRunner: ProcessRunner; + readonly seedTurns: SeedTurns; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that runs `simctl` and `xcrun` for the driver, scoped to the device set it owns, and that every other part reaches simulators through. */ +export class Simctl { + readonly #clock: Clock; + readonly #createTimeoutMs: number; + readonly #deviceRoot: string; + readonly #logger: Logger | undefined; + readonly #ownsDeviceSet: boolean; + readonly #processRunner: ProcessRunner; + readonly #seedTurns: SeedTurns; + + constructor(options: SimctlOptions) { + this.#clock = options.clock; + this.#createTimeoutMs = options.createTimeoutMs; + this.#deviceRoot = options.deviceRoot; + this.#logger = options.logger; + this.#ownsDeviceSet = options.ownsDeviceSet; + this.#processRunner = options.processRunner; + this.#seedTurns = options.seedTurns; + } + + async loadCatalog(): Promise { + const result = await this.run(["list", "-j", "devicetypes", "runtimes"], COMMAND_TIMEOUT_MS); + + try { + return parseCatalog(JSON.parse(result.stdout) as unknown); + } catch (error: unknown) { + if (error instanceof DriverCrashError) { + throw error; + } + + throw new DriverCrashError(`Could not parse simctl device catalog: ${errorMessage(error)}`); + } + } + + async loadRuntimeImages(): Promise { + const result = await this.run(["runtime", "list", "-j"], COMMAND_TIMEOUT_MS); + try { + return parseRuntimeImages(JSON.parse(result.stdout) as unknown); + } catch (error: unknown) { + if (error instanceof DriverCrashError) throw error; + throw new DriverCrashError(`Could not parse simctl runtime list: ${errorMessage(error)}`); + } + } + + async shutdown(udid: string, set?: string): Promise { + const outcome = await this.invoke(["shutdown", udid], COMMAND_TIMEOUT_MS, set); + + if (outcome.kind === "timed-out") { + throw new DriverCrashError(`simctl shutdown ${udid} timed out after ${COMMAND_TIMEOUT_MS}ms`); + } + if (outcome.result.code !== 0 && !alreadyShutdown(outcome.result.stderr)) { + this.assertSuccessful(["shutdown", udid], outcome.result); + } + } + + async bestEffortShutdown(udid: string, set?: string): Promise { + try { + await this.shutdown(udid, set); + } catch { + // why: Boot failure cleanup must not obscure the timeout that triggered it. + } + } + + /** + * why: `simctl clone` and `simctl create` ask CoreSimulator's service for a whole simulator, and + * the service finishes it even when `simctl` is killed. At the limit this stops waiting but lets + * the command run on: the UDID it later prints is the driver's own, so ownership is proven by + * the call, never by the name (safety rule 8), and that simulator is deleted. A deletion of + * the seed waits for that copy (`SeedTurns.hold`); another clone of the seed does not. + */ + async createSimulator( + args: readonly string[], + sets: { readonly destination?: string; readonly seedUdid?: string; readonly source?: string }, + ): Promise { + let ended!: () => void; + const lateEnded = new Promise((resolve) => { + ended = resolve; + }); + const outcome = await this.invoke(args, this.#createTimeoutMs, sets.source, { + killAfterMs: LATE_CREATE_KILL_MS, + onSettled: (result) => { + void deleteLateSimulator(args, result, this.#logger, async (udid) => { + this.checked( + ["delete", udid], + COMMAND_TIMEOUT_MS, + await this.invoke(["delete", udid], COMMAND_TIMEOUT_MS, sets.destination), + ); + }).finally(ended); + }, + }); + if (outcome.kind === "timed-out" && sets.seedUdid !== undefined) { + this.#seedTurns.hold(sets.seedUdid, lateEnded); + } + return this.checked(args, this.#createTimeoutMs, outcome); + } + + async run(args: readonly string[], timeoutMs: number, set?: string): Promise { + return this.checked(args, timeoutMs, await this.invoke(args, timeoutMs, set)); + } + + async runLegacy(args: readonly string[], timeoutMs: number): Promise { + return this.checked(args, timeoutMs, await this.invokeLegacy(args, timeoutMs)); + } + + checked(args: readonly string[], timeoutMs: number, outcome: ProcessOutcome): ProcessResult { + if (outcome.kind === "timed-out") { + throw new DriverCrashError(`simctl ${args.join(" ")} timed out after ${timeoutMs}ms`); + } + + this.assertSuccessful(args, outcome.result); + return outcome.result; + } + + scopedArgv(args: readonly string[], set?: string): string[] { + if (set !== undefined) return ["simctl", "--set", set, ...args]; + // why: Xcode's own set is simctl's default, so a call there carries no `--set`. + if (!this.#ownsDeviceSet) return ["simctl", ...args]; + return ["simctl", "--set", this.#deviceRoot, ...args]; + } + + async invoke( + args: readonly string[], + timeoutMs: number, + set?: string, + late?: LateAnswer, + ): Promise { + return this.invokeXcrun(this.scopedArgv(args, set), timeoutMs, late); + } + + /** + * why: Deliberately unscoped, and the only thing in Simlock that is: the pre-root devices + * `findLegacy` / `destroyLegacy` deal with are in the machine's default set, which is + * where a `--set` would stop reaching them. Only those two may call it, and only for a + * UDID a registry record names -- registry-only destruction (safety rule 1) is satisfied + * by that record, not by the root. The one other caller is `listComponents`, and only for + * `simctl list`, which reads the default set and mutates nothing (ADR 0010 §8). + */ + async invokeLegacy(args: readonly string[], timeoutMs: number): Promise { + return this.invokeXcrun(["simctl", ...args], timeoutMs); + } + + async invokeXcrun( + argv: readonly string[], + timeoutMs: number, + late?: LateAnswer, + ): Promise { + return await runCommand(this.#processRunner, this.#clock, "xcrun", argv, timeoutMs, late); + } + + assertSuccessful(args: readonly string[], result: ProcessResult): void { + if (result.code !== 0) { + const stderr = result.stderr.trim(); + const suffix = stderr === "" ? "" : `: ${stderr}`; + throw new DriverCrashError( + `simctl ${args.join(" ")} exited with code ${String(result.code)}${suffix}`, + ); + } + } +} + +export const COMMAND_TIMEOUT_MS = 30_000; + +/** why: a clone or create past its limit is left running; one still silent this long after is killed. */ +const LATE_CREATE_KILL_MS = 300_000; + +function alreadyShutdown(stderr: string): boolean { + return /Unable to shutdown.*current state:\s*Shutdown/i.test(stderr); +} + +export function unknownDevice(result: ProcessResult): boolean { + return /^Invalid device:/m.test(result.stderr); +} + +export function alreadyBooted(stderr: string): boolean { + return /current state:\s*Booted|already booted/i.test(stderr); +} diff --git a/src/drivers/ios/simulator-boot.ts b/src/drivers/ios/simulator-boot.ts new file mode 100644 index 000000000..50101552f --- /dev/null +++ b/src/drivers/ios/simulator-boot.ts @@ -0,0 +1,94 @@ +import { BootTimeoutError } from "../../core/index.js"; +import type { Clock } from "../../ports/index.js"; + +import { COMMAND_TIMEOUT_MS, Simctl, alreadyBooted } from "./simctl.js"; + +const INSTALLER_POLL_MS = 1_000; + +interface SimulatorBootOptions { + readonly clock: Clock; + readonly ownsDeviceSet: boolean; + readonly simctl: Simctl; +} + +/** why: the part of IosSimctlDriver (ADR 0063) that boots a simulator and waits until it is ready to take requests, on one deadline. */ +export class SimulatorBoot { + readonly #clock: Clock; + readonly #ownsDeviceSet: boolean; + readonly #simctl: Simctl; + + constructor(options: SimulatorBootOptions) { + this.#clock = options.clock; + this.#ownsDeviceSet = options.ownsDeviceSet; + this.#simctl = options.simctl; + } + + async bootAndWait( + udid: string, + deviceId: string, + bootstatusTimeoutMs: number, + set?: string, + refreshBooted = false, + ): Promise { + const deadline = this.#clock.now() + bootstatusTimeoutMs; + const boot = await this.#boot(udid, deadline, set, refreshBooted); + if (boot.kind === "timed-out") { + throw new BootTimeoutError(deviceId); + } + if (boot.result.code !== 0 && !alreadyBooted(boot.result.stderr)) { + this.#simctl.assertSuccessful(["boot", udid], boot.result); + } + const outcome = await this.#simctl.invoke( + ["bootstatus", udid, "-b"], + this.#left(deadline), + set, + ); + if (outcome.kind === "finished") { + this.#simctl.assertSuccessful(["bootstatus", udid, "-b"], outcome.result); + if (await this.#waitForInstaller(udid, deadline, set)) return; + } + + await this.#simctl.bestEffortShutdown(udid, set); + throw new BootTimeoutError(deviceId); + } + + // why: a booted simulator keeps the clipboard sync it booted with, so a lease's device that is + // already up is shut down and booted again once DeviceHub's preference is set. + async #boot(udid: string, deadline: number, set: string | undefined, refreshBooted: boolean) { + const boot = await this.#simctl.invoke(["boot", udid], this.#left(deadline), set); + const refresh = + refreshBooted && + !this.#ownsDeviceSet && + set === undefined && + boot.kind === "finished" && + boot.result.code !== 0 && + alreadyBooted(boot.result.stderr); + if (!refresh) return boot; + await this.#simctl.shutdown(udid); + return await this.#simctl.invoke(["boot", udid], this.#left(deadline)); + } + + #left(deadline: number): number { + return Math.max(0, deadline - this.#clock.now()); + } + + async #waitForInstaller(udid: string, deadline: number, set?: string): Promise { + // why: bootstatus can finish before the installer accepts requests after an erase (#547). + while (this.#clock.now() < deadline) { + const outcome = await this.#simctl.invoke( + ["listapps", udid], + Math.min(COMMAND_TIMEOUT_MS, deadline - this.#clock.now()), + set, + ); + if (outcome.kind === "finished" && outcome.result.code === 0) { + return this.#clock.now() < deadline; + } + const remainingMs = deadline - this.#clock.now(); + if (remainingMs <= 0) break; + await new Promise((resolve) => { + this.#clock.setTimer(Math.min(INSTALLER_POLL_MS, remainingMs), resolve); + }); + } + return false; + } +} diff --git a/src/drivers/ios/slim-boot.ts b/src/drivers/ios/slim-boot.ts new file mode 100644 index 000000000..f2c3ef028 --- /dev/null +++ b/src/drivers/ios/slim-boot.ts @@ -0,0 +1,338 @@ +import { join } from "node:path"; +import { KeptCategoriesError, type DriverDevice, type DeviceMode } from "../../core/index.js"; +import type { Clock, Filesystem, Logger } from "../../ports/index.js"; + +import { errorMessage } from "./command-run.js"; + +import { Simctl } from "./simctl.js"; + +import { iosRuntimeVersionFromId } from "./runtime-version.js"; + +import type { IosDriverConfig } from "./definition.js"; +import { + overridesFor, + resolveSlimCategories, + slimSignature, + type SlimCategory, +} from "./slim-labels.js"; +import { assertKeptEnabled, splitByExcept } from "./slim-except.js"; + +import { parseOverridesPlist, parsePrintDisabled, renderOverridesPlist } from "./slim-overrides.js"; +import { type IosDriverData } from "./simulator-data.js"; + +// why: ADR 0060 §2: simslim's limit for a simulator just after its first boot. +const READBACK_TIMEOUT_MS = 120_000; +const OVERRIDES_FILE_NAME = "disabled.plist"; +export const DEFAULT_OVERRIDES_ROOT = "/private/var/tmp"; +const OVERRIDES_FOLDER_PREFIX = "com.apple.CoreSimulator.SimDevice."; + +export interface SlimmedFact { + readonly deviceId: string; + readonly address: string; + readonly categories: readonly string[]; + readonly labelCount: number; + readonly durationMs: number; + readonly signature: string; + readonly unknownLabels: readonly string[]; +} + +export interface SlimSkippedFact { + readonly deviceId: string; + readonly reason: "runtime-too-old" | "unknown-runtime" | "apply-failed"; + readonly detail: string; +} + +export type SlimOptions = IosDriverConfig["slim"]; + +type SlimWrite = + | { readonly kind: "no-categories"; readonly detail: string } + | { + readonly kind: "written"; + readonly resolved: ReturnType; + readonly wanted: ReadonlyMap; + readonly kept: readonly SlimCategory[]; + readonly startedAt: number; + readonly writeError: string | undefined; + }; + +type SlimPlan = + | { readonly kind: "off" } + | { readonly kind: "skip"; readonly reason: SlimSkippedFact["reason"]; readonly detail: string } + | { readonly kind: "apply"; readonly slim: SlimOptions }; + +interface SlimBootOptions { + readonly clock: Clock; + readonly filesystem: Filesystem; + readonly fullBootTimeoutMs: number; + readonly logger: Logger | undefined; + readonly onSlimmed: ((fact: SlimmedFact) => void) | undefined; + readonly onSlimSkipped: ((fact: SlimSkippedFact) => void) | undefined; + readonly overridesRoot: string; + readonly simctl: Simctl; + readonly slim: SlimOptions | undefined; +} + +/** why: the part of IosSimctlDriver (ADR 0060) that writes a simulator's launchd overrides so its managed services stay off, reads them back after the boot, and copies or removes them with the simulator. */ +export class SlimBoot { + readonly #clock: Clock; + readonly #filesystem: Filesystem; + readonly #fullBootTimeoutMs: number; + readonly #logger: Logger | undefined; + readonly #onSlimmed: ((fact: SlimmedFact) => void) | undefined; + readonly #onSlimSkipped: ((fact: SlimSkippedFact) => void) | undefined; + readonly #overridesRoot: string; + readonly #simctl: Simctl; + readonly #slim: SlimOptions | undefined; + + constructor(options: SlimBootOptions) { + this.#clock = options.clock; + this.#filesystem = options.filesystem; + this.#fullBootTimeoutMs = options.fullBootTimeoutMs; + this.#logger = options.logger; + this.#onSlimmed = options.onSlimmed; + this.#onSlimSkipped = options.onSlimSkipped; + this.#overridesRoot = options.overridesRoot; + this.#simctl = options.simctl; + this.#slim = options.slim; + } + + plan( + data: IosDriverData, + options: { readonly purpose: "prepare" | "recover"; readonly mode: DeviceMode }, + ): { readonly plan: SlimPlan; readonly bootstatusTimeoutMs: number } { + return planSlimBoot(data, this.#slim, this.#fullBootTimeoutMs, options); + } + + async settle( + device: DriverDevice, + data: IosDriverData, + plan: SlimPlan, + write: SlimWrite | undefined, + ): Promise { + if (plan.kind === "skip") { + this.#onSlimSkipped?.({ + deviceId: device.deviceId, + detail: plan.detail, + reason: plan.reason, + }); + } + + if (write === undefined) { + return this.#asIs(device, data, "full"); + } + if (write.kind === "no-categories") { + return this.#skipApply(device, data, write.detail, "full"); + } + + return this.#readBack(device, data, write); + } + + /** why: ADR 0060 §1: the slim file is in place before the copy's first boot, so the copy boots with what the seed was given. */ + async copyOverrides(seedUdid: string, copyUdid: string): Promise { + const from = join(this.#overridesFolder(seedUdid), OVERRIDES_FILE_NAME); + try { + if (!(await this.#filesystem.exists(from))) return; + await this.#filesystem.mkdirp(this.#overridesFolder(copyUdid), { mode: 0o700 }); + await this.#filesystem.copyFile( + from, + join(this.#overridesFolder(copyUdid), OVERRIDES_FILE_NAME), + ); + } catch (error: unknown) { + this.#logger?.warn( + `could not copy the launchd overrides of seed ${seedUdid} to ${copyUdid}: ${errorMessage(error)}`, + ); + } + } + + #asIs(device: DriverDevice, data: IosDriverData, mode: DeviceMode): DriverDevice { + return { + address: data.udid, + deviceId: device.deviceId, + driverData: device.driverData, + mode, + }; + } + + async writeOverrides( + udid: string, + slim: SlimOptions, + except: readonly string[] = [], + ): Promise { + const configured = resolveSlimCategories(slim.categories); + const { kept, resolved } = splitByExcept(configured, except); + if (configured.categories.length === 0) { + return { + detail: `none of the configured slim categories (${(slim.categories ?? []).join(", ")}) are known`, + kind: "no-categories", + }; + } + + const startedAt = this.#clock.now(); + const wanted = overridesFor(resolved.categories, kept); + let writeError: string | undefined; + try { + const folder = this.#overridesFolder(udid); + const file = join(folder, OVERRIDES_FILE_NAME); + const existing = await this.#readOverrides(file); + const merged = new Map([...existing, ...wanted]); + if (!sameEntries(existing, merged)) { + await this.#filesystem.mkdirp(folder, { mode: 0o700 }); + await this.#filesystem.writeFileAtomic(file, renderOverridesPlist(merged), { mode: 0o644 }); + } + } catch (error: unknown) { + writeError = `could not write the launchd overrides file: ${errorMessage(error)}`; + } + + return { kept, kind: "written", resolved, startedAt, wanted, writeError }; + } + + async #readOverrides(file: string): Promise> { + if (!(await this.#filesystem.exists(file))) return new Map(); + const parsed = parseOverridesPlist(await this.#filesystem.readFile(file)); + if (parsed === undefined) + throw new Error("the existing file is not a property list dictionary"); + return parsed; + } + + #overridesFolder(udid: string): string { + return join(this.#overridesRoot, `${OVERRIDES_FOLDER_PREFIX}${udid}`); + } + + async #readBack( + device: DriverDevice, + data: IosDriverData, + write: Extract, + ): Promise { + if (write.kept.length > 0 && write.writeError !== undefined) { + throw new KeptCategoriesError(write.writeError); + } + const outcome = await this.#simctl.invoke( + ["spawn", data.udid, "launchctl", "print-disabled", "system"], + READBACK_TIMEOUT_MS, + ); + const details = write.writeError === undefined ? [] : [write.writeError]; + + if (outcome.kind === "timed-out" || outcome.result.code !== 0) { + details.push(`could not read back the launchd overrides: ${readBackFailure(outcome)}`); + if (write.kept.length > 0) throw new KeptCategoriesError(details.join("; ")); + return this.#skipApply(device, data, details.join("; "), "slim"); + } + return this.#judgeReadBack(device, data, write, details, outcome.result.stdout); + } + + #judgeReadBack( + device: DriverDevice, + data: IosDriverData, + write: Extract, + details: string[], + stdout: string, + ): DriverDevice { + const read = parsePrintDisabled(stdout); + assertKeptEnabled(write.kept, read); + const mismatched = [...write.wanted.keys()] + .sort() + .filter((label) => (read.get(label) ?? false) !== write.wanted.get(label)); + if (mismatched.length > 0) { + details.push( + `${String(mismatched.length)} of ${String(write.wanted.size)} managed labels did not read back as written at boot; first: ${mismatched[0] ?? ""}`, + ); + return this.#skipApply(device, data, details.join("; "), "full"); + } + if (details.length > 0) { + return this.#skipApply(device, data, details.join("; "), "slim"); + } + + // why: an except list naming the whole slim set slims nothing: full, no `device.slimmed`. + if (write.resolved.categories.length === 0) return this.#asIs(device, data, "full"); + this.#reportSlimmed(device, data, write); + return this.#asIs(device, data, "slim"); + } + + #reportSlimmed( + device: DriverDevice, + data: IosDriverData, + write: Extract, + ): void { + this.#onSlimmed?.({ + address: data.udid, + categories: write.resolved.categories.map((category) => category.name), + deviceId: device.deviceId, + durationMs: this.#clock.now() - write.startedAt, + labelCount: [...write.wanted.values()].filter(Boolean).length, + signature: slimSignature(write.resolved.categories), + unknownLabels: [...write.resolved.unknown], + }); + } + + #skipApply( + device: DriverDevice, + data: IosDriverData, + detail: string, + mode: DeviceMode, + ): DriverDevice { + this.#onSlimSkipped?.({ deviceId: device.deviceId, detail, reason: "apply-failed" }); + return this.#asIs(device, data, mode); + } + + // why: simctl delete leaves the folder behind; a failed removal must not fail a destroy that + // already deleted the simulator. + async removeOverrides(udid: string): Promise { + try { + await this.#filesystem.rm(this.#overridesFolder(udid)); + } catch (error: unknown) { + this.#logger?.warn( + `could not remove the launchd overrides folder of ${udid}: ${errorMessage(error)}`, + ); + } + } +} + +export function supportsPersistentSlim(version: readonly [number, number] | undefined): boolean { + if (version === undefined) { + return false; + } + const [major, minor] = version; + return major > 18 || (major === 18 && minor >= 5); +} + +function planSlimBoot( + data: IosDriverData, + slim: SlimOptions | undefined, + fullBootTimeoutMs: number, + options: { readonly purpose: "prepare" | "recover"; readonly mode: DeviceMode }, +): { readonly plan: SlimPlan; readonly bootstatusTimeoutMs: number } { + if (options.purpose === "recover" || slim === undefined || options.mode !== "slim") { + return { bootstatusTimeoutMs: fullBootTimeoutMs, plan: { kind: "off" } }; + } + + const version = iosRuntimeVersionFromId(data.runtimeId); + if (!supportsPersistentSlim(version)) { + return { + bootstatusTimeoutMs: fullBootTimeoutMs, + plan: { + detail: + version === undefined + ? `runtimeId "${data.runtimeId}" does not parse to an iOS version` + : `iOS ${String(version[0])}.${String(version[1])} is below the 18.5 persistent-override floor`, + kind: "skip", + reason: version === undefined ? "unknown-runtime" : "runtime-too-old", + }, + }; + } + + return { bootstatusTimeoutMs: slim.bootTimeoutMs, plan: { kind: "apply", slim } }; +} + +function sameEntries(a: ReadonlyMap, b: ReadonlyMap): boolean { + return a.size === b.size && [...a].every(([label, disabled]) => b.get(label) === disabled); +} + +function readBackFailure( + outcome: + | { readonly kind: "timed-out" } + | { readonly kind: "finished"; readonly result: { code: number | null; stderr: string } }, +): string { + return outcome.kind === "timed-out" + ? `timed out after ${String(READBACK_TIMEOUT_MS)}ms` + : `exit ${String(outcome.result.code)}: ${outcome.result.stderr.trim()}`; +} diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index d8887b03d..7968754b9 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -28,7 +28,6 @@ type SizedFile = { path: string; lines: number }; */ const SHRINK_ONLY: Record = { "src/cli/index.test.ts": 11674, - "src/drivers/ios/index.ts": 2700, }; const CODE_LIMIT = 1400; From 3ccccd06314b09eb3e15a6d07f0956074c5f73d8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:36:59 +0100 Subject: [PATCH 62/83] test(core): the provision call-site scan allows the iOS driver handing provision to its seeds part --- src/core/device-provisioner.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/device-provisioner.test.ts b/src/core/device-provisioner.test.ts index 4fadff3d4..abae8d8c6 100644 --- a/src/core/device-provisioner.test.ts +++ b/src/core/device-provisioner.test.ts @@ -458,9 +458,10 @@ describe("DeviceProvisioner and a component removal (ADR 0010 §8)", () => { [join("leasing", "acquisition-workflow.ts")]: 1, // The same, for a device the warm pool creates for a target. [join("core", "warm-pool", "converger.ts")]: 1, - // why: not a second creator: the Android driver's own `provision` hands the call to its seeds - // part, and its test helpers (never loaded in production) drive the driver to build fixtures. + // why: not a second creator: each driver's own `provision` hands the call to its seeds part, + // and the Android test helpers (never loaded in production) drive the driver to build fixtures. [join("drivers", "android", "index.ts")]: 1, + [join("drivers", "ios", "index.ts")]: 1, [join("drivers", "android", "test-driver-harness.ts")]: 1, [join("drivers", "android", "test-seed-harness.ts")]: 1, // why: the harness's fixture devices, as for registerDevice above. From 3237cfe5d67644dcbba097905641756c27d5b0c8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:48:03 +0100 Subject: [PATCH 63/83] test(core): the call-site scans name the test helpers the test splits created --- src/core/device-provisioner.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/device-provisioner.test.ts b/src/core/device-provisioner.test.ts index abae8d8c6..909d25cfa 100644 --- a/src/core/device-provisioner.test.ts +++ b/src/core/device-provisioner.test.ts @@ -446,8 +446,11 @@ describe("DeviceProvisioner and a component removal (ADR 0010 §8)", () => { // doctor and registry tests split out of one file (ADR 0063). [join("core", "test-doctor.ts")]: 2, [join("core", "test-registry.ts")]: 1, - // why: the harness the lease-acquisition coordinator tests split out of one file share (ADR 0063). + [join("core", "test-registry-corrupt-row.ts")]: 1, + // why: the harnesses the lease-acquisition coordinator and createLeasing tests split out of one + // file share (ADR 0063). [join("leasing", "test-lease-acquisition.ts")]: 1, + [join("leasing", "test-create-leasing.ts")]: 1, // why: the dispatcher tests' seeded warm device, shared by the files that start from one. [join("daemon", "test-dispatcher.ts")]: 1, }); @@ -464,8 +467,11 @@ describe("DeviceProvisioner and a component removal (ADR 0010 §8)", () => { [join("drivers", "ios", "index.ts")]: 1, [join("drivers", "android", "test-driver-harness.ts")]: 1, [join("drivers", "android", "test-seed-harness.ts")]: 1, + [join("drivers", "android", "test-seeds.ts")]: 1, // why: the harness's fixture devices, as for registerDevice above. [join("leasing", "test-lease-acquisition.ts")]: 1, + [join("leasing", "test-create-leasing.ts")]: 1, + [join("core", "test-quarantine-coordinator.ts")]: 1, [join("daemon", "test-dispatcher.ts")]: 1, }); }); From 823e7ba42002bd6775d4d695d02bbf09f8388b39 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:35:42 +0100 Subject: [PATCH 64/83] test(drivers): split session.test.ts by topic --- .../screen/session-body-and-close.test.ts | 178 ++++ .../screen/session-phone-screen-state.test.ts | 239 ++++++ src/drivers/android/screen/session.test.ts | 778 +----------------- src/drivers/android/screen/test-session.ts | 378 +++++++++ 4 files changed, 816 insertions(+), 757 deletions(-) create mode 100644 src/drivers/android/screen/session-body-and-close.test.ts create mode 100644 src/drivers/android/screen/session-phone-screen-state.test.ts create mode 100644 src/drivers/android/screen/test-session.ts diff --git a/src/drivers/android/screen/session-body-and-close.test.ts b/src/drivers/android/screen/session-body-and-close.test.ts new file mode 100644 index 000000000..b0a3b2552 --- /dev/null +++ b/src/drivers/android/screen/session-body-and-close.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from "vitest"; + +import { SCREEN_FRAME_TAG, ScreenUnavailableError } from "../../../core/index.js"; + +import { ALIVE_AFTER_MS, CLOSE_DEADLINE_MS } from "./session.js"; +import { + DEVICE_JAR, + SETTINGS, + DEVICE, + PHONE, + push, + shell, + remove, + setup, + drive, + hasEnded, + frameTag, + waitThenStart, + opened, +} from "./test-session.js"; +import { mediaPacket, SAMPLE } from "./test-scrcpy.js"; + +describe("Android screen capture: the body", () => { + it("reads the socket only when its reader asks for a frame", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.target.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + const before = peer.pulls; + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + expect(peer.pulls).toBe(before); + const reader = feed.body.getReader(); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.meta); + expect(peer.pulls).toBe(before); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.description); + expect(peer.pulls).toBeGreaterThan(before); + expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.keyframe); + }); + + it("delivers every delta, in order, however slowly it is read", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.target.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + // why: a delta frame carries only the picture, so each picture ends in its own index byte. + for (let index = 0; index < 50; index += 1) { + const picture = Uint8Array.from(SAMPLE.p1); + picture[picture.length - 1] = index + 1; + peer.send(mediaPacket({ pts: index }, picture)); + } + const reader = feed.body.getReader(); + const deltas: (number | undefined)[] = []; + for (let index = 0; index < 53; index += 1) { + for (let flush = 0; flush < 3; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + const frame = (await reader.read()).value; + if (frameTag(frame) === SCREEN_FRAME_TAG.delta) deltas.push(frame?.at(-1)); + } + expect(deltas).toEqual(Array.from({ length: 50 }, (_, index) => index + 1)); + }); + + it("says a still screen is alive every few seconds, and keeps the frames that follow in order", async () => { + const world = await setup(); + const feed = await opened(world); + const peer = world.target.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); + const reader = feed.body.getReader(); + for (let index = 0; index < 3; index += 1) await reader.read(); + const flushAll = async (): Promise => { + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + }; + for (const round of [1, 2]) { + const quiet = reader.read(); + let settled = false; + void quiet.then(() => (settled = true)); + await flushAll(); + world.clock.advance(ALIVE_AFTER_MS - 1); + await flushAll(); + expect(settled, `round ${String(round)} before ${String(ALIVE_AFTER_MS)} ms`).toBe(false); + world.clock.advance(1); + const alive = (await quiet).value; + expect(frameTag(alive), `round ${String(round)}`).toBe(SCREEN_FRAME_TAG.alive); + expect(alive?.length, `round ${String(round)}`).toBe(5); + } + const next = reader.read(); + const first = Uint8Array.from(SAMPLE.p1); + first[first.length - 1] = 1; + const second = Uint8Array.from(SAMPLE.p1); + second[second.length - 1] = 2; + peer.send(mediaPacket({ pts: 1 }, first), mediaPacket({ pts: 2 }, second)); + const deltas = [(await next).value, (await reader.read()).value]; + expect(deltas.map(frameTag)).toEqual([SCREEN_FRAME_TAG.delta, SCREEN_FRAME_TAG.delta]); + expect(deltas.map((frame) => frame?.at(-1))).toEqual([1, 2]); + }); + + it("says nothing is alive while the encoder has not painted a first picture, on an emulator and on a phone", async () => { + for (const device of [DEVICE, PHONE]) { + const world = await setup({ device }); + const opening = world.capture.open(device, SETTINGS, new AbortController().signal); + const feed = await drive(world.clock, waitThenStart(world.target, opening)); + const peer = world.target.server.last; + peer.send(mediaPacket({ config: true }, SAMPLE.config)); + const reader = feed.body.getReader(); + expect(frameTag((await reader.read()).value), device.deviceId).toBe(SCREEN_FRAME_TAG.meta); + const quiet = reader.read(); + let settled = false; + void quiet.then(() => (settled = true)); + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + world.clock.advance(ALIVE_AFTER_MS * 3); + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + expect(settled, device.deviceId).toBe(false); + } + }); + + it("errors with the server's reason when the device side ends, and cleans up", async () => { + const world = await setup({ + expectations: [push(), shell({ stderrLines: ["[server] ERROR: Encoding error"] }), remove()], + }); + const feed = await opened(world); + const reader = feed.body.getReader(); + await reader.read(); + world.target.server.last.close(); + const error = await drive(world.clock, reader.read()).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message).toMatch(/Encoding error/); + await drive(world.clock, feed.close()); + expect(world.target.steps.at(-1)).toBe(`shell rm -f ${DEVICE_JAR}`); + }); +}); + +describe("Android screen capture: close", () => { + it("closes the socket, kills a server that outlives it, and removes the jar, once however often it is called", async () => { + const world = await setup(); + const feed = await opened(world); + const closes = Promise.all([feed.close(), feed.close()]); + await drive(world.clock, closes); + await drive(world.clock, feed.close()); + expect(world.target.server.last.closed).toBe(true); + expect(await hasEnded(world.target.runner.handles[1]!)).toBe(true); + expect(world.target.log.filter((entry) => entry.includes("rm"))).toHaveLength(1); + }); + + it("kills a server still running a second after its socket closed, with SIGKILL", async () => { + const world = await setup({ + expectations: [push(), shell({ ignoresSigterm: true }), remove()], + }); + const feed = await opened(world); + const closing = feed.close(); + const server = world.target.runner.handles[1]!; + await hasEnded(server); + world.clock.advance(999); + expect(await hasEnded(server)).toBe(false); + world.clock.advance(1); + expect(await hasEnded(server)).toBe(true); + await drive(world.clock, closing); + }); + + it("resolves within its deadline even when removing the jar never returns", async () => { + const world = await setup({ expectations: [push(), shell(), { ...remove(), hangs: true }] }); + const feed = await opened(world); + const started = world.clock.now(); + await drive(world.clock, feed.close()); + expect(world.clock.now() - started).toBeGreaterThanOrEqual(CLOSE_DEADLINE_MS); + expect(world.clock.now() - started).toBeLessThanOrEqual(CLOSE_DEADLINE_MS + 50); + }); + + it("cancelling the body closes the capture", async () => { + const world = await setup(); + const feed = await opened(world); + await drive(world.clock, feed.body.cancel()); + expect(world.target.server.last.closed).toBe(true); + expect(world.target.log.filter((entry) => entry.includes("rm"))).toHaveLength(1); + }); +}); diff --git a/src/drivers/android/screen/session-phone-screen-state.test.ts b/src/drivers/android/screen/session-phone-screen-state.test.ts new file mode 100644 index 000000000..f2247336d --- /dev/null +++ b/src/drivers/android/screen/session-phone-screen-state.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; + +import { SCREEN_FRAME_TAG, ScreenUnavailableError } from "../../../core/index.js"; + +import { START_DEADLINE_MS } from "./session.js"; +import { + SERVER, + DEVICE_JAR, + SETTINGS, + SCREEN_READ_MS, + WAKE_MS, + PHONE, + GATE_REFUSAL, + SCREEN_OFF, + FakeTarget, + setup, + drive, + streamStart, + frameTag, + waitThenStart, +} from "./test-session.js"; + +describe("Android screen capture: a phone's screen state", () => { + it("wakes a phone whose screen is off through its target with a 10 s limit, reads again, and pushes once the second read says it is on, for each wakefulness that means off", async () => { + for (const printed of ["Asleep", "Dozing", "Asleep\r", "Dozing "]) { + const world = await setup({ + device: PHONE, + power: [ + { code: 0, stdout: ` mWakefulness=${printed}\n` }, + { code: 0, stdout: " mWakefulness=Awake\n" }, + ], + }); + const opening = world.capture.open(PHONE, SETTINGS, new AbortController().signal); + const feed = await drive(world.clock, waitThenStart(world.target, opening)); + expect(frameTag((await feed.body.getReader().read()).value), printed).toBe( + SCREEN_FRAME_TAG.meta, + ); + + expect(world.target.screenSteps, printed).toEqual(["read", "wake", "read"]); + expect(world.target.wakes, printed).toEqual([WAKE_MS]); + expect(world.target.powerReads, printed).toEqual([SCREEN_READ_MS, SCREEN_READ_MS]); + expect(world.target.log[0], printed).toBe(`push ${SERVER} ${DEVICE_JAR}`); + } + }); + + it("refuses a phone whose second read still says the screen is off, with its own reason, and pushes nothing, whatever the wake did", async () => { + for (const second of ["Asleep", "Dozing"]) { + for (const wake of ["ok", "fail", "slow"] as const) { + const world = await setup({ + device: PHONE, + wake, + power: [ + { code: 0, stdout: " mWakefulness=Asleep\n" }, + { code: 0, stdout: ` mWakefulness=${second}\n` }, + ], + }); + const error = await drive( + world.clock, + world.capture.open(PHONE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + + const name = `${second} after a wake that is ${wake}`; + expect(error, name).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message, name).toBe(SCREEN_OFF); + // why: a wake that exited non-zero or timed out is not looked at; the second read decides. + expect(world.target.screenSteps, name).toEqual(["read", "wake", "read"]); + expect(world.target.steps, name).toEqual([]); + expect(world.target.connects, name).toEqual([]); + } + } + }); + + it.each([ + { name: "Awake", power: { code: 0, stdout: " mWakefulness=Awake\n" } }, + { name: "Dreaming", power: { code: 0, stdout: " mWakefulness=Dreaming\n" } }, + { name: "Awaken", power: { code: 0, stdout: " mWakefulness=Awaken\n" } }, + { name: "asleep", power: { code: 0, stdout: " mWakefulness=asleep\n" } }, + { name: "no matching line", power: { code: 0, stdout: " mPolicyBoost=0\n" } }, + { name: "a non-zero exit", power: { code: 1, stdout: " mWakefulness=Asleep\n" } }, + { name: "a slow read", power: "slow" }, + ] as const)( + "goes on to the push, without a wake, after one 10 s screen read that says $name", + async ({ name, power }) => { + const world = await setup({ device: PHONE, power }); + const opening = world.capture.open(PHONE, SETTINGS, new AbortController().signal); + // why: a timed-out read ends 10 s of fake clock after the open, and waitThenStart's capped + // flush-wait cannot span it; the file's uncapped watcher pattern waits for the peer whenever + // it appears, and drive still ends the run if the open never reaches it. + const watcher = (async () => { + while (world.target.server.peers.length === 0) + await new Promise((resolve) => setImmediate(resolve)); + world.target.server.last.send(streamStart()); + })(); + const feed = await drive(world.clock, opening); + await watcher; + expect(frameTag((await feed.body.getReader().read()).value), name).toBe( + SCREEN_FRAME_TAG.meta, + ); + + expect(world.target.powerReads, name).toEqual([SCREEN_READ_MS]); + expect(world.target.wakes, name).toEqual([]); + expect(world.target.log[0], name).toBe(`push ${SERVER} ${DEVICE_JAR}`); + }, + ); + + it("checks for a stop after each of the first read, the wake and the second read, and does nothing after the step it was stopped in", async () => { + const asleep = { code: 0, stdout: " mWakefulness=Asleep\n" }; + const awake = { code: 0, stdout: " mWakefulness=Awake\n" }; + const cases = [ + { name: "the first read", power: [asleep], wake: "ok", wakes: 0, reads: 1, hold: true }, + { name: "the wake", power: [asleep, awake], wake: "slow", wakes: 1, reads: 1, hold: false }, + { + name: "the second read", + power: [asleep, "slow"], + wake: "ok", + wakes: 1, + reads: 2, + hold: false, + }, + ] as const; + for (const stage of cases) { + const stop = new AbortController(); + const world = await setup({ + device: PHONE, + power: stage.power as unknown as FakeTarget["power"], + wake: stage.wake, + }); + world.target.holdReads = stage.hold; + const opening = world.capture.open(PHONE, SETTINGS, stop.signal); + // why: stops at the step itself: the step is in flight once its entry is recorded. + while ( + stage.name === "the first read" + ? world.target.powerReads.length < 1 + : stage.name === "the wake" + ? world.target.wakes.length < 1 + : world.target.powerReads.length < 2 + ) + await new Promise((resolve) => setImmediate(resolve)); + stop.abort(); + const failed = await drive(world.clock, opening).catch((caught: unknown) => caught); + + expect(failed, stage.name).toBeInstanceOf(ScreenUnavailableError); + expect(world.target.wakes.length, stage.name).toBe(stage.wakes); + expect(world.target.powerReads.length, stage.name).toBe(stage.reads); + expect(world.target.log, stage.name).toEqual([]); + expect(world.target.connects, stage.name).toEqual([]); + } + }); + + it("fails the open with a target's own reason and runs nothing after it, for the first screen read, the wake, the second screen read, the push, the server start and a connection attempt", async () => { + const stages: readonly { + readonly name: string; + readonly power?: FakeTarget["power"]; + readonly wake?: FakeTarget["wake"]; + readonly refuses: (args: readonly string[]) => boolean; + readonly screenBefore: string[]; + readonly stepsBefore: string[]; + }[] = [ + { + name: "the first screen read", + power: "refuse", + refuses: () => false, + screenBefore: ["read"], + stepsBefore: [], + }, + { + // why: the wake's exit is ignored, but a closed check at the wake still refuses the open. + name: "the wake", + power: { code: 0, stdout: " mWakefulness=Asleep\n" }, + wake: "refuse", + refuses: () => false, + screenBefore: ["read", "wake"], + stepsBefore: [], + }, + { + name: "the second screen read", + power: [{ code: 0, stdout: " mWakefulness=Asleep\n" }, "refuse"], + refuses: () => false, + screenBefore: ["read", "wake", "read"], + stepsBefore: [], + }, + { + name: "the push", + refuses: (args) => args[0] === "push", + screenBefore: ["read"], + stepsBefore: [], + }, + { + name: "the server start", + refuses: (args) => args.includes("app_process"), + screenBefore: ["read"], + stepsBefore: [`push ${SERVER} ${DEVICE_JAR}`, `shell rm -f ${DEVICE_JAR}`], + }, + { + // why: the server's spawn sits between the push and the connection attempt, so it is in the + // log by the time the attempt's check refuses; only the `rm -f` follows it. + name: "a connection attempt", + refuses: (args) => args[0] === "connect", + screenBefore: ["read"], + stepsBefore: [`push ${SERVER} ${DEVICE_JAR}`, "server", `shell rm -f ${DEVICE_JAR}`], + }, + ]; + for (const stage of stages) { + const world = await setup({ + device: PHONE, + ...(stage.power === undefined ? {} : { power: stage.power }), + ...(stage.wake === undefined ? {} : { wake: stage.wake }), + refuses: stage.refuses, + }); + const started = world.clock.now(); + const error = await drive( + world.clock, + world.capture.open(PHONE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + + expect(error, stage.name).toBeInstanceOf(ScreenUnavailableError); + expect((error as Error).message, stage.name).toBe(GATE_REFUSAL); + // why: a refusal that was retried instead of propagated would have spent the start deadline on + // it, and the viewer would get a start timeout instead of the check's reason. + expect(world.clock.now() - started, stage.name).toBeLessThan(START_DEADLINE_MS); + expect(world.target.steps, stage.name).toEqual(stage.stepsBefore); + expect(world.target.screenSteps, stage.name).toEqual(stage.screenBefore); + } + }); + + it("sends nothing and settles when the version check is still closed at the jar's removal", async () => { + const world = await setup({ + device: PHONE, + refuses: (args) => args.includes("app_process") || args[0] === "shell", + }); + const error = await drive( + world.clock, + world.capture.open(PHONE, SETTINGS, new AbortController().signal), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(ScreenUnavailableError); + expect(world.target.steps).toEqual([`push ${SERVER} ${DEVICE_JAR}`]); + }); +}); diff --git a/src/drivers/android/screen/session.test.ts b/src/drivers/android/screen/session.test.ts index 693bea75a..fb78df1fb 100644 --- a/src/drivers/android/screen/session.test.ts +++ b/src/drivers/android/screen/session.test.ts @@ -1,388 +1,27 @@ import { describe, expect, it } from "vitest"; -import { - type PhysicalDriverDevice, - SCREEN_FRAME_TAG, - ScreenUnavailableError, - type VirtualDriverDevice, -} from "../../../core/index.js"; -import { - FakeClock, - MemoryFilesystem, - type ProcessResult, - ScriptedProcessRunner, - type ScriptedProcessExpectation, - type TcpStream, -} from "../../../ports/index.js"; -import { - ALIVE_AFTER_MS, - CLOSE_DEADLINE_MS, - CONNECT_RETRY_MS, - createAndroidScreenCapture, - type ScreenTarget, - START_DEADLINE_MS, -} from "./session.js"; -import { codecId, concatBytes, mediaPacket, SAMPLE, sessionPacket } from "./test-scrcpy.js"; - -const ADB = "/sdk/platform-tools/adb"; -const SERVER = "/simlock/scrcpy-server-v5.0.1"; -const SERIAL = "emulator-5586"; -const PHONE_SERIAL = "R58M123ABC"; -const SCID = 0x1234abcd; -const DEVICE_JAR = "/data/local/tmp/simlock-scrcpy-1234abcd.jar"; -const SETTINGS = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; -const SCREEN_READ_MS = 10_000; -const WAKE_MS = 10_000; -const DEVICE: VirtualDriverDevice = { - address: SERIAL, - deviceId: "simlock_1", - driverData: {}, - physical: false, -}; -const PHONE: PhysicalDriverDevice = { - address: PHONE_SERIAL, - deviceId: PHONE_SERIAL, - driverData: {}, - enrolledApps: [], - physical: true, -}; - -// why: the reason a phone's target gives when the shared adb server's version check is closed. The -// capture is expected to pass it through as its own, unchanged, and to run nothing after it. -const GATE_REFUSAL = - "the adb server on port 5037 speaks protocol 40 and Simlock's adb speaks 41. " + - "No command goes to physical Android devices until they match, and Simlock will not restart that server."; -const SCREEN_OFF = - "The phone's screen is off and did not turn on. Turn it on and ask for the stream again."; -const WAKE_ARGS = "shell input keyevent KEYCODE_WAKEUP"; - -type Behaviour = "accept" | "refuse" | "no-transport" | "connection-refused" | "silent"; - -class Peer { - readonly written: string[] = []; - closed = false; - pulls = 0; - readonly #queue: Uint8Array[] = []; - #wake: (() => void) | undefined; - readonly stream: TcpStream; - - constructor() { - const readable = new ReadableStream( - { - cancel: () => { - this.close(); - }, - pull: async (controller) => { - this.pulls += 1; - while (this.#queue.length === 0 && !this.closed) { - await new Promise((resolve) => { - this.#wake = resolve; - }); - } - const next = this.#queue.shift(); - if (next === undefined) controller.close(); - else controller.enqueue(next); - }, - }, - { highWaterMark: 0 }, - ); - this.stream = { - close: () => { - this.close(); - }, - end: () => undefined, - readable, - write: async (bytes) => { - this.written.push(typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes)); - }, - }; - } - - send(...chunks: (Uint8Array | string)[]): void { - for (const chunk of chunks) { - this.#queue.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk); - } - this.#wake?.(); - } - - close(): void { - this.closed = true; - this.#wake?.(); - } -} - -// why: the adb server on the far side of a target's `connect`; what it answers is scripted per attempt. -class AdbServer { - readonly peers: Peer[] = []; - readonly #behaviours: Behaviour[]; - - constructor( - behaviours: Behaviour[] = [], - readonly otherwise: Behaviour = "accept", - // why: a peer that stays silent and open until this moment of the fake clock, then refuses. - private readonly acceptFromMs: number | undefined = undefined, - ) { - this.#behaviours = behaviours; - } - - newPeer(clock: FakeClock): Peer | undefined { - if (this.acceptFromMs !== undefined && clock.now() < this.acceptFromMs) { - const waiting = new Peer(); - waiting.send("OKAY", "FAIL0006closed"); - return waiting; - } - const behaviour = this.#behaviours.shift() ?? this.otherwise; - if (behaviour === "connection-refused") throw new Error("connect ECONNREFUSED 127.0.0.1"); - const peer = new Peer(); - this.peers.push(peer); - if (behaviour === "no-transport") peer.send(`FAIL0020device '${SERIAL}' not found`); - if (behaviour === "refuse") peer.send("OKAY", "FAIL0006closed"); - if (behaviour === "accept") peer.send("OKAY", "OKAY", new Uint8Array([0])); - return peer; - } - - get last(): Peer { - const peer = this.peers.at(-1); - if (peer === undefined) throw new Error("no connection"); - return peer; - } -} - -interface PowerAnswer { - readonly code: number | null; - readonly stdout: string; -} - -// why: the device's capture target as the driver builds it. `args` start at the adb subcommand, so -// the serial and the adb server are the target's and not the capture's: this fake records what it -// was asked to run, and the commands themselves are the scripted runner's, so a push can fail, a -// server can hang and a kill can be escalated exactly as before the target existed. -class FakeTarget implements ScreenTarget { - readonly log: string[] = []; - readonly powerReads: number[] = []; - readonly wakes: number[] = []; - // why: the order of the screen steps, so "reads again after the wake" can fail. - readonly screenSteps: string[] = []; - readonly connects: number[] = []; - readonly runner: ScriptedProcessRunner; - readonly server: AdbServer; - // why: `"refuse"` is the version check refusing the read, `"slow"` a read that ends later. - // why: a list answers the reads in turn, the last one for every read after it. - power: PowerAnswer | "refuse" | "slow" | readonly (PowerAnswer | "refuse" | "slow")[] = { - code: 0, - stdout: " mWakefulness=Awake\n", - }; - // why: what the wake does: exits 0, exits non-zero, hangs to its limit, or is refused by the check. - wake: "ok" | "fail" | "slow" | "refuse" = "ok"; - // why: holds each screen read the check lets through in flight for the read limit before it answers as scripted (a refused read throws before the hold), so a stop can arrive while a read that will say Asleep is still running. - holdReads = false; - // why: stands in for a check that closes mid-session: a command whose args this picks out is refused as a closed check refuses it. - refuses: (args: readonly string[]) => boolean = () => false; - // why: stand-ins for the check a target runs before its own command or connection, so a bound can be measured against it. - spawnDelayMs = 0; - connectDelayMs = 0; - #clock: FakeClock; - - constructor( - readonly serial: string, - clock: FakeClock, - expectations: ScriptedProcessExpectation[], - server: AdbServer | undefined, - ) { - this.#clock = clock; - this.runner = new ScriptedProcessRunner(expectations); - this.server = server ?? new AdbServer(); - } - - get steps(): string[] { - return this.log.map((entry) => (entry.includes("app_process") ? "server" : entry)); - } - - async run(args: readonly string[], timeoutMs: number): Promise { - if (isScreenRead(args)) return await this.#readScreen(timeoutMs); - if (args.join(" ") === WAKE_ARGS) { - this.wakes.push(timeoutMs); - this.screenSteps.push("wake"); - if (this.wake === "refuse") throw new ScreenUnavailableError(GATE_REFUSAL); - if (this.wake === "slow") { - await this.#sleep(WAKE_MS); - return { code: null, stderr: "", stdout: "" }; - } - return { code: this.wake === "fail" ? 1 : 0, stderr: "", stdout: "" }; - } - if (this.refuses(args)) throw new ScreenUnavailableError(GATE_REFUSAL); - this.log.push(args.join(" ")); - return await this.runner.run(ADB, args); - } - - async spawn(args: readonly string[]): Promise { - if (this.spawnDelayMs > 0) await this.#sleep(this.spawnDelayMs); - if (this.refuses(args)) throw new ScreenUnavailableError(GATE_REFUSAL); - this.log.push(args.join(" ")); - return await this.runner.spawn(ADB, args); - } - - async connect(timeoutMs: number): Promise { - this.connects.push(timeoutMs); - if (this.refuses(["connect"])) throw new ScreenUnavailableError(GATE_REFUSAL); - if (this.connectDelayMs > 0) await this.#sleep(this.connectDelayMs); - const peer = this.server.newPeer(this.#clock); - if (peer === undefined) throw new Error("connect ECONNREFUSED 127.0.0.1"); - return peer.stream; - } - - async #readScreen(timeoutMs: number): Promise { - const answer = Array.isArray(this.power) - ? (this.power[Math.min(this.powerReads.length, this.power.length - 1)] as PowerAnswer) - : (this.power as PowerAnswer | "refuse" | "slow"); - this.powerReads.push(timeoutMs); - this.screenSteps.push("read"); - if (answer === "refuse") throw new ScreenUnavailableError(GATE_REFUSAL); - if (this.holdReads || answer === "slow") await this.#sleep(SCREEN_READ_MS); - if (answer === "slow") return { code: null, stderr: "", stdout: "" }; - return { code: answer.code, stderr: "", stdout: answer.stdout }; - } - - #sleep(ms: number): Promise { - return new Promise((resolve) => { - this.#clock.setTimer(ms, resolve); - }); - } -} +import { SCREEN_FRAME_TAG, ScreenUnavailableError } from "../../../core/index.js"; -function isScreenRead(args: readonly string[]): boolean { - return args.join(" ") === "shell dumpsys power"; -} - -function push(result: Partial<{ code: number; stderr: string }> = {}): ScriptedProcessExpectation { - return { - match: { - args: ["push", SERVER, /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/], - command: ADB, - }, - result: { code: result.code ?? 0, stderr: result.stderr ?? "", stdout: "" }, - }; -} - -// why: a phone's server command carries one more option (stay_awake) than an emulator's 20 arguments. -function shell( - extra: Partial = {}, - options = 20, -): ScriptedProcessExpectation { - return { - hangs: true, - match: { - args: ["shell", ...Array.from({ length: options }, () => /.*/)], - command: ADB, - }, - ...extra, - }; -} - -function remove(): ScriptedProcessExpectation { - return { - match: { - args: ["shell", "rm", "-f", /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/], - command: ADB, - }, - }; -} - -// why: both devices' worlds — the other device's target is here so "it went through this device's -// target" can fail; a capture that aimed at the wrong device would show up in the other's lists. -async function setup( - options: { - readonly acceptFromMs?: number; - readonly device?: VirtualDriverDevice | PhysicalDriverDevice; - readonly expectations?: ScriptedProcessExpectation[]; - readonly jar?: boolean; - readonly power?: FakeTarget["power"]; - readonly wake?: FakeTarget["wake"]; - readonly refuses?: (args: readonly string[]) => boolean; - readonly scid?: (() => number) | null; - readonly server?: AdbServer; - } = {}, -) { - const clock = new FakeClock(1_000); - const filesystem = new MemoryFilesystem(); - if (options.jar !== false) { - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(SERVER, "jar"); - } - const device = options.device ?? DEVICE; - const phone = device.physical === true; - const server = - options.server ?? - new AdbServer( - [], - "accept", - options.acceptFromMs === undefined ? undefined : 1_000 + options.acceptFromMs, - ); - const target = new FakeTarget( - phone ? PHONE_SERIAL : SERIAL, - clock, - options.expectations ?? [push(), shell({}, phone ? 21 : 20), remove()], - server, - ); - const scripted = { power: options.power, refuses: options.refuses, wake: options.wake }; - Object.assign( - target, - Object.fromEntries(Object.entries(scripted).filter(([, value]) => value !== undefined)), - ); - const other = new FakeTarget(phone ? SERIAL : PHONE_SERIAL, clock, [], new AdbServer()); - const capture = createAndroidScreenCapture({ - clock, - filesystem, - ...(options.scid === null ? {} : { scid: options.scid ?? (() => SCID) }), - serverPath: SERVER, - targetFor: async (asked) => (asked === device ? target : other), - }); - return { capture, clock, other, target }; -} - -async function drive(clock: FakeClock, promise: Promise, stepMs = 50): Promise { - let settled = false; - promise.then( - () => (settled = true), - () => (settled = true), - ); - for (let spent = 0; spent <= 120_000; spent += stepMs) { - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - if (settled) return await promise; - clock.advance(stepMs); - } - throw new Error("did not settle within two fake minutes"); -} - -function streamStart(): Uint8Array { - return concatBytes(codecId(), sessionPacket(96, 64)); -} - -async function hasEnded(handle: { wait(): Promise }): Promise { - let ended = false; - void handle.wait().then(() => (ended = true)); - for (let flush = 0; flush < 5; flush += 1) await new Promise((resolve) => setImmediate(resolve)); - return ended; -} - -function frameTag(frame: Uint8Array | undefined): number | undefined { - return frame?.[4]; -} - -async function waitThenStart(target: FakeTarget, opening: Promise): Promise { - for (let tries = 0; tries < 100 && target.server.peers.length === 0; tries += 1) { - await new Promise((resolve) => setImmediate(resolve)); - } - target.server.last.send(streamStart()); - return await opening; -} - -async function opened(setupResult: Awaited>) { - const { capture, clock, target } = setupResult; - const opening = capture.open(DEVICE, SETTINGS, new AbortController().signal); - return await drive(clock, waitThenStart(target, opening)); -} +import { CLOSE_DEADLINE_MS, CONNECT_RETRY_MS, START_DEADLINE_MS } from "./session.js"; +import { + SERVER, + DEVICE_JAR, + SETTINGS, + SCREEN_READ_MS, + DEVICE, + PHONE, + AdbServer, + push, + shell, + remove, + setup, + drive, + streamStart, + frameTag, + waitThenStart, + opened, +} from "./test-session.js"; +import { codecId } from "./test-scrcpy.js"; describe("Android screen capture: open", () => { it("pushes the server to a per-session path and starts it with that session's id and the settings", async () => { @@ -669,224 +308,6 @@ describe("Android screen capture: open", () => { }); }); -describe("Android screen capture: a phone's screen state", () => { - it("wakes a phone whose screen is off through its target with a 10 s limit, reads again, and pushes once the second read says it is on, for each wakefulness that means off", async () => { - for (const printed of ["Asleep", "Dozing", "Asleep\r", "Dozing "]) { - const world = await setup({ - device: PHONE, - power: [ - { code: 0, stdout: ` mWakefulness=${printed}\n` }, - { code: 0, stdout: " mWakefulness=Awake\n" }, - ], - }); - const opening = world.capture.open(PHONE, SETTINGS, new AbortController().signal); - const feed = await drive(world.clock, waitThenStart(world.target, opening)); - expect(frameTag((await feed.body.getReader().read()).value), printed).toBe( - SCREEN_FRAME_TAG.meta, - ); - - expect(world.target.screenSteps, printed).toEqual(["read", "wake", "read"]); - expect(world.target.wakes, printed).toEqual([WAKE_MS]); - expect(world.target.powerReads, printed).toEqual([SCREEN_READ_MS, SCREEN_READ_MS]); - expect(world.target.log[0], printed).toBe(`push ${SERVER} ${DEVICE_JAR}`); - } - }); - - it("refuses a phone whose second read still says the screen is off, with its own reason, and pushes nothing, whatever the wake did", async () => { - for (const second of ["Asleep", "Dozing"]) { - for (const wake of ["ok", "fail", "slow"] as const) { - const world = await setup({ - device: PHONE, - wake, - power: [ - { code: 0, stdout: " mWakefulness=Asleep\n" }, - { code: 0, stdout: ` mWakefulness=${second}\n` }, - ], - }); - const error = await drive( - world.clock, - world.capture.open(PHONE, SETTINGS, new AbortController().signal), - ).catch((caught: unknown) => caught); - - const name = `${second} after a wake that is ${wake}`; - expect(error, name).toBeInstanceOf(ScreenUnavailableError); - expect((error as Error).message, name).toBe(SCREEN_OFF); - // why: a wake that exited non-zero or timed out is not looked at; the second read decides. - expect(world.target.screenSteps, name).toEqual(["read", "wake", "read"]); - expect(world.target.steps, name).toEqual([]); - expect(world.target.connects, name).toEqual([]); - } - } - }); - - it.each([ - { name: "Awake", power: { code: 0, stdout: " mWakefulness=Awake\n" } }, - { name: "Dreaming", power: { code: 0, stdout: " mWakefulness=Dreaming\n" } }, - { name: "Awaken", power: { code: 0, stdout: " mWakefulness=Awaken\n" } }, - { name: "asleep", power: { code: 0, stdout: " mWakefulness=asleep\n" } }, - { name: "no matching line", power: { code: 0, stdout: " mPolicyBoost=0\n" } }, - { name: "a non-zero exit", power: { code: 1, stdout: " mWakefulness=Asleep\n" } }, - { name: "a slow read", power: "slow" }, - ] as const)( - "goes on to the push, without a wake, after one 10 s screen read that says $name", - async ({ name, power }) => { - const world = await setup({ device: PHONE, power }); - const opening = world.capture.open(PHONE, SETTINGS, new AbortController().signal); - // why: a timed-out read ends 10 s of fake clock after the open, and waitThenStart's capped - // flush-wait cannot span it; the file's uncapped watcher pattern waits for the peer whenever - // it appears, and drive still ends the run if the open never reaches it. - const watcher = (async () => { - while (world.target.server.peers.length === 0) - await new Promise((resolve) => setImmediate(resolve)); - world.target.server.last.send(streamStart()); - })(); - const feed = await drive(world.clock, opening); - await watcher; - expect(frameTag((await feed.body.getReader().read()).value), name).toBe( - SCREEN_FRAME_TAG.meta, - ); - - expect(world.target.powerReads, name).toEqual([SCREEN_READ_MS]); - expect(world.target.wakes, name).toEqual([]); - expect(world.target.log[0], name).toBe(`push ${SERVER} ${DEVICE_JAR}`); - }, - ); - - it("checks for a stop after each of the first read, the wake and the second read, and does nothing after the step it was stopped in", async () => { - const asleep = { code: 0, stdout: " mWakefulness=Asleep\n" }; - const awake = { code: 0, stdout: " mWakefulness=Awake\n" }; - const cases = [ - { name: "the first read", power: [asleep], wake: "ok", wakes: 0, reads: 1, hold: true }, - { name: "the wake", power: [asleep, awake], wake: "slow", wakes: 1, reads: 1, hold: false }, - { - name: "the second read", - power: [asleep, "slow"], - wake: "ok", - wakes: 1, - reads: 2, - hold: false, - }, - ] as const; - for (const stage of cases) { - const stop = new AbortController(); - const world = await setup({ - device: PHONE, - power: stage.power as unknown as FakeTarget["power"], - wake: stage.wake, - }); - world.target.holdReads = stage.hold; - const opening = world.capture.open(PHONE, SETTINGS, stop.signal); - // why: stops at the step itself: the step is in flight once its entry is recorded. - while ( - stage.name === "the first read" - ? world.target.powerReads.length < 1 - : stage.name === "the wake" - ? world.target.wakes.length < 1 - : world.target.powerReads.length < 2 - ) - await new Promise((resolve) => setImmediate(resolve)); - stop.abort(); - const failed = await drive(world.clock, opening).catch((caught: unknown) => caught); - - expect(failed, stage.name).toBeInstanceOf(ScreenUnavailableError); - expect(world.target.wakes.length, stage.name).toBe(stage.wakes); - expect(world.target.powerReads.length, stage.name).toBe(stage.reads); - expect(world.target.log, stage.name).toEqual([]); - expect(world.target.connects, stage.name).toEqual([]); - } - }); - - it("fails the open with a target's own reason and runs nothing after it, for the first screen read, the wake, the second screen read, the push, the server start and a connection attempt", async () => { - const stages: readonly { - readonly name: string; - readonly power?: FakeTarget["power"]; - readonly wake?: FakeTarget["wake"]; - readonly refuses: (args: readonly string[]) => boolean; - readonly screenBefore: string[]; - readonly stepsBefore: string[]; - }[] = [ - { - name: "the first screen read", - power: "refuse", - refuses: () => false, - screenBefore: ["read"], - stepsBefore: [], - }, - { - // why: the wake's exit is ignored, but a closed check at the wake still refuses the open. - name: "the wake", - power: { code: 0, stdout: " mWakefulness=Asleep\n" }, - wake: "refuse", - refuses: () => false, - screenBefore: ["read", "wake"], - stepsBefore: [], - }, - { - name: "the second screen read", - power: [{ code: 0, stdout: " mWakefulness=Asleep\n" }, "refuse"], - refuses: () => false, - screenBefore: ["read", "wake", "read"], - stepsBefore: [], - }, - { - name: "the push", - refuses: (args) => args[0] === "push", - screenBefore: ["read"], - stepsBefore: [], - }, - { - name: "the server start", - refuses: (args) => args.includes("app_process"), - screenBefore: ["read"], - stepsBefore: [`push ${SERVER} ${DEVICE_JAR}`, `shell rm -f ${DEVICE_JAR}`], - }, - { - // why: the server's spawn sits between the push and the connection attempt, so it is in the - // log by the time the attempt's check refuses; only the `rm -f` follows it. - name: "a connection attempt", - refuses: (args) => args[0] === "connect", - screenBefore: ["read"], - stepsBefore: [`push ${SERVER} ${DEVICE_JAR}`, "server", `shell rm -f ${DEVICE_JAR}`], - }, - ]; - for (const stage of stages) { - const world = await setup({ - device: PHONE, - ...(stage.power === undefined ? {} : { power: stage.power }), - ...(stage.wake === undefined ? {} : { wake: stage.wake }), - refuses: stage.refuses, - }); - const started = world.clock.now(); - const error = await drive( - world.clock, - world.capture.open(PHONE, SETTINGS, new AbortController().signal), - ).catch((caught: unknown) => caught); - - expect(error, stage.name).toBeInstanceOf(ScreenUnavailableError); - expect((error as Error).message, stage.name).toBe(GATE_REFUSAL); - // why: a refusal that was retried instead of propagated would have spent the start deadline on - // it, and the viewer would get a start timeout instead of the check's reason. - expect(world.clock.now() - started, stage.name).toBeLessThan(START_DEADLINE_MS); - expect(world.target.steps, stage.name).toEqual(stage.stepsBefore); - expect(world.target.screenSteps, stage.name).toEqual(stage.screenBefore); - } - }); - - it("sends nothing and settles when the version check is still closed at the jar's removal", async () => { - const world = await setup({ - device: PHONE, - refuses: (args) => args.includes("app_process") || args[0] === "shell", - }); - const error = await drive( - world.clock, - world.capture.open(PHONE, SETTINGS, new AbortController().signal), - ).catch((caught: unknown) => caught); - - expect(error).toBeInstanceOf(ScreenUnavailableError); - expect(world.target.steps).toEqual([`push ${SERVER} ${DEVICE_JAR}`]); - }); -}); - describe("Android screen capture: the start bound", () => { it("does not spend the 15 s start bound on the target's check that runs before the server is spawned", async () => { // why: the check runs inside the target, before its process exists. The bound is measured from the @@ -929,160 +350,3 @@ describe("Android screen capture: the start bound", () => { expect(failedAt).toBeLessThanOrEqual(START_DEADLINE_MS + 5_000 + CLOSE_DEADLINE_MS); }); }); - -describe("Android screen capture: the body", () => { - it("reads the socket only when its reader asks for a frame", async () => { - const world = await setup(); - const feed = await opened(world); - const peer = world.target.server.last; - peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); - const before = peer.pulls; - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - expect(peer.pulls).toBe(before); - const reader = feed.body.getReader(); - expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.meta); - expect(peer.pulls).toBe(before); - expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.description); - expect(peer.pulls).toBeGreaterThan(before); - expect(frameTag((await reader.read()).value)).toBe(SCREEN_FRAME_TAG.keyframe); - }); - - it("delivers every delta, in order, however slowly it is read", async () => { - const world = await setup(); - const feed = await opened(world); - const peer = world.target.server.last; - peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); - // why: a delta frame carries only the picture, so each picture ends in its own index byte. - for (let index = 0; index < 50; index += 1) { - const picture = Uint8Array.from(SAMPLE.p1); - picture[picture.length - 1] = index + 1; - peer.send(mediaPacket({ pts: index }, picture)); - } - const reader = feed.body.getReader(); - const deltas: (number | undefined)[] = []; - for (let index = 0; index < 53; index += 1) { - for (let flush = 0; flush < 3; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - const frame = (await reader.read()).value; - if (frameTag(frame) === SCREEN_FRAME_TAG.delta) deltas.push(frame?.at(-1)); - } - expect(deltas).toEqual(Array.from({ length: 50 }, (_, index) => index + 1)); - }); - - it("says a still screen is alive every few seconds, and keeps the frames that follow in order", async () => { - const world = await setup(); - const feed = await opened(world); - const peer = world.target.server.last; - peer.send(mediaPacket({ config: true }, SAMPLE.config), mediaPacket({ key: true }, SAMPLE.idr)); - const reader = feed.body.getReader(); - for (let index = 0; index < 3; index += 1) await reader.read(); - const flushAll = async (): Promise => { - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - }; - for (const round of [1, 2]) { - const quiet = reader.read(); - let settled = false; - void quiet.then(() => (settled = true)); - await flushAll(); - world.clock.advance(ALIVE_AFTER_MS - 1); - await flushAll(); - expect(settled, `round ${String(round)} before ${String(ALIVE_AFTER_MS)} ms`).toBe(false); - world.clock.advance(1); - const alive = (await quiet).value; - expect(frameTag(alive), `round ${String(round)}`).toBe(SCREEN_FRAME_TAG.alive); - expect(alive?.length, `round ${String(round)}`).toBe(5); - } - const next = reader.read(); - const first = Uint8Array.from(SAMPLE.p1); - first[first.length - 1] = 1; - const second = Uint8Array.from(SAMPLE.p1); - second[second.length - 1] = 2; - peer.send(mediaPacket({ pts: 1 }, first), mediaPacket({ pts: 2 }, second)); - const deltas = [(await next).value, (await reader.read()).value]; - expect(deltas.map(frameTag)).toEqual([SCREEN_FRAME_TAG.delta, SCREEN_FRAME_TAG.delta]); - expect(deltas.map((frame) => frame?.at(-1))).toEqual([1, 2]); - }); - - it("says nothing is alive while the encoder has not painted a first picture, on an emulator and on a phone", async () => { - for (const device of [DEVICE, PHONE]) { - const world = await setup({ device }); - const opening = world.capture.open(device, SETTINGS, new AbortController().signal); - const feed = await drive(world.clock, waitThenStart(world.target, opening)); - const peer = world.target.server.last; - peer.send(mediaPacket({ config: true }, SAMPLE.config)); - const reader = feed.body.getReader(); - expect(frameTag((await reader.read()).value), device.deviceId).toBe(SCREEN_FRAME_TAG.meta); - const quiet = reader.read(); - let settled = false; - void quiet.then(() => (settled = true)); - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - world.clock.advance(ALIVE_AFTER_MS * 3); - for (let flush = 0; flush < 10; flush += 1) - await new Promise((resolve) => setImmediate(resolve)); - expect(settled, device.deviceId).toBe(false); - } - }); - - it("errors with the server's reason when the device side ends, and cleans up", async () => { - const world = await setup({ - expectations: [push(), shell({ stderrLines: ["[server] ERROR: Encoding error"] }), remove()], - }); - const feed = await opened(world); - const reader = feed.body.getReader(); - await reader.read(); - world.target.server.last.close(); - const error = await drive(world.clock, reader.read()).catch((caught: unknown) => caught); - expect(error).toBeInstanceOf(ScreenUnavailableError); - expect((error as Error).message).toMatch(/Encoding error/); - await drive(world.clock, feed.close()); - expect(world.target.steps.at(-1)).toBe(`shell rm -f ${DEVICE_JAR}`); - }); -}); - -describe("Android screen capture: close", () => { - it("closes the socket, kills a server that outlives it, and removes the jar, once however often it is called", async () => { - const world = await setup(); - const feed = await opened(world); - const closes = Promise.all([feed.close(), feed.close()]); - await drive(world.clock, closes); - await drive(world.clock, feed.close()); - expect(world.target.server.last.closed).toBe(true); - expect(await hasEnded(world.target.runner.handles[1]!)).toBe(true); - expect(world.target.log.filter((entry) => entry.includes("rm"))).toHaveLength(1); - }); - - it("kills a server still running a second after its socket closed, with SIGKILL", async () => { - const world = await setup({ - expectations: [push(), shell({ ignoresSigterm: true }), remove()], - }); - const feed = await opened(world); - const closing = feed.close(); - const server = world.target.runner.handles[1]!; - await hasEnded(server); - world.clock.advance(999); - expect(await hasEnded(server)).toBe(false); - world.clock.advance(1); - expect(await hasEnded(server)).toBe(true); - await drive(world.clock, closing); - }); - - it("resolves within its deadline even when removing the jar never returns", async () => { - const world = await setup({ expectations: [push(), shell(), { ...remove(), hangs: true }] }); - const feed = await opened(world); - const started = world.clock.now(); - await drive(world.clock, feed.close()); - expect(world.clock.now() - started).toBeGreaterThanOrEqual(CLOSE_DEADLINE_MS); - expect(world.clock.now() - started).toBeLessThanOrEqual(CLOSE_DEADLINE_MS + 50); - }); - - it("cancelling the body closes the capture", async () => { - const world = await setup(); - const feed = await opened(world); - await drive(world.clock, feed.body.cancel()); - expect(world.target.server.last.closed).toBe(true); - expect(world.target.log.filter((entry) => entry.includes("rm"))).toHaveLength(1); - }); -}); diff --git a/src/drivers/android/screen/test-session.ts b/src/drivers/android/screen/test-session.ts new file mode 100644 index 000000000..29e9a2398 --- /dev/null +++ b/src/drivers/android/screen/test-session.ts @@ -0,0 +1,378 @@ +import { + type PhysicalDriverDevice, + ScreenUnavailableError, + type VirtualDriverDevice, +} from "../../../core/index.js"; +import { + FakeClock, + MemoryFilesystem, + type ProcessResult, + ScriptedProcessRunner, + type ScriptedProcessExpectation, + type TcpStream, +} from "../../../ports/index.js"; + +import { createAndroidScreenCapture, type ScreenTarget } from "./session.js"; +import { codecId, concatBytes, sessionPacket } from "./test-scrcpy.js"; + +export const ADB = "/sdk/platform-tools/adb"; +export const SERVER = "/simlock/scrcpy-server-v5.0.1"; +export const SERIAL = "emulator-5586"; +export const PHONE_SERIAL = "R58M123ABC"; +export const SCID = 0x1234abcd; +export const DEVICE_JAR = "/data/local/tmp/simlock-scrcpy-1234abcd.jar"; +export const SETTINGS = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; +export const SCREEN_READ_MS = 10_000; +export const WAKE_MS = 10_000; +export const DEVICE: VirtualDriverDevice = { + address: SERIAL, + deviceId: "simlock_1", + driverData: {}, + physical: false, +}; +export const PHONE: PhysicalDriverDevice = { + address: PHONE_SERIAL, + deviceId: PHONE_SERIAL, + driverData: {}, + enrolledApps: [], + physical: true, +}; + +// why: the reason a phone's target gives when the shared adb server's version check is closed. The +// capture is expected to pass it through as its own, unchanged, and to run nothing after it. +export const GATE_REFUSAL = + "the adb server on port 5037 speaks protocol 40 and Simlock's adb speaks 41. " + + "No command goes to physical Android devices until they match, and Simlock will not restart that server."; +export const SCREEN_OFF = + "The phone's screen is off and did not turn on. Turn it on and ask for the stream again."; +export const WAKE_ARGS = "shell input keyevent KEYCODE_WAKEUP"; + +export type Behaviour = "accept" | "refuse" | "no-transport" | "connection-refused" | "silent"; + +export class Peer { + readonly written: string[] = []; + closed = false; + pulls = 0; + readonly #queue: Uint8Array[] = []; + #wake: (() => void) | undefined; + readonly stream: TcpStream; + + constructor() { + const readable = new ReadableStream( + { + cancel: () => { + this.close(); + }, + pull: async (controller) => { + this.pulls += 1; + while (this.#queue.length === 0 && !this.closed) { + await new Promise((resolve) => { + this.#wake = resolve; + }); + } + const next = this.#queue.shift(); + if (next === undefined) controller.close(); + else controller.enqueue(next); + }, + }, + { highWaterMark: 0 }, + ); + this.stream = { + close: () => { + this.close(); + }, + end: () => undefined, + readable, + write: async (bytes) => { + this.written.push(typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes)); + }, + }; + } + + send(...chunks: (Uint8Array | string)[]): void { + for (const chunk of chunks) { + this.#queue.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk); + } + this.#wake?.(); + } + + close(): void { + this.closed = true; + this.#wake?.(); + } +} + +// why: the adb server on the far side of a target's `connect`; what it answers is scripted per attempt. +export class AdbServer { + readonly peers: Peer[] = []; + readonly #behaviours: Behaviour[]; + + constructor( + behaviours: Behaviour[] = [], + readonly otherwise: Behaviour = "accept", + // why: a peer that stays silent and open until this moment of the fake clock, then refuses. + private readonly acceptFromMs: number | undefined = undefined, + ) { + this.#behaviours = behaviours; + } + + newPeer(clock: FakeClock): Peer | undefined { + if (this.acceptFromMs !== undefined && clock.now() < this.acceptFromMs) { + const waiting = new Peer(); + waiting.send("OKAY", "FAIL0006closed"); + return waiting; + } + const behaviour = this.#behaviours.shift() ?? this.otherwise; + if (behaviour === "connection-refused") throw new Error("connect ECONNREFUSED 127.0.0.1"); + const peer = new Peer(); + this.peers.push(peer); + if (behaviour === "no-transport") peer.send(`FAIL0020device '${SERIAL}' not found`); + if (behaviour === "refuse") peer.send("OKAY", "FAIL0006closed"); + if (behaviour === "accept") peer.send("OKAY", "OKAY", new Uint8Array([0])); + return peer; + } + + get last(): Peer { + const peer = this.peers.at(-1); + if (peer === undefined) throw new Error("no connection"); + return peer; + } +} + +export interface PowerAnswer { + readonly code: number | null; + readonly stdout: string; +} + +// why: the device's capture target as the driver builds it. `args` start at the adb subcommand, so +// the serial and the adb server are the target's and not the capture's: this fake records what it +// was asked to run, and the commands themselves are the scripted runner's, so a push can fail, a +// server can hang and a kill can be escalated exactly as before the target existed. +export class FakeTarget implements ScreenTarget { + readonly log: string[] = []; + readonly powerReads: number[] = []; + readonly wakes: number[] = []; + // why: the order of the screen steps, so "reads again after the wake" can fail. + readonly screenSteps: string[] = []; + readonly connects: number[] = []; + readonly runner: ScriptedProcessRunner; + readonly server: AdbServer; + // why: `"refuse"` is the version check refusing the read, `"slow"` a read that ends later. + // why: a list answers the reads in turn, the last one for every read after it. + power: PowerAnswer | "refuse" | "slow" | readonly (PowerAnswer | "refuse" | "slow")[] = { + code: 0, + stdout: " mWakefulness=Awake\n", + }; + // why: what the wake does: exits 0, exits non-zero, hangs to its limit, or is refused by the check. + wake: "ok" | "fail" | "slow" | "refuse" = "ok"; + // why: holds each screen read the check lets through in flight for the read limit before it answers as scripted (a refused read throws before the hold), so a stop can arrive while a read that will say Asleep is still running. + holdReads = false; + // why: stands in for a check that closes mid-session: a command whose args this picks out is refused as a closed check refuses it. + refuses: (args: readonly string[]) => boolean = () => false; + // why: stand-ins for the check a target runs before its own command or connection, so a bound can be measured against it. + spawnDelayMs = 0; + connectDelayMs = 0; + #clock: FakeClock; + + constructor( + readonly serial: string, + clock: FakeClock, + expectations: ScriptedProcessExpectation[], + server: AdbServer | undefined, + ) { + this.#clock = clock; + this.runner = new ScriptedProcessRunner(expectations); + this.server = server ?? new AdbServer(); + } + + get steps(): string[] { + return this.log.map((entry) => (entry.includes("app_process") ? "server" : entry)); + } + + async run(args: readonly string[], timeoutMs: number): Promise { + if (isScreenRead(args)) return await this.#readScreen(timeoutMs); + if (args.join(" ") === WAKE_ARGS) { + this.wakes.push(timeoutMs); + this.screenSteps.push("wake"); + if (this.wake === "refuse") throw new ScreenUnavailableError(GATE_REFUSAL); + if (this.wake === "slow") { + await this.#sleep(WAKE_MS); + return { code: null, stderr: "", stdout: "" }; + } + return { code: this.wake === "fail" ? 1 : 0, stderr: "", stdout: "" }; + } + if (this.refuses(args)) throw new ScreenUnavailableError(GATE_REFUSAL); + this.log.push(args.join(" ")); + return await this.runner.run(ADB, args); + } + + async spawn(args: readonly string[]): Promise { + if (this.spawnDelayMs > 0) await this.#sleep(this.spawnDelayMs); + if (this.refuses(args)) throw new ScreenUnavailableError(GATE_REFUSAL); + this.log.push(args.join(" ")); + return await this.runner.spawn(ADB, args); + } + + async connect(timeoutMs: number): Promise { + this.connects.push(timeoutMs); + if (this.refuses(["connect"])) throw new ScreenUnavailableError(GATE_REFUSAL); + if (this.connectDelayMs > 0) await this.#sleep(this.connectDelayMs); + const peer = this.server.newPeer(this.#clock); + if (peer === undefined) throw new Error("connect ECONNREFUSED 127.0.0.1"); + return peer.stream; + } + + async #readScreen(timeoutMs: number): Promise { + const answer = Array.isArray(this.power) + ? (this.power[Math.min(this.powerReads.length, this.power.length - 1)] as PowerAnswer) + : (this.power as PowerAnswer | "refuse" | "slow"); + this.powerReads.push(timeoutMs); + this.screenSteps.push("read"); + if (answer === "refuse") throw new ScreenUnavailableError(GATE_REFUSAL); + if (this.holdReads || answer === "slow") await this.#sleep(SCREEN_READ_MS); + if (answer === "slow") return { code: null, stderr: "", stdout: "" }; + return { code: answer.code, stderr: "", stdout: answer.stdout }; + } + + #sleep(ms: number): Promise { + return new Promise((resolve) => { + this.#clock.setTimer(ms, resolve); + }); + } +} + +export function isScreenRead(args: readonly string[]): boolean { + return args.join(" ") === "shell dumpsys power"; +} + +export function push( + result: Partial<{ code: number; stderr: string }> = {}, +): ScriptedProcessExpectation { + return { + match: { + args: ["push", SERVER, /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/], + command: ADB, + }, + result: { code: result.code ?? 0, stderr: result.stderr ?? "", stdout: "" }, + }; +} + +// why: a phone's server command carries one more option (stay_awake) than an emulator's 20 arguments. +export function shell( + extra: Partial = {}, + options = 20, +): ScriptedProcessExpectation { + return { + hangs: true, + match: { + args: ["shell", ...Array.from({ length: options }, () => /.*/)], + command: ADB, + }, + ...extra, + }; +} + +export function remove(): ScriptedProcessExpectation { + return { + match: { + args: ["shell", "rm", "-f", /^\/data\/local\/tmp\/simlock-scrcpy-[0-9a-f]{8}\.jar$/], + command: ADB, + }, + }; +} + +// why: both devices' worlds — the other device's target is here so "it went through this device's +// target" can fail; a capture that aimed at the wrong device would show up in the other's lists. +export async function setup( + options: { + readonly acceptFromMs?: number; + readonly device?: VirtualDriverDevice | PhysicalDriverDevice; + readonly expectations?: ScriptedProcessExpectation[]; + readonly jar?: boolean; + readonly power?: FakeTarget["power"]; + readonly wake?: FakeTarget["wake"]; + readonly refuses?: (args: readonly string[]) => boolean; + readonly scid?: (() => number) | null; + readonly server?: AdbServer; + } = {}, +) { + const clock = new FakeClock(1_000); + const filesystem = new MemoryFilesystem(); + if (options.jar !== false) { + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(SERVER, "jar"); + } + const device = options.device ?? DEVICE; + const phone = device.physical === true; + const server = + options.server ?? + new AdbServer( + [], + "accept", + options.acceptFromMs === undefined ? undefined : 1_000 + options.acceptFromMs, + ); + const target = new FakeTarget( + phone ? PHONE_SERIAL : SERIAL, + clock, + options.expectations ?? [push(), shell({}, phone ? 21 : 20), remove()], + server, + ); + const scripted = { power: options.power, refuses: options.refuses, wake: options.wake }; + Object.assign( + target, + Object.fromEntries(Object.entries(scripted).filter(([, value]) => value !== undefined)), + ); + const other = new FakeTarget(phone ? SERIAL : PHONE_SERIAL, clock, [], new AdbServer()); + const capture = createAndroidScreenCapture({ + clock, + filesystem, + ...(options.scid === null ? {} : { scid: options.scid ?? (() => SCID) }), + serverPath: SERVER, + targetFor: async (asked) => (asked === device ? target : other), + }); + return { capture, clock, other, target }; +} + +export async function drive(clock: FakeClock, promise: Promise, stepMs = 50): Promise { + let settled = false; + promise.then( + () => (settled = true), + () => (settled = true), + ); + for (let spent = 0; spent <= 120_000; spent += stepMs) { + for (let flush = 0; flush < 10; flush += 1) + await new Promise((resolve) => setImmediate(resolve)); + if (settled) return await promise; + clock.advance(stepMs); + } + throw new Error("did not settle within two fake minutes"); +} + +export function streamStart(): Uint8Array { + return concatBytes(codecId(), sessionPacket(96, 64)); +} + +export async function hasEnded(handle: { wait(): Promise }): Promise { + let ended = false; + void handle.wait().then(() => (ended = true)); + for (let flush = 0; flush < 5; flush += 1) await new Promise((resolve) => setImmediate(resolve)); + return ended; +} + +export function frameTag(frame: Uint8Array | undefined): number | undefined { + return frame?.[4]; +} + +export async function waitThenStart(target: FakeTarget, opening: Promise): Promise { + for (let tries = 0; tries < 100 && target.server.peers.length === 0; tries += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + target.server.last.send(streamStart()); + return await opening; +} + +export async function opened(setupResult: Awaited>) { + const { capture, clock, target } = setupResult; + const opening = capture.open(DEVICE, SETTINGS, new AbortController().signal); + return await drive(clock, waitThenStart(target, opening)); +} From 0ed29903187fa2c9e02837aecb83ac9666e4e0be Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:29:22 +0100 Subject: [PATCH 65/83] test(history): split usage.test.ts by topic --- src/history/projections/usage/test-usage.ts | 222 ++++ .../usage/usage-device-facts-reads.test.ts | 275 ++++ .../projections/usage/usage-minutes.test.ts | 418 ++++++ .../usage-versions-rebuild-prune.test.ts | 322 +++++ src/history/projections/usage/usage.test.ts | 1161 +---------------- 5 files changed, 1254 insertions(+), 1144 deletions(-) create mode 100644 src/history/projections/usage/test-usage.ts create mode 100644 src/history/projections/usage/usage-device-facts-reads.test.ts create mode 100644 src/history/projections/usage/usage-minutes.test.ts create mode 100644 src/history/projections/usage/usage-versions-rebuild-prune.test.ts diff --git a/src/history/projections/usage/test-usage.ts b/src/history/projections/usage/test-usage.ts new file mode 100644 index 000000000..87440d52a --- /dev/null +++ b/src/history/projections/usage/test-usage.ts @@ -0,0 +1,222 @@ +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { EventBus, type EventMap, type EventName } from "../../../bus/index.js"; +import { + FakeClock, + JsonLinesLogger, + MemoryLogSink, + NodeFilesystem, + NodeSqliteOpener, +} from "../../../ports/index.js"; +import { + createEventHistory, + HISTORY_DB_NAME, + type HistoryProjectionDefinition, + type EventHistoryHandle, +} from "../../index.js"; +import { + type MinuteBucket, + openUsageReads, + type RequestRow, + USAGE_PROJECTION, + type UsageReads, + type UsageValue, +} from "./index.js"; + +export const MIN = 60_000; +export const HOUR = 3_600_000; +export const DAY = 24 * HOUR; +export const T0 = 1_760_605_920_000; +export const ALL = Number.MAX_SAFE_INTEGER; +export const directories: string[] = []; + +export interface RigOptions { + readonly directory?: string; + readonly at?: number; + readonly fleet?: boolean; + readonly eventRetention?: number; + readonly metricsRetention?: number; + readonly projections?: readonly HistoryProjectionDefinition[]; + readonly start?: boolean; +} + +export interface Rig { + readonly directory: string; + readonly path: string; + readonly clock: FakeClock; + readonly bus: EventBus; + readonly logs: MemoryLogSink; + readonly history: EventHistoryHandle; + at(timestamp: number): void; + emit(event: Event, payload: EventMap[Event]): void; + relay(workerId: string, event: string, payload: Record, at?: number): void; +} + +export async function rig(options: RigOptions = {}): Promise { + let directory = options.directory; + if (directory === undefined) { + directory = await mkdtemp(join(tmpdir(), "simlock-usage-")); + directories.push(directory); + } + const clock = new FakeClock(options.at ?? T0); + let nextId = 0; + const bus = new EventBus(clock, 10_000, undefined, { + generate: () => `${options.at ?? T0}_${(nextId += 1)}`, + }); + const logs = new MemoryLogSink(); + const history = createEventHistory({ + bus, + clock, + config: { maxBytes: 256 * 1024 * 1024, retention: options.eventRetention ?? 7 * DAY }, + dataDirectory: directory, + filesystem: new NodeFilesystem(), + logger: new JsonLinesLogger({ clock, sink: logs }), + projectionOptions: { + fleet: options.fleet ?? false, + metricsRetentionMs: options.metricsRetention ?? 90 * DAY, + }, + projections: options.projections ?? [USAGE_PROJECTION], + sqlite: new NodeSqliteOpener(), + }); + if (options.start !== false) await history.start(); + let relayed = 0; + return { + at: (timestamp) => clock.advance(timestamp - clock.now()), + bus, + clock, + directory, + emit: (event, payload) => { + bus.emit(event, payload, "test"); + }, + history, + logs, + path: join(directory, HISTORY_DB_NAME), + relay: (workerId, event, payload, at) => { + relayed += 1; + bus.republish({ + event: event as EventName, + id: `evt_relay_${options.at ?? T0}_${relayed}`, + module: "gateway", + payload: { ...payload, workerId } as never, + timestamp: at ?? clock.now(), + }); + }, + }; +} + +export async function readWith( + rigged: { readonly path: string }, + read: (reads: UsageReads) => T, +) { + const database = await new NodeSqliteOpener().open(rigged.path, { readOnly: true }); + try { + return read(openUsageReads(database)); + } finally { + database.close(); + } +} + +export async function count(rigged: { readonly path: string }, sql: string): Promise { + const database = await new NodeSqliteOpener().open(rigged.path, { readOnly: true }); + try { + return Number(database.prepare(sql).get()?.n); + } finally { + database.close(); + } +} + +export async function execOn(path: string, ...statements: string[]): Promise { + const database = await new NodeSqliteOpener().open(path, { readOnly: false }); + try { + for (const statement of statements) database.exec(statement); + } finally { + database.close(); + } +} + +export const rowsOf = (rigged: Rig) => + readWith(rigged, (reads) => reads.requestRows({ from: 0, to: ALL })); +export const factsOf = (rigged: Rig) => + readWith(rigged, (reads) => reads.deviceFacts({ from: 0, to: ALL })); +export const buckets = ( + rigged: Rig, + query: { + readonly from: number; + readonly to: number; + readonly platform: string; + readonly workerId?: string; + readonly bucketMs?: number; + }, +) => readWith(rigged, (reads) => reads.minuteBuckets({ bucketMs: MIN, workerId: "", ...query })); + +export function row(rows: readonly RequestRow[], requestId: string): RequestRow { + const found = rows.find((candidate) => candidate.requestId === requestId); + if (found === undefined) throw new Error(`no request row for ${requestId}`); + return found; +} + +export const requested = (requestId: string, over: Partial = {}) => ({ + requestId, + requestSpec: { model: "iPhone 16", platform: "ios" }, + requester: "alice", + waitPolicy: "wait", + ...over, +}); +export const granted = ( + leaseId: string, + requestId: string, + over: Partial = {}, +): EventMap["lease.granted"] => ({ + deviceId: `dev-${leaseId}`, + leaseId, + platform: "ios", + requestId, + requester: "alice", + source: "warm", + ...over, +}); +export const released = ( + leaseId: string, + reason: "explicit" | "killed" | "device-lost" = "explicit", +) => ({ + deviceId: `dev-${leaseId}`, + leaseId, + ownerId: "alice", + reason, +}); +export const rejected = ( + requestId: string, + reason: EventMap["lease.rejected"]["reason"], + over: Partial = {}, +): EventMap["lease.rejected"] => ({ + reason, + requestId, + requestSpec: { platform: "ios" }, + requester: "alice", + ...over, +}); +export const figures = (running: number, reserved: number, maxRunning: number) => ({ + maxRunning, + reserved, + running, + warm: 0, +}); +export const capacity = (input: { + readonly ios?: readonly [number, number, number]; + readonly android?: readonly [number, number, number]; + readonly global?: readonly [number, number, number]; + readonly ram?: readonly [number, number]; +}): EventMap["capacity.changed"] => ({ + android: figures(...(input.android ?? [0, 0, 4])), + global: figures(...(input.global ?? [0, 0, 8])), + ios: figures(...(input.ios ?? [0, 0, 4])), + ...(input.ram === undefined + ? {} + : { ramBudget: { limitBytes: input.ram[1], usedBytes: input.ram[0] } }), +}); + +export function figure(bucket: MinuteBucket | undefined, name: UsageValue) { + return bucket?.[name]; +} diff --git a/src/history/projections/usage/usage-device-facts-reads.test.ts b/src/history/projections/usage/usage-device-facts-reads.test.ts new file mode 100644 index 000000000..f740eee00 --- /dev/null +++ b/src/history/projections/usage/usage-device-facts-reads.test.ts @@ -0,0 +1,275 @@ +import { rm } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NodeSqliteOpener } from "../../../ports/index.js"; +import { openUsageReads } from "./index.js"; +import { + MIN, + HOUR, + T0, + ALL, + directories, + type Rig, + rig, + readWith, + factsOf, + buckets, + requested, + capacity, + figure, +} from "./test-usage.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("usage projection: device facts", () => { + it("one device fact row per listed device event and per lease.declined, with durationMs, platform from the event (absent when the event has none) and workerId on a relayed one; a device.ready with bootDuration 0 writes none", async () => { + const r = await rig({ fleet: true }); + r.emit("device.provisioned", { + deviceId: "d1", + driver: "fake", + duration: 1_200, + spec: { platform: "android" }, + }); + r.at(T0 + 1_000); + r.emit("device.ready", { bootDuration: 3_400, deviceId: "d1", platform: "ios" }); + r.at(T0 + 2_000); + r.emit("device.ready", { bootDuration: 0, deviceId: "d1", platform: "ios" }); + r.at(T0 + 3_000); + r.emit("device.quarantined", { + deviceId: "d2", + kind: "stalled", + maxRetries: 3, + nextRetryAt: T0 + HOUR, + platform: "ios", + }); + r.at(T0 + 4_000); + r.emit("device.recovered", { + attempts: 1, + deviceId: "d3", + duration: 900, + leaseId: "L", + platform: "android", + }); + r.at(T0 + 5_000); + r.emit("device.quarantine-recovered", { + attempts: 2, + deviceId: "d4", + platform: "ios", + strategy: "erase", + }); + r.at(T0 + 6_000); + r.emit("device.recovery-failed", { + attempts: 3, + deviceId: "d5", + error: "e", + leaseId: "L", + platform: "ios", + reason: "r", + }); + r.at(T0 + 7_000); + r.emit("device.quarantine-abandoned", { attempts: 3, deviceId: "d6", platform: "android" }); + r.at(T0 + 8_000); + r.emit("device.quarantine-stranded", { + attempts: 3, + deviceId: "d7", + error: "e", + platform: "ios", + }); + r.at(T0 + 9_000); + r.emit("device.ready", { bootDuration: 2_000, deviceId: "d8" } as never); + r.at(T0 + 10_000); + r.relay("w1", "device.ready", { bootDuration: 5_000, deviceId: "d9", platform: "android" }); + r.at(T0 + 11_000); + r.emit("lease.declined", { + fleetRequestId: "F", + reason: "no-wait", + requestId: "P", + requestSpec: { platform: "android" }, + requester: "alice", + }); + r.emit("lease.requested", requested("P", { fleetRequestId: "F" })); + + const facts = await factsOf(r); + + expect(facts).toMatchObject([ + { + at: T0, + deviceId: "d1", + durationMs: 1_200, + kind: "device.provisioned", + platform: "android", + }, + { at: T0 + 1_000, deviceId: "d1", durationMs: 3_400, kind: "device.ready", platform: "ios" }, + { at: T0 + 3_000, deviceId: "d2", kind: "device.quarantined", platform: "ios" }, + { + at: T0 + 4_000, + deviceId: "d3", + durationMs: 900, + kind: "device.recovered", + platform: "android", + }, + { at: T0 + 5_000, deviceId: "d4", kind: "device.quarantine-recovered", platform: "ios" }, + { at: T0 + 6_000, deviceId: "d5", kind: "device.recovery-failed", platform: "ios" }, + { at: T0 + 7_000, deviceId: "d6", kind: "device.quarantine-abandoned", platform: "android" }, + { at: T0 + 8_000, deviceId: "d7", kind: "device.quarantine-stranded", platform: "ios" }, + { at: T0 + 9_000, deviceId: "d8", durationMs: 2_000, kind: "device.ready" }, + { + at: T0 + 10_000, + deviceId: "d9", + durationMs: 5_000, + kind: "device.ready", + platform: "android", + workerId: "w1", + }, + { at: T0 + 11_000, kind: "lease.declined", platform: "android" }, + ]); + expect(facts[8]?.platform).toBeUndefined(); + expect(facts[0]?.workerId).toBeUndefined(); + await r.history.stop(); + }); +}); + +describe("usage reads", () => { + it("history.usage.oldestTs() is the earliest of the three tables' times, each table the earliest in one case, and undefined when all are empty", async () => { + const cases: ReadonlyArray<{ + readonly name: string; + readonly expected: number | undefined; + readonly act: (r: Rig) => void; + }> = [ + { act: () => undefined, expected: undefined, name: "empty" }, + { + act: (r) => { + r.at(T0 + 1_000); + r.emit("lease.requested", requested("R1", { fleetRequestId: "F1" })); + r.at(T0 + 5_000); + r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); + r.at(T0 + 2 * MIN); + r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); + }, + expected: T0 + 1_000, + name: "request rows", + }, + { + act: (r) => { + r.at(T0 + 90_000); + r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); + r.at(T0 + 100_000); + r.emit("lease.requested", requested("R1")); + r.at(T0 + 110_000); + r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); + }, + expected: T0 + MIN, + name: "minute rows", + }, + { + act: (r) => { + r.at(T0 + 10_000); + r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); + r.at(T0 + MIN + 20_000); + r.emit("lease.requested", requested("R1")); + r.at(T0 + 3 * MIN); + r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); + }, + expected: T0 + 10_000, + name: "device fact rows", + }, + ]; + for (const scenario of cases) { + const r = await rig(); + scenario.act(r); + expect(await readWith(r, (reads) => reads.oldestTs()), scenario.name).toBe(scenario.expected); + await r.history.stop(); + } + }); + + it("history.usage.requestRows and deviceFacts return rows with from <= time < to (one at from in, one at to out), in their stated order", async () => { + const r = await rig(); + r.at(T0 - 1); + r.emit("lease.requested", requested("before")); + r.emit("device.ready", { bootDuration: 1, deviceId: "d-before", platform: "ios" }); + r.at(T0); + r.emit("lease.requested", requested("b")); + r.emit("lease.requested", requested("a")); + r.emit("device.ready", { bootDuration: 1, deviceId: "d-1", platform: "ios" }); + r.at(T0 + 500); + r.emit("device.ready", { bootDuration: 1, deviceId: "d-2", platform: "ios" }); + r.at(T0 + 1_000); + r.emit("lease.requested", requested("at-to")); + r.emit("device.ready", { bootDuration: 1, deviceId: "d-to", platform: "ios" }); + + const requests = await readWith(r, (reads) => reads.requestRows({ from: T0, to: T0 + 1_000 })); + const facts = await readWith(r, (reads) => reads.deviceFacts({ from: T0, to: T0 + 1_000 })); + + expect(requests.map((candidate) => candidate.requestId)).toEqual(["a", "b"]); + expect(facts.map((fact) => fact.deviceId)).toEqual(["d-1", "d-2"]); + await r.history.stop(); + }); + + it("history.usage.minuteBuckets returns, per bucket, the largest minute peak, the summed integral and summed knownMs for the asked key only", async () => { + const r = await rig(); + r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [1, 0, 4] })); + r.at(T0 + MIN + 30_000); + r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [4, 0, 4] })); + r.at(T0 + 2 * MIN + 10_000); + r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [2, 0, 4] })); + + const wide = await buckets(r, { + bucketMs: 5 * MIN, + from: T0, + platform: "ios", + to: T0 + 10 * MIN, + }); + const later = await buckets(r, { from: T0 + 3 * MIN, platform: "ios", to: T0 + 5 * MIN }); + + expect(wide).toHaveLength(2); + expect(wide[0]).toMatchObject({ knownMs: 5 * MIN, start: T0 }); + expect(figure(wide[0], "slotsUsed")).toEqual({ + integral: 90_000 + 4 * 40_000 + 2 * 170_000, + peak: 4, + }); + expect(figure(wide[1], "slotsUsed")).toEqual({ integral: 2 * 5 * MIN, peak: 2 }); + expect(later.map((bucket) => bucket.start)).toEqual([T0 + 3 * MIN, T0 + 4 * MIN]); + expect(figure(later[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + expect(figure(later[0], "ramUsedBytes")).toBeUndefined(); + await r.history.stop(); + }); + + it("the four reads in one read transaction see the same commit while the writer commits between them", async () => { + const r = await rig(); + r.emit("lease.requested", requested("R1")); + const database = await new NodeSqliteOpener().open(r.path, { readOnly: true }); + const reads = openUsageReads(database); + try { + database.exec("BEGIN"); + const before = reads.requestRows({ from: 0, to: ALL }).length; + const minutes = { bucketMs: MIN, from: T0, platform: "ios", to: T0 + MIN, workerId: "" }; + const minutesBefore = reads.minuteBuckets(minutes); + r.at(T0 + 1_000); + r.emit("lease.requested", requested("R2")); + r.emit("device.ready", { bootDuration: 1, deviceId: "d", platform: "ios" }); + r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + const during = { + facts: reads.deviceFacts({ from: 0, to: ALL }).length, + oldest: reads.oldestTs(), + requests: reads.requestRows({ from: 0, to: ALL }).length, + }; + const minutesDuring = reads.minuteBuckets(minutes); + database.exec("COMMIT"); + + expect(before).toBe(1); + expect(during).toEqual({ facts: 0, oldest: T0, requests: 1 }); + expect(minutesDuring).toEqual(minutesBefore); + expect(figure(minutesBefore[0], "slotsUsed")).toBeUndefined(); + expect(reads.requestRows({ from: 0, to: ALL })).toHaveLength(2); + expect(figure(reads.minuteBuckets(minutes)[0], "slotsUsed")).toBeDefined(); + } finally { + database.close(); + } + await r.history.stop(); + }); +}); diff --git a/src/history/projections/usage/usage-minutes.test.ts b/src/history/projections/usage/usage-minutes.test.ts new file mode 100644 index 000000000..5b9d35738 --- /dev/null +++ b/src/history/projections/usage/usage-minutes.test.ts @@ -0,0 +1,418 @@ +import { rm } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { type EventMap } from "../../../bus/index.js"; +import { type UsageValue } from "./index.js"; +import { + MIN, + T0, + directories, + rig, + rowsOf, + factsOf, + buckets, + row, + requested, + granted, + released, + rejected, + capacity, + figure, +} from "./test-usage.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("usage projection: minute rows", () => { + it("a relayed platform that is not ios or android mints no platform key: a capacity key, a request spec and a device platform of another value change no minute row and leave the platform absent", async () => { + const r = await rig({ fleet: false }); + const others = ["tv", "all", "IOS"]; + others.forEach((platform, index) => { + r.relay( + "w1", + "capacity.changed", + { [platform]: { running: 1, reserved: 1, maxRunning: 4 } } as never, + T0 + index * 10, + ); + r.relay( + "w1", + "lease.requested", + requested(`R-${platform}`, { requestSpec: { platform } }), + T0 + index * 10 + 1, + ); + r.relay( + "w1", + "device.ready", + { deviceId: `d-${platform}`, bootDuration: 900, platform } as never, + T0 + index * 10 + 2, + ); + }); + + const rows = await rowsOf(r); + const facts = await factsOf(r); + + for (const platform of ["tv", "IOS", "ios", "android"]) { + expect(await buckets(r, { from: T0, platform, to: T0 + MIN })).toEqual([]); + } + const all = await buckets(r, { from: T0, platform: "all", to: T0 + MIN }); + expect(all).toHaveLength(1); + expect(figure(all[0], "waiting")?.peak).toBe(3); + for (const name of ["slotsUsed", "slotsMax"] as const) { + expect(figure(all[0], name)).toBeUndefined(); + } + for (const platform of others) expect(row(rows, `R-${platform}`).platform).toBe(""); + expect(facts.map((fact) => [fact.deviceId, fact.durationMs, fact.platform])).toEqual( + others.map((platform) => [`d-${platform}`, 900, undefined]), + ); + await r.history.stop(); + }); + + it("slotsUsed is running plus reserved and slotsMax per platform and on all; RAM used and limit and queue depth (both lines summed) only on all", async () => { + const r = await rig(); + r.emit( + "capacity.changed", + capacity({ android: [0, 0, 3], global: [1, 1, 6], ios: [1, 1, 4], ram: [100, 1_000] }), + ); + r.emit("queue.changed", { depth: 3 }); + r.emit("queue.changed", { depth: 2, physical: true }); + r.at(T0 + 3 * MIN); + r.emit("queue.changed", { depth: 1, physical: true }); + r.at(T0 + 5 * MIN); + r.emit( + "capacity.changed", + capacity({ android: [0, 0, 3], global: [1, 1, 6], ios: [1, 1, 4], ram: [100, 1_000] }), + ); + + const ios = await buckets(r, { from: T0, platform: "ios", to: T0 + MIN }); + const all = await buckets(r, { from: T0, platform: "all", to: T0 + MIN }); + + expect(ios).toHaveLength(1); + expect(figure(ios[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + expect(figure(ios[0], "slotsMax")).toEqual({ integral: 4 * MIN, peak: 4 }); + for (const name of ["ramUsedBytes", "ramLimitBytes", "queueDepth"] as const) { + expect(figure(ios[0], name)).toBeUndefined(); + } + expect(figure(all[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + expect(figure(all[0], "slotsMax")).toEqual({ integral: 6 * MIN, peak: 6 }); + expect(figure(all[0], "ramUsedBytes")).toEqual({ integral: 100 * MIN, peak: 100 }); + expect(figure(all[0], "ramLimitBytes")).toEqual({ integral: 1_000 * MIN, peak: 1_000 }); + expect(figure(all[0], "queueDepth")).toEqual({ integral: 5 * MIN, peak: 5 }); + await r.history.stop(); + }); + + it("peak, last and integral of a value that steps twice in a minute are the highest, the final and the time-weighted sum; an all row's peak is the highest total at one moment, not the sum of platform peaks", async () => { + const r = await rig(); + r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); + r.at(T0 + 15_000); + r.emit("capacity.changed", capacity({ ios: [3, 0, 4] })); + r.at(T0 + 30_000); + r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + r.at(T0 + 2 * MIN); + r.emit("lease.requested", requested("R1")); + r.emit("lease.requested", requested("R2", { requestSpec: { platform: "android" } })); + r.emit("lease.granted", granted("L1", "R1")); + r.at(T0 + 2 * MIN + 10_000); + r.emit("lease.released", released("L1")); + r.at(T0 + 2 * MIN + 20_000); + r.emit("lease.granted", granted("L2", "R2", { platform: "android" })); + + const first = await buckets(r, { from: T0, platform: "ios", to: T0 + 2 * MIN }); + const held = (platform: string) => + buckets(r, { from: T0 + 2 * MIN, platform, to: T0 + 3 * MIN }); + + expect(figure(first[0], "slotsUsed")).toEqual({ integral: 15_000 + 45_000 + 60_000, peak: 3 }); + expect(figure(first[1], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + expect(figure((await held("all"))[0], "leasesHeld")?.peak).toBe(1); + expect(figure((await held("ios"))[0], "leasesHeld")?.peak).toBe(1); + expect(figure((await held("android"))[0], "leasesHeld")?.peak).toBe(1); + await r.history.stop(); + }); + + it("leasesHeld by lease.granted's platform and waiting per platform rise and fall with grants, ends and rejections, across two minutes", async () => { + const r = await rig(); + r.emit("lease.requested", requested("R1")); + r.at(T0 + 30_000); + r.emit("lease.granted", granted("L1", "R1")); + r.at(T0 + 40_000); + r.emit("lease.requested", requested("R2")); + r.at(T0 + 50_000); + r.emit("lease.rejected", rejected("R2", "no-wait")); + r.at(T0 + MIN + 30_000); + r.emit("lease.released", released("L1")); + + const ios = await buckets(r, { from: T0, platform: "ios", to: T0 + 2 * MIN }); + const android = await buckets(r, { from: T0, platform: "android", to: T0 + 2 * MIN }); + + expect(ios).toHaveLength(2); + expect(figure(ios[0], "waiting")).toEqual({ integral: 40_000, peak: 1 }); + expect(figure(ios[0], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); + expect(figure(ios[1], "waiting")).toEqual({ integral: 0, peak: 0 }); + expect(figure(ios[1], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); + expect( + android.flatMap((bucket) => [figure(bucket, "waiting"), figure(bucket, "leasesHeld")]), + ).toEqual([]); + await r.history.stop(); + }); + + it("a step stamped before its key's last step counts at the last step's time, on a relayed row and on a * row", async () => { + const r = await rig({ fleet: true }); + r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 4] }), T0 + 10_000); + r.relay("w1", "capacity.changed", capacity({ global: [3, 0, 4], ios: [3, 0, 4] }), T0 + 5_000); + r.relay("w2", "capacity.changed", capacity({ global: [0, 0, 4], ios: [0, 0, 4] }), T0 + 20_000); + r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 4] }), T0 + 30_000); + r.relay("w2", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] }), T0 + 25_000); + + const own = await buckets(r, { from: T0, platform: "all", to: T0 + MIN, workerId: "w1" }); + const fleet = await buckets(r, { from: T0, platform: "all", to: T0 + MIN, workerId: "*" }); + + expect(figure(own[0], "slotsUsed")).toEqual({ integral: 3 * 20_000 + 1 * 30_000, peak: 3 }); + expect(figure(fleet[0], "slotsUsed")).toEqual({ + integral: 150_000, + peak: 3, + }); + await r.history.stop(); + }); + + it("on a gateway, the * row sums the workers' slots, RAM and leases and holds only the gateway's own queue depth and waiting; after worker.disconnected that worker leaves the sums", async () => { + const r = await rig({ fleet: true }); + r.relay( + "w1", + "capacity.changed", + capacity({ global: [1, 0, 4], ios: [1, 0, 2], ram: [100, 1_000] }), + ); + r.relay( + "w2", + "capacity.changed", + capacity({ global: [2, 0, 4], ios: [2, 0, 2], ram: [200, 1_000] }), + ); + r.relay("w1", "lease.granted", granted("WL1", "X1")); + r.relay("w2", "lease.granted", granted("WL2", "X2", { platform: "android" })); + r.at(T0 + 10_000); + r.relay("w1", "queue.changed", { depth: 9 }); + r.relay("w1", "lease.requested", requested("W-req")); + r.at(T0 + 20_000); + r.emit("queue.changed", { depth: 4 }); + r.emit("lease.requested", requested("G1")); + r.at(T0 + MIN); + r.emit("worker.disconnected", { leaseCount: 1, workerId: "w1" }); + + const first = (platform: string, workerId = "*") => + buckets(r, { from: T0, platform, to: T0 + MIN, workerId }); + const later = (platform: string, workerId = "*") => + buckets(r, { from: T0 + MIN, platform, to: T0 + 2 * MIN, workerId }); + const peakOf = async (query: ReturnType, value: UsageValue) => + figure((await query)[0], value)?.peak; + + expect(await peakOf(first("all"), "slotsUsed")).toBe(3); + expect(await peakOf(first("all"), "slotsMax")).toBe(8); + expect(await peakOf(first("all"), "ramUsedBytes")).toBe(300); + expect(await peakOf(first("all"), "ramLimitBytes")).toBe(2_000); + expect(await peakOf(first("all"), "leasesHeld")).toBe(2); + expect(await peakOf(first("ios"), "slotsUsed")).toBe(3); + expect(await peakOf(first("ios"), "leasesHeld")).toBe(1); + expect(await peakOf(first("all"), "queueDepth")).toBe(4); + expect(await peakOf(first("all"), "waiting")).toBe(1); + expect(await peakOf(first("all", "w1"), "queueDepth")).toBe(9); + expect(figure((await later("all"))[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + expect(figure((await later("all"))[0], "ramUsedBytes")).toEqual({ + integral: 200 * MIN, + peak: 200, + }); + expect(figure((await later("all"))[0], "leasesHeld")).toEqual({ integral: MIN, peak: 1 }); + expect(await later("all", "w1")).toEqual([]); + await r.history.stop(); + }); + + it("a worker that reconnects after worker.disconnected counts its held leases in the * row again", async () => { + const r = await rig({ fleet: true }); + r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 2] }), T0); + r.relay("w1", "lease.granted", granted("WL1", "X1"), T0 + 1_000); + r.at(T0 + MIN); + r.emit("worker.disconnected", { leaseCount: 1, workerId: "w1" }); + r.at(T0 + 2 * MIN); + r.relay( + "w1", + "capacity.changed", + capacity({ global: [1, 0, 4], ios: [1, 0, 2] }), + T0 + 2 * MIN, + ); + + const star = (minute: number) => + buckets(r, { + from: T0 + minute * MIN, + platform: "ios", + to: T0 + (minute + 1) * MIN, + workerId: "*", + }); + + expect(figure((await star(0))[0], "leasesHeld")).toMatchObject({ peak: 1 }); + expect(figure((await star(1))[0], "leasesHeld")).toEqual({ integral: 0, peak: 0 }); + expect(figure((await star(2))[0], "leasesHeld")).toEqual({ integral: MIN, peak: 1 }); + await r.history.stop(); + }); + + it("daemon.stopping ends the carried levels until the next daemon.started, and on a gateway worker.disconnected ends that worker's levels, so no buckets appear in the gap", async () => { + const r = await rig({ fleet: true }); + r.emit("capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); + r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); + r.at(T0 + 4 * MIN + 30_000); + r.emit("lease.requested", requested("R1")); + r.at(T0 + 5 * MIN); + r.emit("worker.disconnected", { leaseCount: 0, workerId: "w1" }); + r.at(T0 + 6 * MIN); + r.emit("daemon.stopping", { reason: "signal" }); + r.at(T0 + 9 * MIN); + r.emit("daemon.started", { configSnapshot: {}, version: "1" }); + r.emit("capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); + r.at(T0 + 11 * MIN + 30_000); + r.emit("daemon.stopping", { reason: "signal" }); + + const carried = await buckets(r, { from: T0, platform: "ios", to: T0 + 12 * MIN }); + const worker = await buckets(r, { + from: T0, + platform: "ios", + to: T0 + 12 * MIN, + workerId: "w1", + }); + + expect(carried.map((bucket) => (bucket.start - T0) / MIN)).toEqual([ + 0, 1, 2, 3, 4, 5, 9, 10, 11, + ]); + expect(carried.slice(0, 8).every((bucket) => bucket.knownMs === MIN)).toBe(true); + expect(carried[8]).toMatchObject({ knownMs: 30_000 }); + expect(figure(carried[8], "slotsUsed")).toEqual({ integral: 2 * 30_000, peak: 2 }); + expect(worker.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 1, 2, 3, 4]); + await r.history.stop(); + }); + + it("a value is carried across minutes with no row, from before from, and a first value mid-minute counts from its step", async () => { + const r = await rig(); + r.at(T0 + 4 * MIN + 30_000); + r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + r.at(T0 + 9 * MIN); + r.emit("lease.requested", requested("R1")); + + const whole = await buckets(r, { from: T0, platform: "ios", to: T0 + 9 * MIN }); + const later = await buckets(r, { from: T0 + 6 * MIN, platform: "ios", to: T0 + 8 * MIN }); + + expect(whole.map((bucket) => bucket.start - T0)).toEqual([4, 5, 6, 7, 8].map((m) => m * MIN)); + expect(whole[0]).toMatchObject({ knownMs: 30_000 }); + expect(figure(whole[0], "slotsUsed")).toEqual({ integral: 60_000, peak: 2 }); + expect(later).toHaveLength(2); + expect(figure(later[1], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); + await r.history.stop(); + }); + + it("nothing is carried over a crash's downtime", async () => { + const first = await rig(); + first.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + first.at(T0 + 30_000); + first.emit("lease.requested", requested("R1")); + await first.history.stop(); + + const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); + second.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + second.emit("daemon.started", { configSnapshot: {}, version: "1" }); + second.emit("lease.requested", requested("R2")); + second.at(T0 + 12 * MIN); + second.emit("lease.requested", requested("R3")); + + const read = await buckets(second, { from: T0, platform: "ios", to: T0 + 13 * MIN }); + + expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 10, 11, 12]); + await second.history.stop(); + }); + + it("a start catch-up reads one run in seq order, so a relayed event stamped before the run's own start does not begin a new run", async () => { + const r = await rig({ fleet: true, start: false }); + r.at(T0 + 2_000); + r.emit("daemon.started", { configSnapshot: {}, version: "1" }); + r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] }), T0 + 1_500); + await r.history.start(); + r.at(T0 + 3 * MIN); + r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); + + const read = await buckets(r, { from: T0, platform: "ios", to: T0 + 4 * MIN, workerId: "*" }); + + expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 1, 2, 3]); + await r.history.stop(); + }); + + it("a key the next run has not stepped by its daemon.started is closed there, not carried over the downtime", async () => { + const first = await rig(); + first.emit("capacity.changed", capacity({ android: [1, 0, 3], ios: [2, 0, 4] })); + await first.history.stop(); + + const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); + const { android: _unreported, ...iosOnly } = capacity({ ios: [2, 0, 4] }); + second.emit("capacity.changed", iosOnly as EventMap["capacity.changed"]); + second.emit("daemon.started", { configSnapshot: {}, version: "1" }); + second.at(T0 + 12 * MIN); + second.emit("capacity.changed", iosOnly as EventMap["capacity.changed"]); + + const read = await buckets(second, { from: T0, platform: "android", to: T0 + 13 * MIN }); + + expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0]); + await second.history.stop(); + }); + + it("after a restart, open requests and held leases are restored, leasesHeld and waiting get a step at daemon.started, and every row written before the stop is unchanged", async () => { + const first = await rig(); + first.emit("lease.requested", requested("R1")); + first.emit("lease.granted", granted("L1", "R1")); + first.emit("lease.requested", requested("R2")); + first.emit("lease.requested", requested("R3")); + first.at(T0 + 20_000); + first.emit("lease.rejected", rejected("R3", "no-wait")); + first.emit("daemon.stopping", { reason: "signal" }); + const rowsBefore = await rowsOf(first); + const minutesBefore = await buckets(first, { from: T0, platform: "ios", to: T0 + MIN }); + await first.history.stop(); + + const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); + second.emit("capacity.changed", capacity({ ios: [2, 0, 4], ram: [5, 9] })); + second.emit("queue.changed", { depth: 3 }); + second.emit("daemon.started", { configSnapshot: {}, version: "1" }); + second.at(T0 + 10 * MIN + 30_000); + second.emit("lease.released", released("L1")); + second.emit("lease.rejected", rejected("R2", "daemon-restarted")); + + const rows = await rowsOf(second); + const minutes = await buckets(second, { + from: T0 + 10 * MIN, + platform: "ios", + to: T0 + 11 * MIN, + }); + + expect(row(rows, "R1")).toMatchObject({ + endedAt: T0 + 10 * MIN + 30_000, + endReason: "explicit", + heldMs: 10 * MIN + 30_000, + }); + expect(row(rows, "R2")).toMatchObject({ reason: "daemon-restarted" }); + expect(row(rows, "R3")).toEqual(row(rowsBefore, "R3")); + expect(figure(minutes[0], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); + expect(figure(minutes[0], "waiting")).toEqual({ integral: 30_000, peak: 1 }); + expect(await buckets(second, { from: T0, platform: "ios", to: T0 + MIN })).toEqual( + minutesBefore, + ); + const after = await buckets(second, { + from: T0 + 10 * MIN, + platform: "all", + to: T0 + 12 * MIN, + }); + expect(after.map((bucket) => figure(bucket, "ramUsedBytes")?.peak)).toEqual([5, 5]); + expect(after.map((bucket) => figure(bucket, "queueDepth")?.peak)).toEqual([3, 3]); + const slots = await buckets(second, { + from: T0 + 10 * MIN, + platform: "ios", + to: T0 + 12 * MIN, + }); + expect(slots.map((bucket) => figure(bucket, "slotsUsed")?.peak)).toEqual([2, 2]); + await second.history.stop(); + }); +}); diff --git a/src/history/projections/usage/usage-versions-rebuild-prune.test.ts b/src/history/projections/usage/usage-versions-rebuild-prune.test.ts new file mode 100644 index 000000000..b5402eed1 --- /dev/null +++ b/src/history/projections/usage/usage-versions-rebuild-prune.test.ts @@ -0,0 +1,322 @@ +import { rm } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NodeSqliteOpener } from "../../../ports/index.js"; +import { type HistoryProjectionDefinition } from "../../index.js"; +import { + MIN, + HOUR, + DAY, + T0, + directories, + type Rig, + rig, + count, + execOn, + rowsOf, + factsOf, + buckets, + row, + requested, + granted, + capacity, +} from "./test-usage.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("usage projection: versions", () => { + const versioned = ( + version: number, + migrations: readonly string[], + ): HistoryProjectionDefinition => ({ + check: () => true, + create: (database) => ({ + apply: (envelope) => { + if (version === 1) { + database.prepare("INSERT INTO kept (id) VALUES (?)").run(envelope.id); + } + }, + prune: () => undefined, + rebuild: () => undefined, + }), + migrations, + name: "kept", + version, + }); + const CREATE = "CREATE TABLE kept (id TEXT)"; + const ADD = "ALTER TABLE kept ADD COLUMN extra INTEGER DEFAULT 7"; + + it("a stored version below the code's is migrated and keeps its rows", async () => { + const first = await rig({ projections: [versioned(1, [CREATE])] }); + first.emit("daemon.stopping", { reason: "a" }); + await first.history.stop(); + + const second = await rig({ + directory: first.directory, + projections: [versioned(2, [CREATE, ADD])], + }); + const database = await new NodeSqliteOpener().open(second.path, { readOnly: true }); + try { + const kept = database.prepare("SELECT id, extra FROM kept").all(); + expect(kept).toHaveLength(1); + expect(kept[0]?.extra).toBe(7); + } finally { + database.close(); + } + await second.history.stop(); + }); + + it("a migration that throws, or a stored version above the code's, makes start() reject naming the projection and versions, with every row kept", async () => { + const first = await rig({ projections: [versioned(1, [CREATE])] }); + first.emit("daemon.stopping", { reason: "a" }); + await first.history.stop(); + + const broken = await rig({ + directory: first.directory, + projections: [versioned(2, [CREATE, "ALTER TABLE missing ADD COLUMN x INTEGER"])], + start: false, + }); + await expect(broken.history.start()).rejects.toThrow(/kept.*1.*2/s); + await broken.history.stop(); + const accepted = await rig({ + directory: first.directory, + projections: [versioned(3, [CREATE, ADD, ADD.replace("extra", "more")])], + }); + accepted.emit("daemon.stopping", { reason: "b" }); + await accepted.history.stop(); + + const newer = await rig({ + directory: first.directory, + projections: [versioned(2, [CREATE, ADD])], + start: false, + }); + await expect(newer.history.start()).rejects.toThrow(/kept.*3.*2/s); + await newer.history.stop(); + + expect(await count(first, "SELECT count(*) AS n FROM kept")).toBe(1); + }); +}); + +describe("usage projection: rebuild", () => { + async function seed(r: Rig, requests: number): Promise { + for (let index = 0; index < requests; index += 1) { + r.at(T0 + index); + r.emit("lease.requested", requested(`R${index}`)); + if (index % 7 === 0) r.emit("lease.granted", granted(`L${index}`, `R${index}`)); + } + r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); + r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); + } + const breakMinuteTable = (path: string) => + execOn(path, "ALTER TABLE minute_rows DROP COLUMN knownMs"); + + it("a failed check rebuilds in batches with other work running between them: rows older than the oldest event are kept, the rest equal the live rows, an event emitted mid-rebuild is projected once, and status() gives rebuiltAt after; a start with nothing to fix gives neither field", async () => { + const first = await rig({ eventRetention: DAY }); + first.at(T0 - 2 * DAY); + first.emit("lease.requested", requested("OLD")); + first.at(T0); + await seed(first, 2_100); + const live = { + facts: await factsOf(first), + minutes: await buckets(first, { from: T0, platform: "ios", to: T0 + 5 * MIN }), + requests: (await rowsOf(first)).filter((candidate) => candidate.requestId !== "OLD"), + }; + await first.history.stop(); + await breakMinuteTable(first.path); + + const second = await rig({ + at: T0 + MIN, + directory: first.directory, + eventRetention: DAY, + start: false, + }); + let between = 0; + const ticker = setInterval(() => undefined, 1); + const tick = (): void => { + const status = second.history.status(); + if (status.opened && status.rebuiltAt === undefined) { + between += 1; + if (between === 1) second.emit("lease.requested", requested("MID")); + } + if (status.rebuiltAt === undefined) setImmediate(tick); + }; + setImmediate(tick); + await second.history.start(); + clearInterval(ticker); + + const requests = await rowsOf(second); + expect(second.history.status().rebuiltAt).toBe(T0 + MIN); + expect(between).toBeGreaterThanOrEqual(2); + expect(requests.filter((candidate) => candidate.requestId === "MID")).toHaveLength(1); + expect(requests.filter((candidate) => candidate.requestId === "OLD")).toHaveLength(1); + expect(requests.filter((candidate) => !["MID", "OLD"].includes(candidate.requestId))).toEqual( + live.requests, + ); + expect(await factsOf(second)).toEqual(live.facts); + const withMid = live.minutes.map((bucket) => + bucket.start < T0 + MIN + ? bucket + : { + // why: the first run was never stopped, so the second run (MID is its first event) carries none of its other levels over the downtime. + knownMs: bucket.knownMs, + start: bucket.start, + waiting: { + integral: (bucket.waiting?.integral ?? 0) + MIN, + peak: (bucket.waiting?.peak ?? 0) + 1, + }, + }, + ); + expect(await buckets(second, { from: T0, platform: "ios", to: T0 + 5 * MIN })).toEqual(withMid); + await second.history.stop(); + + const third = await rig({ at: T0 + 2 * MIN, directory: first.directory, eventRetention: DAY }); + expect(third.history.status()).not.toHaveProperty("rebuiltAt"); + expect(third.history.status()).not.toHaveProperty("rebuildFailed"); + await third.history.stop(); + }); + + it("a stop mid-rebuild leaves the mark and the next start rebuilds", async () => { + const first = await rig(); + await seed(first, 2_100); + const live = await rowsOf(first); + await first.history.stop(); + await breakMinuteTable(first.path); + + const second = await rig({ at: T0 + MIN, directory: first.directory, start: false }); + const starting = second.history.start(); + const stopping = new Promise((resolve) => { + const wait = (): void => { + const status = second.history.status(); + if (status.opened && status.rebuiltAt === undefined) + void second.history.stop().then(resolve); + else setImmediate(wait); + }; + setImmediate(wait); + }); + await Promise.all([starting, stopping]); + + const third = await rig({ at: T0 + 2 * MIN, directory: first.directory }); + expect(third.history.status().rebuiltAt).toBe(T0 + 2 * MIN); + expect(await rowsOf(third)).toEqual(live); + await third.history.stop(); + }); + + it("a rebuild that throws leaves no rows for the events' window, rebuildFailed true, writing true, no rebuiltAt, one error, and later events are still written", async () => { + const first = await rig(); + await seed(first, 2_100); + await first.history.stop(); + await breakMinuteTable(first.path); + await execOn( + first.path, + "CREATE TRIGGER refuse BEFORE INSERT ON request_rows WHEN NEW.requestId = 'R1500' BEGIN SELECT RAISE(ABORT, 'refused'); END", + ); + + const second = await rig({ at: T0 + MIN, directory: first.directory }); + second.emit("lease.requested", requested("LATER")); + const status = second.history.status(); + const stored = await second.history.replay({ sinceTs: 0 }); + + expect(status).toMatchObject({ rebuildFailed: true, writing: true }); + expect(status).not.toHaveProperty("rebuiltAt"); + expect(second.logs.records.filter((record) => record.level === "error")).toHaveLength(1); + expect(await rowsOf(second)).toEqual([]); + expect( + stored.some( + (envelope) => + envelope.payload && (envelope.payload as { requestId?: string }).requestId === "LATER", + ), + ).toBe(true); + await second.history.stop(); + }); +}); + +describe("usage projection: prune", () => { + it("prune deletes request, minute, device fact and held lease rows older than metrics.retention and keeps one at the cutoff, at start and one hour later", async () => { + const retention = 10 * DAY; + const cutoff = T0; + const first = await rig({ + at: cutoff - 5 * MIN, + eventRetention: DAY, + fleet: true, + metricsRetention: retention, + }); + const mark = (at: number, name: string): void => { + first.at(at); + first.emit("lease.requested", requested(`R-${name}`)); + first.emit("device.ready", { bootDuration: 5, deviceId: `d-${name}`, platform: "ios" }); + first.relay("w1", "lease.granted", granted(`L-${name}`, `X-${name}`)); + first.emit("daemon.stopping", { reason: name }); + first.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); + }; + mark(cutoff - 1_000, "old"); + mark(cutoff, "edge"); + mark(cutoff + HOUR, "new"); + await first.history.stop(); + + const second = await rig({ + at: cutoff + retention, + directory: first.directory, + eventRetention: DAY, + fleet: true, + metricsRetention: retention, + }); + const heldSource = async (name: string, workerLeaseId: string) => { + second.emit("lease.requested", requested(`Q-${name}`)); + second.emit("request.granted", { + leaseId: `G-${name}`, + requestId: `Q-${name}`, + worker: "w1", + workerLeaseId, + }); + return row(await rowsOf(second), `Q-${name}`).source; + }; + const minutesOf = async () => + ( + await buckets(second, { + from: cutoff - 2 * MIN, + platform: "ios", + to: cutoff + HOUR + MIN, + workerId: "*", + }) + ).map((bucket) => bucket.start); + const afterStart = { + facts: (await factsOf(second)).map((fact) => fact.deviceId), + minutes: await minutesOf(), + requests: (await rowsOf(second)).map((candidate) => candidate.requestId), + }; + const startSources = [ + await heldSource("start-old", "L-old"), + await heldSource("start-edge", "L-edge"), + await heldSource("start-new", "L-new"), + ]; + second.clock.advance(HOUR); + const afterHour = { + facts: (await factsOf(second)).map((fact) => fact.deviceId), + minutes: await minutesOf(), + requests: (await rowsOf(second)) + .map((candidate) => candidate.requestId) + .filter((requestId) => requestId.startsWith("R-")), + }; + const hourSources = [ + await heldSource("hour-edge", "L-edge"), + await heldSource("hour-new", "L-new"), + ]; + + expect(afterStart.requests).toEqual(["R-edge", "R-new"]); + expect(afterStart.facts).toEqual(["d-edge", "d-new"]); + expect(afterStart.minutes).toEqual([cutoff, cutoff + HOUR]); + expect(startSources).toEqual([undefined, "warm", "warm"]); + expect(afterHour).toEqual({ + facts: ["d-new"], + minutes: [cutoff + HOUR], + requests: ["R-new"], + }); + expect(hourSources).toEqual([undefined, "warm"]); + await second.history.stop(); + }); +}); diff --git a/src/history/projections/usage/usage.test.ts b/src/history/projections/usage/usage.test.ts index 34f03b6b6..b374a8004 100644 --- a/src/history/projections/usage/usage.test.ts +++ b/src/history/projections/usage/usage.test.ts @@ -1,37 +1,23 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { rm } from "node:fs/promises"; + import { afterEach, describe, expect, it } from "vitest"; -import { EventBus, type EventMap, type EventName } from "../../../bus/index.js"; -import { - FakeClock, - JsonLinesLogger, - MemoryLogSink, - NodeFilesystem, - NodeSqliteOpener, -} from "../../../ports/index.js"; -import { - createEventHistory, - HISTORY_DB_NAME, - type HistoryProjectionDefinition, - type EventHistoryHandle, -} from "../../index.js"; import { - type MinuteBucket, - openUsageReads, - type RequestRow, - USAGE_PROJECTION, - type UsageReads, - type UsageValue, -} from "./index.js"; - -const MIN = 60_000; -const HOUR = 3_600_000; -const DAY = 24 * HOUR; -const T0 = 1_760_605_920_000; -const ALL = Number.MAX_SAFE_INTEGER; -const directories: string[] = []; + MIN, + T0, + directories, + type Rig, + rig, + rowsOf, + factsOf, + buckets, + row, + requested, + granted, + released, + rejected, + figure, +} from "./test-usage.js"; afterEach(async () => { await Promise.all( @@ -39,189 +25,6 @@ afterEach(async () => { ); }); -interface RigOptions { - readonly directory?: string; - readonly at?: number; - readonly fleet?: boolean; - readonly eventRetention?: number; - readonly metricsRetention?: number; - readonly projections?: readonly HistoryProjectionDefinition[]; - readonly start?: boolean; -} - -interface Rig { - readonly directory: string; - readonly path: string; - readonly clock: FakeClock; - readonly bus: EventBus; - readonly logs: MemoryLogSink; - readonly history: EventHistoryHandle; - at(timestamp: number): void; - emit(event: Event, payload: EventMap[Event]): void; - relay(workerId: string, event: string, payload: Record, at?: number): void; -} - -async function rig(options: RigOptions = {}): Promise { - let directory = options.directory; - if (directory === undefined) { - directory = await mkdtemp(join(tmpdir(), "simlock-usage-")); - directories.push(directory); - } - const clock = new FakeClock(options.at ?? T0); - let nextId = 0; - const bus = new EventBus(clock, 10_000, undefined, { - generate: () => `${options.at ?? T0}_${(nextId += 1)}`, - }); - const logs = new MemoryLogSink(); - const history = createEventHistory({ - bus, - clock, - config: { maxBytes: 256 * 1024 * 1024, retention: options.eventRetention ?? 7 * DAY }, - dataDirectory: directory, - filesystem: new NodeFilesystem(), - logger: new JsonLinesLogger({ clock, sink: logs }), - projectionOptions: { - fleet: options.fleet ?? false, - metricsRetentionMs: options.metricsRetention ?? 90 * DAY, - }, - projections: options.projections ?? [USAGE_PROJECTION], - sqlite: new NodeSqliteOpener(), - }); - if (options.start !== false) await history.start(); - let relayed = 0; - return { - at: (timestamp) => clock.advance(timestamp - clock.now()), - bus, - clock, - directory, - emit: (event, payload) => { - bus.emit(event, payload, "test"); - }, - history, - logs, - path: join(directory, HISTORY_DB_NAME), - relay: (workerId, event, payload, at) => { - relayed += 1; - bus.republish({ - event: event as EventName, - id: `evt_relay_${options.at ?? T0}_${relayed}`, - module: "gateway", - payload: { ...payload, workerId } as never, - timestamp: at ?? clock.now(), - }); - }, - }; -} - -async function readWith(rigged: { readonly path: string }, read: (reads: UsageReads) => T) { - const database = await new NodeSqliteOpener().open(rigged.path, { readOnly: true }); - try { - return read(openUsageReads(database)); - } finally { - database.close(); - } -} - -async function count(rigged: { readonly path: string }, sql: string): Promise { - const database = await new NodeSqliteOpener().open(rigged.path, { readOnly: true }); - try { - return Number(database.prepare(sql).get()?.n); - } finally { - database.close(); - } -} - -async function execOn(path: string, ...statements: string[]): Promise { - const database = await new NodeSqliteOpener().open(path, { readOnly: false }); - try { - for (const statement of statements) database.exec(statement); - } finally { - database.close(); - } -} - -const rowsOf = (rigged: Rig) => - readWith(rigged, (reads) => reads.requestRows({ from: 0, to: ALL })); -const factsOf = (rigged: Rig) => - readWith(rigged, (reads) => reads.deviceFacts({ from: 0, to: ALL })); -const buckets = ( - rigged: Rig, - query: { - readonly from: number; - readonly to: number; - readonly platform: string; - readonly workerId?: string; - readonly bucketMs?: number; - }, -) => readWith(rigged, (reads) => reads.minuteBuckets({ bucketMs: MIN, workerId: "", ...query })); - -function row(rows: readonly RequestRow[], requestId: string): RequestRow { - const found = rows.find((candidate) => candidate.requestId === requestId); - if (found === undefined) throw new Error(`no request row for ${requestId}`); - return found; -} - -const requested = (requestId: string, over: Partial = {}) => ({ - requestId, - requestSpec: { model: "iPhone 16", platform: "ios" }, - requester: "alice", - waitPolicy: "wait", - ...over, -}); -const granted = ( - leaseId: string, - requestId: string, - over: Partial = {}, -): EventMap["lease.granted"] => ({ - deviceId: `dev-${leaseId}`, - leaseId, - platform: "ios", - requestId, - requester: "alice", - source: "warm", - ...over, -}); -const released = (leaseId: string, reason: "explicit" | "killed" | "device-lost" = "explicit") => ({ - deviceId: `dev-${leaseId}`, - leaseId, - ownerId: "alice", - reason, -}); -const rejected = ( - requestId: string, - reason: EventMap["lease.rejected"]["reason"], - over: Partial = {}, -): EventMap["lease.rejected"] => ({ - reason, - requestId, - requestSpec: { platform: "ios" }, - requester: "alice", - ...over, -}); -const figures = (running: number, reserved: number, maxRunning: number) => ({ - maxRunning, - reserved, - running, - warm: 0, -}); -const capacity = (input: { - readonly ios?: readonly [number, number, number]; - readonly android?: readonly [number, number, number]; - readonly global?: readonly [number, number, number]; - readonly ram?: readonly [number, number]; -}): EventMap["capacity.changed"] => ({ - android: figures(...(input.android ?? [0, 0, 4])), - global: figures(...(input.global ?? [0, 0, 8])), - ios: figures(...(input.ios ?? [0, 0, 4])), - ...(input.ram === undefined - ? {} - : { ramBudget: { limitBytes: input.ram[1], usedBytes: input.ram[0] } }), -}); - -function figure(bucket: MinuteBucket | undefined, name: UsageValue) { - return bucket?.[name]; -} - describe("usage projection: request rows", () => { it("a request granted, held and released gives one row with requestedAt, grantedAt, source, leaseId, endedAt, heldMs and endReason explicit; an expired lease gives endReason expired", async () => { const r = await rig(); @@ -535,933 +338,3 @@ describe("usage projection: a gateway", () => { await second.history.stop(); }); }); - -describe("usage projection: minute rows", () => { - it("a relayed platform that is not ios or android mints no platform key: a capacity key, a request spec and a device platform of another value change no minute row and leave the platform absent", async () => { - const r = await rig({ fleet: false }); - const others = ["tv", "all", "IOS"]; - others.forEach((platform, index) => { - r.relay( - "w1", - "capacity.changed", - { [platform]: { running: 1, reserved: 1, maxRunning: 4 } } as never, - T0 + index * 10, - ); - r.relay( - "w1", - "lease.requested", - requested(`R-${platform}`, { requestSpec: { platform } }), - T0 + index * 10 + 1, - ); - r.relay( - "w1", - "device.ready", - { deviceId: `d-${platform}`, bootDuration: 900, platform } as never, - T0 + index * 10 + 2, - ); - }); - - const rows = await rowsOf(r); - const facts = await factsOf(r); - - for (const platform of ["tv", "IOS", "ios", "android"]) { - expect(await buckets(r, { from: T0, platform, to: T0 + MIN })).toEqual([]); - } - const all = await buckets(r, { from: T0, platform: "all", to: T0 + MIN }); - expect(all).toHaveLength(1); - expect(figure(all[0], "waiting")?.peak).toBe(3); - for (const name of ["slotsUsed", "slotsMax"] as const) { - expect(figure(all[0], name)).toBeUndefined(); - } - for (const platform of others) expect(row(rows, `R-${platform}`).platform).toBe(""); - expect(facts.map((fact) => [fact.deviceId, fact.durationMs, fact.platform])).toEqual( - others.map((platform) => [`d-${platform}`, 900, undefined]), - ); - await r.history.stop(); - }); - - it("slotsUsed is running plus reserved and slotsMax per platform and on all; RAM used and limit and queue depth (both lines summed) only on all", async () => { - const r = await rig(); - r.emit( - "capacity.changed", - capacity({ android: [0, 0, 3], global: [1, 1, 6], ios: [1, 1, 4], ram: [100, 1_000] }), - ); - r.emit("queue.changed", { depth: 3 }); - r.emit("queue.changed", { depth: 2, physical: true }); - r.at(T0 + 3 * MIN); - r.emit("queue.changed", { depth: 1, physical: true }); - r.at(T0 + 5 * MIN); - r.emit( - "capacity.changed", - capacity({ android: [0, 0, 3], global: [1, 1, 6], ios: [1, 1, 4], ram: [100, 1_000] }), - ); - - const ios = await buckets(r, { from: T0, platform: "ios", to: T0 + MIN }); - const all = await buckets(r, { from: T0, platform: "all", to: T0 + MIN }); - - expect(ios).toHaveLength(1); - expect(figure(ios[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - expect(figure(ios[0], "slotsMax")).toEqual({ integral: 4 * MIN, peak: 4 }); - for (const name of ["ramUsedBytes", "ramLimitBytes", "queueDepth"] as const) { - expect(figure(ios[0], name)).toBeUndefined(); - } - expect(figure(all[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - expect(figure(all[0], "slotsMax")).toEqual({ integral: 6 * MIN, peak: 6 }); - expect(figure(all[0], "ramUsedBytes")).toEqual({ integral: 100 * MIN, peak: 100 }); - expect(figure(all[0], "ramLimitBytes")).toEqual({ integral: 1_000 * MIN, peak: 1_000 }); - expect(figure(all[0], "queueDepth")).toEqual({ integral: 5 * MIN, peak: 5 }); - await r.history.stop(); - }); - - it("peak, last and integral of a value that steps twice in a minute are the highest, the final and the time-weighted sum; an all row's peak is the highest total at one moment, not the sum of platform peaks", async () => { - const r = await rig(); - r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); - r.at(T0 + 15_000); - r.emit("capacity.changed", capacity({ ios: [3, 0, 4] })); - r.at(T0 + 30_000); - r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - r.at(T0 + 2 * MIN); - r.emit("lease.requested", requested("R1")); - r.emit("lease.requested", requested("R2", { requestSpec: { platform: "android" } })); - r.emit("lease.granted", granted("L1", "R1")); - r.at(T0 + 2 * MIN + 10_000); - r.emit("lease.released", released("L1")); - r.at(T0 + 2 * MIN + 20_000); - r.emit("lease.granted", granted("L2", "R2", { platform: "android" })); - - const first = await buckets(r, { from: T0, platform: "ios", to: T0 + 2 * MIN }); - const held = (platform: string) => - buckets(r, { from: T0 + 2 * MIN, platform, to: T0 + 3 * MIN }); - - expect(figure(first[0], "slotsUsed")).toEqual({ integral: 15_000 + 45_000 + 60_000, peak: 3 }); - expect(figure(first[1], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - expect(figure((await held("all"))[0], "leasesHeld")?.peak).toBe(1); - expect(figure((await held("ios"))[0], "leasesHeld")?.peak).toBe(1); - expect(figure((await held("android"))[0], "leasesHeld")?.peak).toBe(1); - await r.history.stop(); - }); - - it("leasesHeld by lease.granted's platform and waiting per platform rise and fall with grants, ends and rejections, across two minutes", async () => { - const r = await rig(); - r.emit("lease.requested", requested("R1")); - r.at(T0 + 30_000); - r.emit("lease.granted", granted("L1", "R1")); - r.at(T0 + 40_000); - r.emit("lease.requested", requested("R2")); - r.at(T0 + 50_000); - r.emit("lease.rejected", rejected("R2", "no-wait")); - r.at(T0 + MIN + 30_000); - r.emit("lease.released", released("L1")); - - const ios = await buckets(r, { from: T0, platform: "ios", to: T0 + 2 * MIN }); - const android = await buckets(r, { from: T0, platform: "android", to: T0 + 2 * MIN }); - - expect(ios).toHaveLength(2); - expect(figure(ios[0], "waiting")).toEqual({ integral: 40_000, peak: 1 }); - expect(figure(ios[0], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); - expect(figure(ios[1], "waiting")).toEqual({ integral: 0, peak: 0 }); - expect(figure(ios[1], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); - expect( - android.flatMap((bucket) => [figure(bucket, "waiting"), figure(bucket, "leasesHeld")]), - ).toEqual([]); - await r.history.stop(); - }); - - it("a step stamped before its key's last step counts at the last step's time, on a relayed row and on a * row", async () => { - const r = await rig({ fleet: true }); - r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 4] }), T0 + 10_000); - r.relay("w1", "capacity.changed", capacity({ global: [3, 0, 4], ios: [3, 0, 4] }), T0 + 5_000); - r.relay("w2", "capacity.changed", capacity({ global: [0, 0, 4], ios: [0, 0, 4] }), T0 + 20_000); - r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 4] }), T0 + 30_000); - r.relay("w2", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] }), T0 + 25_000); - - const own = await buckets(r, { from: T0, platform: "all", to: T0 + MIN, workerId: "w1" }); - const fleet = await buckets(r, { from: T0, platform: "all", to: T0 + MIN, workerId: "*" }); - - expect(figure(own[0], "slotsUsed")).toEqual({ integral: 3 * 20_000 + 1 * 30_000, peak: 3 }); - expect(figure(fleet[0], "slotsUsed")).toEqual({ - integral: 150_000, - peak: 3, - }); - await r.history.stop(); - }); - - it("on a gateway, the * row sums the workers' slots, RAM and leases and holds only the gateway's own queue depth and waiting; after worker.disconnected that worker leaves the sums", async () => { - const r = await rig({ fleet: true }); - r.relay( - "w1", - "capacity.changed", - capacity({ global: [1, 0, 4], ios: [1, 0, 2], ram: [100, 1_000] }), - ); - r.relay( - "w2", - "capacity.changed", - capacity({ global: [2, 0, 4], ios: [2, 0, 2], ram: [200, 1_000] }), - ); - r.relay("w1", "lease.granted", granted("WL1", "X1")); - r.relay("w2", "lease.granted", granted("WL2", "X2", { platform: "android" })); - r.at(T0 + 10_000); - r.relay("w1", "queue.changed", { depth: 9 }); - r.relay("w1", "lease.requested", requested("W-req")); - r.at(T0 + 20_000); - r.emit("queue.changed", { depth: 4 }); - r.emit("lease.requested", requested("G1")); - r.at(T0 + MIN); - r.emit("worker.disconnected", { leaseCount: 1, workerId: "w1" }); - - const first = (platform: string, workerId = "*") => - buckets(r, { from: T0, platform, to: T0 + MIN, workerId }); - const later = (platform: string, workerId = "*") => - buckets(r, { from: T0 + MIN, platform, to: T0 + 2 * MIN, workerId }); - const peakOf = async (query: ReturnType, value: UsageValue) => - figure((await query)[0], value)?.peak; - - expect(await peakOf(first("all"), "slotsUsed")).toBe(3); - expect(await peakOf(first("all"), "slotsMax")).toBe(8); - expect(await peakOf(first("all"), "ramUsedBytes")).toBe(300); - expect(await peakOf(first("all"), "ramLimitBytes")).toBe(2_000); - expect(await peakOf(first("all"), "leasesHeld")).toBe(2); - expect(await peakOf(first("ios"), "slotsUsed")).toBe(3); - expect(await peakOf(first("ios"), "leasesHeld")).toBe(1); - expect(await peakOf(first("all"), "queueDepth")).toBe(4); - expect(await peakOf(first("all"), "waiting")).toBe(1); - expect(await peakOf(first("all", "w1"), "queueDepth")).toBe(9); - expect(figure((await later("all"))[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - expect(figure((await later("all"))[0], "ramUsedBytes")).toEqual({ - integral: 200 * MIN, - peak: 200, - }); - expect(figure((await later("all"))[0], "leasesHeld")).toEqual({ integral: MIN, peak: 1 }); - expect(await later("all", "w1")).toEqual([]); - await r.history.stop(); - }); - - it("a worker that reconnects after worker.disconnected counts its held leases in the * row again", async () => { - const r = await rig({ fleet: true }); - r.relay("w1", "capacity.changed", capacity({ global: [1, 0, 4], ios: [1, 0, 2] }), T0); - r.relay("w1", "lease.granted", granted("WL1", "X1"), T0 + 1_000); - r.at(T0 + MIN); - r.emit("worker.disconnected", { leaseCount: 1, workerId: "w1" }); - r.at(T0 + 2 * MIN); - r.relay( - "w1", - "capacity.changed", - capacity({ global: [1, 0, 4], ios: [1, 0, 2] }), - T0 + 2 * MIN, - ); - - const star = (minute: number) => - buckets(r, { - from: T0 + minute * MIN, - platform: "ios", - to: T0 + (minute + 1) * MIN, - workerId: "*", - }); - - expect(figure((await star(0))[0], "leasesHeld")).toMatchObject({ peak: 1 }); - expect(figure((await star(1))[0], "leasesHeld")).toEqual({ integral: 0, peak: 0 }); - expect(figure((await star(2))[0], "leasesHeld")).toEqual({ integral: MIN, peak: 1 }); - await r.history.stop(); - }); - - it("daemon.stopping ends the carried levels until the next daemon.started, and on a gateway worker.disconnected ends that worker's levels, so no buckets appear in the gap", async () => { - const r = await rig({ fleet: true }); - r.emit("capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); - r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); - r.at(T0 + 4 * MIN + 30_000); - r.emit("lease.requested", requested("R1")); - r.at(T0 + 5 * MIN); - r.emit("worker.disconnected", { leaseCount: 0, workerId: "w1" }); - r.at(T0 + 6 * MIN); - r.emit("daemon.stopping", { reason: "signal" }); - r.at(T0 + 9 * MIN); - r.emit("daemon.started", { configSnapshot: {}, version: "1" }); - r.emit("capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); - r.at(T0 + 11 * MIN + 30_000); - r.emit("daemon.stopping", { reason: "signal" }); - - const carried = await buckets(r, { from: T0, platform: "ios", to: T0 + 12 * MIN }); - const worker = await buckets(r, { - from: T0, - platform: "ios", - to: T0 + 12 * MIN, - workerId: "w1", - }); - - expect(carried.map((bucket) => (bucket.start - T0) / MIN)).toEqual([ - 0, 1, 2, 3, 4, 5, 9, 10, 11, - ]); - expect(carried.slice(0, 8).every((bucket) => bucket.knownMs === MIN)).toBe(true); - expect(carried[8]).toMatchObject({ knownMs: 30_000 }); - expect(figure(carried[8], "slotsUsed")).toEqual({ integral: 2 * 30_000, peak: 2 }); - expect(worker.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 1, 2, 3, 4]); - await r.history.stop(); - }); - - it("a value is carried across minutes with no row, from before from, and a first value mid-minute counts from its step", async () => { - const r = await rig(); - r.at(T0 + 4 * MIN + 30_000); - r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - r.at(T0 + 9 * MIN); - r.emit("lease.requested", requested("R1")); - - const whole = await buckets(r, { from: T0, platform: "ios", to: T0 + 9 * MIN }); - const later = await buckets(r, { from: T0 + 6 * MIN, platform: "ios", to: T0 + 8 * MIN }); - - expect(whole.map((bucket) => bucket.start - T0)).toEqual([4, 5, 6, 7, 8].map((m) => m * MIN)); - expect(whole[0]).toMatchObject({ knownMs: 30_000 }); - expect(figure(whole[0], "slotsUsed")).toEqual({ integral: 60_000, peak: 2 }); - expect(later).toHaveLength(2); - expect(figure(later[1], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - await r.history.stop(); - }); - - it("nothing is carried over a crash's downtime", async () => { - const first = await rig(); - first.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - first.at(T0 + 30_000); - first.emit("lease.requested", requested("R1")); - await first.history.stop(); - - const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); - second.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - second.emit("daemon.started", { configSnapshot: {}, version: "1" }); - second.emit("lease.requested", requested("R2")); - second.at(T0 + 12 * MIN); - second.emit("lease.requested", requested("R3")); - - const read = await buckets(second, { from: T0, platform: "ios", to: T0 + 13 * MIN }); - - expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 10, 11, 12]); - await second.history.stop(); - }); - - it("a start catch-up reads one run in seq order, so a relayed event stamped before the run's own start does not begin a new run", async () => { - const r = await rig({ fleet: true, start: false }); - r.at(T0 + 2_000); - r.emit("daemon.started", { configSnapshot: {}, version: "1" }); - r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] }), T0 + 1_500); - await r.history.start(); - r.at(T0 + 3 * MIN); - r.relay("w1", "capacity.changed", capacity({ global: [2, 0, 4], ios: [2, 0, 4] })); - - const read = await buckets(r, { from: T0, platform: "ios", to: T0 + 4 * MIN, workerId: "*" }); - - expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0, 1, 2, 3]); - await r.history.stop(); - }); - - it("a key the next run has not stepped by its daemon.started is closed there, not carried over the downtime", async () => { - const first = await rig(); - first.emit("capacity.changed", capacity({ android: [1, 0, 3], ios: [2, 0, 4] })); - await first.history.stop(); - - const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); - const { android: _unreported, ...iosOnly } = capacity({ ios: [2, 0, 4] }); - second.emit("capacity.changed", iosOnly as EventMap["capacity.changed"]); - second.emit("daemon.started", { configSnapshot: {}, version: "1" }); - second.at(T0 + 12 * MIN); - second.emit("capacity.changed", iosOnly as EventMap["capacity.changed"]); - - const read = await buckets(second, { from: T0, platform: "android", to: T0 + 13 * MIN }); - - expect(read.map((bucket) => (bucket.start - T0) / MIN)).toEqual([0]); - await second.history.stop(); - }); - - it("after a restart, open requests and held leases are restored, leasesHeld and waiting get a step at daemon.started, and every row written before the stop is unchanged", async () => { - const first = await rig(); - first.emit("lease.requested", requested("R1")); - first.emit("lease.granted", granted("L1", "R1")); - first.emit("lease.requested", requested("R2")); - first.emit("lease.requested", requested("R3")); - first.at(T0 + 20_000); - first.emit("lease.rejected", rejected("R3", "no-wait")); - first.emit("daemon.stopping", { reason: "signal" }); - const rowsBefore = await rowsOf(first); - const minutesBefore = await buckets(first, { from: T0, platform: "ios", to: T0 + MIN }); - await first.history.stop(); - - const second = await rig({ at: T0 + 10 * MIN, directory: first.directory }); - second.emit("capacity.changed", capacity({ ios: [2, 0, 4], ram: [5, 9] })); - second.emit("queue.changed", { depth: 3 }); - second.emit("daemon.started", { configSnapshot: {}, version: "1" }); - second.at(T0 + 10 * MIN + 30_000); - second.emit("lease.released", released("L1")); - second.emit("lease.rejected", rejected("R2", "daemon-restarted")); - - const rows = await rowsOf(second); - const minutes = await buckets(second, { - from: T0 + 10 * MIN, - platform: "ios", - to: T0 + 11 * MIN, - }); - - expect(row(rows, "R1")).toMatchObject({ - endedAt: T0 + 10 * MIN + 30_000, - endReason: "explicit", - heldMs: 10 * MIN + 30_000, - }); - expect(row(rows, "R2")).toMatchObject({ reason: "daemon-restarted" }); - expect(row(rows, "R3")).toEqual(row(rowsBefore, "R3")); - expect(figure(minutes[0], "leasesHeld")).toEqual({ integral: 30_000, peak: 1 }); - expect(figure(minutes[0], "waiting")).toEqual({ integral: 30_000, peak: 1 }); - expect(await buckets(second, { from: T0, platform: "ios", to: T0 + MIN })).toEqual( - minutesBefore, - ); - const after = await buckets(second, { - from: T0 + 10 * MIN, - platform: "all", - to: T0 + 12 * MIN, - }); - expect(after.map((bucket) => figure(bucket, "ramUsedBytes")?.peak)).toEqual([5, 5]); - expect(after.map((bucket) => figure(bucket, "queueDepth")?.peak)).toEqual([3, 3]); - const slots = await buckets(second, { - from: T0 + 10 * MIN, - platform: "ios", - to: T0 + 12 * MIN, - }); - expect(slots.map((bucket) => figure(bucket, "slotsUsed")?.peak)).toEqual([2, 2]); - await second.history.stop(); - }); -}); - -describe("usage projection: device facts", () => { - it("one device fact row per listed device event and per lease.declined, with durationMs, platform from the event (absent when the event has none) and workerId on a relayed one; a device.ready with bootDuration 0 writes none", async () => { - const r = await rig({ fleet: true }); - r.emit("device.provisioned", { - deviceId: "d1", - driver: "fake", - duration: 1_200, - spec: { platform: "android" }, - }); - r.at(T0 + 1_000); - r.emit("device.ready", { bootDuration: 3_400, deviceId: "d1", platform: "ios" }); - r.at(T0 + 2_000); - r.emit("device.ready", { bootDuration: 0, deviceId: "d1", platform: "ios" }); - r.at(T0 + 3_000); - r.emit("device.quarantined", { - deviceId: "d2", - kind: "stalled", - maxRetries: 3, - nextRetryAt: T0 + HOUR, - platform: "ios", - }); - r.at(T0 + 4_000); - r.emit("device.recovered", { - attempts: 1, - deviceId: "d3", - duration: 900, - leaseId: "L", - platform: "android", - }); - r.at(T0 + 5_000); - r.emit("device.quarantine-recovered", { - attempts: 2, - deviceId: "d4", - platform: "ios", - strategy: "erase", - }); - r.at(T0 + 6_000); - r.emit("device.recovery-failed", { - attempts: 3, - deviceId: "d5", - error: "e", - leaseId: "L", - platform: "ios", - reason: "r", - }); - r.at(T0 + 7_000); - r.emit("device.quarantine-abandoned", { attempts: 3, deviceId: "d6", platform: "android" }); - r.at(T0 + 8_000); - r.emit("device.quarantine-stranded", { - attempts: 3, - deviceId: "d7", - error: "e", - platform: "ios", - }); - r.at(T0 + 9_000); - r.emit("device.ready", { bootDuration: 2_000, deviceId: "d8" } as never); - r.at(T0 + 10_000); - r.relay("w1", "device.ready", { bootDuration: 5_000, deviceId: "d9", platform: "android" }); - r.at(T0 + 11_000); - r.emit("lease.declined", { - fleetRequestId: "F", - reason: "no-wait", - requestId: "P", - requestSpec: { platform: "android" }, - requester: "alice", - }); - r.emit("lease.requested", requested("P", { fleetRequestId: "F" })); - - const facts = await factsOf(r); - - expect(facts).toMatchObject([ - { - at: T0, - deviceId: "d1", - durationMs: 1_200, - kind: "device.provisioned", - platform: "android", - }, - { at: T0 + 1_000, deviceId: "d1", durationMs: 3_400, kind: "device.ready", platform: "ios" }, - { at: T0 + 3_000, deviceId: "d2", kind: "device.quarantined", platform: "ios" }, - { - at: T0 + 4_000, - deviceId: "d3", - durationMs: 900, - kind: "device.recovered", - platform: "android", - }, - { at: T0 + 5_000, deviceId: "d4", kind: "device.quarantine-recovered", platform: "ios" }, - { at: T0 + 6_000, deviceId: "d5", kind: "device.recovery-failed", platform: "ios" }, - { at: T0 + 7_000, deviceId: "d6", kind: "device.quarantine-abandoned", platform: "android" }, - { at: T0 + 8_000, deviceId: "d7", kind: "device.quarantine-stranded", platform: "ios" }, - { at: T0 + 9_000, deviceId: "d8", durationMs: 2_000, kind: "device.ready" }, - { - at: T0 + 10_000, - deviceId: "d9", - durationMs: 5_000, - kind: "device.ready", - platform: "android", - workerId: "w1", - }, - { at: T0 + 11_000, kind: "lease.declined", platform: "android" }, - ]); - expect(facts[8]?.platform).toBeUndefined(); - expect(facts[0]?.workerId).toBeUndefined(); - await r.history.stop(); - }); -}); - -describe("usage reads", () => { - it("history.usage.oldestTs() is the earliest of the three tables' times, each table the earliest in one case, and undefined when all are empty", async () => { - const cases: ReadonlyArray<{ - readonly name: string; - readonly expected: number | undefined; - readonly act: (r: Rig) => void; - }> = [ - { act: () => undefined, expected: undefined, name: "empty" }, - { - act: (r) => { - r.at(T0 + 1_000); - r.emit("lease.requested", requested("R1", { fleetRequestId: "F1" })); - r.at(T0 + 5_000); - r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); - r.at(T0 + 2 * MIN); - r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); - }, - expected: T0 + 1_000, - name: "request rows", - }, - { - act: (r) => { - r.at(T0 + 90_000); - r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); - r.at(T0 + 100_000); - r.emit("lease.requested", requested("R1")); - r.at(T0 + 110_000); - r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); - }, - expected: T0 + MIN, - name: "minute rows", - }, - { - act: (r) => { - r.at(T0 + 10_000); - r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); - r.at(T0 + MIN + 20_000); - r.emit("lease.requested", requested("R1")); - r.at(T0 + 3 * MIN); - r.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); - }, - expected: T0 + 10_000, - name: "device fact rows", - }, - ]; - for (const scenario of cases) { - const r = await rig(); - scenario.act(r); - expect(await readWith(r, (reads) => reads.oldestTs()), scenario.name).toBe(scenario.expected); - await r.history.stop(); - } - }); - - it("history.usage.requestRows and deviceFacts return rows with from <= time < to (one at from in, one at to out), in their stated order", async () => { - const r = await rig(); - r.at(T0 - 1); - r.emit("lease.requested", requested("before")); - r.emit("device.ready", { bootDuration: 1, deviceId: "d-before", platform: "ios" }); - r.at(T0); - r.emit("lease.requested", requested("b")); - r.emit("lease.requested", requested("a")); - r.emit("device.ready", { bootDuration: 1, deviceId: "d-1", platform: "ios" }); - r.at(T0 + 500); - r.emit("device.ready", { bootDuration: 1, deviceId: "d-2", platform: "ios" }); - r.at(T0 + 1_000); - r.emit("lease.requested", requested("at-to")); - r.emit("device.ready", { bootDuration: 1, deviceId: "d-to", platform: "ios" }); - - const requests = await readWith(r, (reads) => reads.requestRows({ from: T0, to: T0 + 1_000 })); - const facts = await readWith(r, (reads) => reads.deviceFacts({ from: T0, to: T0 + 1_000 })); - - expect(requests.map((candidate) => candidate.requestId)).toEqual(["a", "b"]); - expect(facts.map((fact) => fact.deviceId)).toEqual(["d-1", "d-2"]); - await r.history.stop(); - }); - - it("history.usage.minuteBuckets returns, per bucket, the largest minute peak, the summed integral and summed knownMs for the asked key only", async () => { - const r = await rig(); - r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [1, 0, 4] })); - r.at(T0 + MIN + 30_000); - r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [4, 0, 4] })); - r.at(T0 + 2 * MIN + 10_000); - r.emit("capacity.changed", capacity({ android: [7, 0, 9], ios: [2, 0, 4] })); - - const wide = await buckets(r, { - bucketMs: 5 * MIN, - from: T0, - platform: "ios", - to: T0 + 10 * MIN, - }); - const later = await buckets(r, { from: T0 + 3 * MIN, platform: "ios", to: T0 + 5 * MIN }); - - expect(wide).toHaveLength(2); - expect(wide[0]).toMatchObject({ knownMs: 5 * MIN, start: T0 }); - expect(figure(wide[0], "slotsUsed")).toEqual({ - integral: 90_000 + 4 * 40_000 + 2 * 170_000, - peak: 4, - }); - expect(figure(wide[1], "slotsUsed")).toEqual({ integral: 2 * 5 * MIN, peak: 2 }); - expect(later.map((bucket) => bucket.start)).toEqual([T0 + 3 * MIN, T0 + 4 * MIN]); - expect(figure(later[0], "slotsUsed")).toEqual({ integral: 2 * MIN, peak: 2 }); - expect(figure(later[0], "ramUsedBytes")).toBeUndefined(); - await r.history.stop(); - }); - - it("the four reads in one read transaction see the same commit while the writer commits between them", async () => { - const r = await rig(); - r.emit("lease.requested", requested("R1")); - const database = await new NodeSqliteOpener().open(r.path, { readOnly: true }); - const reads = openUsageReads(database); - try { - database.exec("BEGIN"); - const before = reads.requestRows({ from: 0, to: ALL }).length; - const minutes = { bucketMs: MIN, from: T0, platform: "ios", to: T0 + MIN, workerId: "" }; - const minutesBefore = reads.minuteBuckets(minutes); - r.at(T0 + 1_000); - r.emit("lease.requested", requested("R2")); - r.emit("device.ready", { bootDuration: 1, deviceId: "d", platform: "ios" }); - r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - const during = { - facts: reads.deviceFacts({ from: 0, to: ALL }).length, - oldest: reads.oldestTs(), - requests: reads.requestRows({ from: 0, to: ALL }).length, - }; - const minutesDuring = reads.minuteBuckets(minutes); - database.exec("COMMIT"); - - expect(before).toBe(1); - expect(during).toEqual({ facts: 0, oldest: T0, requests: 1 }); - expect(minutesDuring).toEqual(minutesBefore); - expect(figure(minutesBefore[0], "slotsUsed")).toBeUndefined(); - expect(reads.requestRows({ from: 0, to: ALL })).toHaveLength(2); - expect(figure(reads.minuteBuckets(minutes)[0], "slotsUsed")).toBeDefined(); - } finally { - database.close(); - } - await r.history.stop(); - }); -}); - -describe("usage projection: versions", () => { - const versioned = ( - version: number, - migrations: readonly string[], - ): HistoryProjectionDefinition => ({ - check: () => true, - create: (database) => ({ - apply: (envelope) => { - if (version === 1) { - database.prepare("INSERT INTO kept (id) VALUES (?)").run(envelope.id); - } - }, - prune: () => undefined, - rebuild: () => undefined, - }), - migrations, - name: "kept", - version, - }); - const CREATE = "CREATE TABLE kept (id TEXT)"; - const ADD = "ALTER TABLE kept ADD COLUMN extra INTEGER DEFAULT 7"; - - it("a stored version below the code's is migrated and keeps its rows", async () => { - const first = await rig({ projections: [versioned(1, [CREATE])] }); - first.emit("daemon.stopping", { reason: "a" }); - await first.history.stop(); - - const second = await rig({ - directory: first.directory, - projections: [versioned(2, [CREATE, ADD])], - }); - const database = await new NodeSqliteOpener().open(second.path, { readOnly: true }); - try { - const kept = database.prepare("SELECT id, extra FROM kept").all(); - expect(kept).toHaveLength(1); - expect(kept[0]?.extra).toBe(7); - } finally { - database.close(); - } - await second.history.stop(); - }); - - it("a migration that throws, or a stored version above the code's, makes start() reject naming the projection and versions, with every row kept", async () => { - const first = await rig({ projections: [versioned(1, [CREATE])] }); - first.emit("daemon.stopping", { reason: "a" }); - await first.history.stop(); - - const broken = await rig({ - directory: first.directory, - projections: [versioned(2, [CREATE, "ALTER TABLE missing ADD COLUMN x INTEGER"])], - start: false, - }); - await expect(broken.history.start()).rejects.toThrow(/kept.*1.*2/s); - await broken.history.stop(); - const accepted = await rig({ - directory: first.directory, - projections: [versioned(3, [CREATE, ADD, ADD.replace("extra", "more")])], - }); - accepted.emit("daemon.stopping", { reason: "b" }); - await accepted.history.stop(); - - const newer = await rig({ - directory: first.directory, - projections: [versioned(2, [CREATE, ADD])], - start: false, - }); - await expect(newer.history.start()).rejects.toThrow(/kept.*3.*2/s); - await newer.history.stop(); - - expect(await count(first, "SELECT count(*) AS n FROM kept")).toBe(1); - }); -}); - -describe("usage projection: rebuild", () => { - async function seed(r: Rig, requests: number): Promise { - for (let index = 0; index < requests; index += 1) { - r.at(T0 + index); - r.emit("lease.requested", requested(`R${index}`)); - if (index % 7 === 0) r.emit("lease.granted", granted(`L${index}`, `R${index}`)); - } - r.emit("device.ready", { bootDuration: 10, deviceId: "d", platform: "ios" }); - r.emit("capacity.changed", capacity({ ios: [2, 0, 4] })); - } - const breakMinuteTable = (path: string) => - execOn(path, "ALTER TABLE minute_rows DROP COLUMN knownMs"); - - it("a failed check rebuilds in batches with other work running between them: rows older than the oldest event are kept, the rest equal the live rows, an event emitted mid-rebuild is projected once, and status() gives rebuiltAt after; a start with nothing to fix gives neither field", async () => { - const first = await rig({ eventRetention: DAY }); - first.at(T0 - 2 * DAY); - first.emit("lease.requested", requested("OLD")); - first.at(T0); - await seed(first, 2_100); - const live = { - facts: await factsOf(first), - minutes: await buckets(first, { from: T0, platform: "ios", to: T0 + 5 * MIN }), - requests: (await rowsOf(first)).filter((candidate) => candidate.requestId !== "OLD"), - }; - await first.history.stop(); - await breakMinuteTable(first.path); - - const second = await rig({ - at: T0 + MIN, - directory: first.directory, - eventRetention: DAY, - start: false, - }); - let between = 0; - const ticker = setInterval(() => undefined, 1); - const tick = (): void => { - const status = second.history.status(); - if (status.opened && status.rebuiltAt === undefined) { - between += 1; - if (between === 1) second.emit("lease.requested", requested("MID")); - } - if (status.rebuiltAt === undefined) setImmediate(tick); - }; - setImmediate(tick); - await second.history.start(); - clearInterval(ticker); - - const requests = await rowsOf(second); - expect(second.history.status().rebuiltAt).toBe(T0 + MIN); - expect(between).toBeGreaterThanOrEqual(2); - expect(requests.filter((candidate) => candidate.requestId === "MID")).toHaveLength(1); - expect(requests.filter((candidate) => candidate.requestId === "OLD")).toHaveLength(1); - expect(requests.filter((candidate) => !["MID", "OLD"].includes(candidate.requestId))).toEqual( - live.requests, - ); - expect(await factsOf(second)).toEqual(live.facts); - const withMid = live.minutes.map((bucket) => - bucket.start < T0 + MIN - ? bucket - : { - // why: the first run was never stopped, so the second run (MID is its first event) carries none of its other levels over the downtime. - knownMs: bucket.knownMs, - start: bucket.start, - waiting: { - integral: (bucket.waiting?.integral ?? 0) + MIN, - peak: (bucket.waiting?.peak ?? 0) + 1, - }, - }, - ); - expect(await buckets(second, { from: T0, platform: "ios", to: T0 + 5 * MIN })).toEqual(withMid); - await second.history.stop(); - - const third = await rig({ at: T0 + 2 * MIN, directory: first.directory, eventRetention: DAY }); - expect(third.history.status()).not.toHaveProperty("rebuiltAt"); - expect(third.history.status()).not.toHaveProperty("rebuildFailed"); - await third.history.stop(); - }); - - it("a stop mid-rebuild leaves the mark and the next start rebuilds", async () => { - const first = await rig(); - await seed(first, 2_100); - const live = await rowsOf(first); - await first.history.stop(); - await breakMinuteTable(first.path); - - const second = await rig({ at: T0 + MIN, directory: first.directory, start: false }); - const starting = second.history.start(); - const stopping = new Promise((resolve) => { - const wait = (): void => { - const status = second.history.status(); - if (status.opened && status.rebuiltAt === undefined) - void second.history.stop().then(resolve); - else setImmediate(wait); - }; - setImmediate(wait); - }); - await Promise.all([starting, stopping]); - - const third = await rig({ at: T0 + 2 * MIN, directory: first.directory }); - expect(third.history.status().rebuiltAt).toBe(T0 + 2 * MIN); - expect(await rowsOf(third)).toEqual(live); - await third.history.stop(); - }); - - it("a rebuild that throws leaves no rows for the events' window, rebuildFailed true, writing true, no rebuiltAt, one error, and later events are still written", async () => { - const first = await rig(); - await seed(first, 2_100); - await first.history.stop(); - await breakMinuteTable(first.path); - await execOn( - first.path, - "CREATE TRIGGER refuse BEFORE INSERT ON request_rows WHEN NEW.requestId = 'R1500' BEGIN SELECT RAISE(ABORT, 'refused'); END", - ); - - const second = await rig({ at: T0 + MIN, directory: first.directory }); - second.emit("lease.requested", requested("LATER")); - const status = second.history.status(); - const stored = await second.history.replay({ sinceTs: 0 }); - - expect(status).toMatchObject({ rebuildFailed: true, writing: true }); - expect(status).not.toHaveProperty("rebuiltAt"); - expect(second.logs.records.filter((record) => record.level === "error")).toHaveLength(1); - expect(await rowsOf(second)).toEqual([]); - expect( - stored.some( - (envelope) => - envelope.payload && (envelope.payload as { requestId?: string }).requestId === "LATER", - ), - ).toBe(true); - await second.history.stop(); - }); -}); - -describe("usage projection: prune", () => { - it("prune deletes request, minute, device fact and held lease rows older than metrics.retention and keeps one at the cutoff, at start and one hour later", async () => { - const retention = 10 * DAY; - const cutoff = T0; - const first = await rig({ - at: cutoff - 5 * MIN, - eventRetention: DAY, - fleet: true, - metricsRetention: retention, - }); - const mark = (at: number, name: string): void => { - first.at(at); - first.emit("lease.requested", requested(`R-${name}`)); - first.emit("device.ready", { bootDuration: 5, deviceId: `d-${name}`, platform: "ios" }); - first.relay("w1", "lease.granted", granted(`L-${name}`, `X-${name}`)); - first.emit("daemon.stopping", { reason: name }); - first.emit("capacity.changed", capacity({ ios: [1, 0, 4] })); - }; - mark(cutoff - 1_000, "old"); - mark(cutoff, "edge"); - mark(cutoff + HOUR, "new"); - await first.history.stop(); - - const second = await rig({ - at: cutoff + retention, - directory: first.directory, - eventRetention: DAY, - fleet: true, - metricsRetention: retention, - }); - const heldSource = async (name: string, workerLeaseId: string) => { - second.emit("lease.requested", requested(`Q-${name}`)); - second.emit("request.granted", { - leaseId: `G-${name}`, - requestId: `Q-${name}`, - worker: "w1", - workerLeaseId, - }); - return row(await rowsOf(second), `Q-${name}`).source; - }; - const minutesOf = async () => - ( - await buckets(second, { - from: cutoff - 2 * MIN, - platform: "ios", - to: cutoff + HOUR + MIN, - workerId: "*", - }) - ).map((bucket) => bucket.start); - const afterStart = { - facts: (await factsOf(second)).map((fact) => fact.deviceId), - minutes: await minutesOf(), - requests: (await rowsOf(second)).map((candidate) => candidate.requestId), - }; - const startSources = [ - await heldSource("start-old", "L-old"), - await heldSource("start-edge", "L-edge"), - await heldSource("start-new", "L-new"), - ]; - second.clock.advance(HOUR); - const afterHour = { - facts: (await factsOf(second)).map((fact) => fact.deviceId), - minutes: await minutesOf(), - requests: (await rowsOf(second)) - .map((candidate) => candidate.requestId) - .filter((requestId) => requestId.startsWith("R-")), - }; - const hourSources = [ - await heldSource("hour-edge", "L-edge"), - await heldSource("hour-new", "L-new"), - ]; - - expect(afterStart.requests).toEqual(["R-edge", "R-new"]); - expect(afterStart.facts).toEqual(["d-edge", "d-new"]); - expect(afterStart.minutes).toEqual([cutoff, cutoff + HOUR]); - expect(startSources).toEqual([undefined, "warm", "warm"]); - expect(afterHour).toEqual({ - facts: ["d-new"], - minutes: [cutoff + HOUR], - requests: ["R-new"], - }); - expect(hourSources).toEqual([undefined, "warm"]); - await second.history.stop(); - }); -}); From e2567a56c553be9f7a62ca8d595be9c1981ed082 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:31:47 +0100 Subject: [PATCH 66/83] test(history): split history.test.ts by topic --- src/history/history-prune.test.ts | 197 ++++++ src/history/history-reader.test.ts | 83 +++ src/history/history-start-recovery.test.ts | 346 ++++++++++ src/history/history.test.ts | 740 +-------------------- src/history/test-history.ts | 170 +++++ 5 files changed, 811 insertions(+), 725 deletions(-) create mode 100644 src/history/history-prune.test.ts create mode 100644 src/history/history-reader.test.ts create mode 100644 src/history/history-start-recovery.test.ts create mode 100644 src/history/test-history.ts diff --git a/src/history/history-prune.test.ts b/src/history/history-prune.test.ts new file mode 100644 index 000000000..9c6591345 --- /dev/null +++ b/src/history/history-prune.test.ts @@ -0,0 +1,197 @@ +import { rm } from "node:fs/promises"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { type EventEnvelope } from "../bus/index.js"; +import { plainProjection } from "./testing.js"; +import { + NOW, + HOUR, + directories, + rig, + stopping, + relayed, + levelRecords, + settle, + sizeOf, +} from "./test-history.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("createEventHistory", () => { + it("prune deletes event rows older than retention and keeps one stamped exactly at the cutoff", async () => { + const rigged = await rig(); + const retention = 2 * HOUR; + const cutoff = NOW - retention; + relayed(rigged.bus, "evt_old", cutoff - 1); + const atCutoff = relayed(rigged.bus, "evt_cutoff", cutoff); + const newer = relayed(rigged.bus, "evt_newer", cutoff + 1); + const history = rigged.history({ retention }); + + await history.start(); + const replayed = await history.replay({ sinceTs: 0 }); + await history.stop(); + + expect(replayed).toEqual([atCutoff, newer]); + }); + + it("prune deletes the oldest event rows, 1 000 per step, until their stored size fits maxBytes, keeps the newest, and after the prune history.db plus its -wal is smaller than before it and the -wal is empty", async () => { + const rigged = await rig({ capacity: 10 }); + const writer = rigged.history(); + await writer.start(); + const padding = "é".repeat(1_000); + const emitted: EventEnvelope[] = []; + for (let index = 0; index < 2_500; index += 1) { + rigged.clock.advance(1); + emitted.push(stopping(rigged.bus, `${padding}${index}`)); + } + await writer.stop(); + const storedSize = (envelope: EventEnvelope): number => + Buffer.byteLength(JSON.stringify(envelope), "utf8"); + const newest1200 = emitted + .slice(-1_200) + .reduce((sum, envelope) => sum + storedSize(envelope), 0); + const diskSize = async (): Promise => + (await sizeOf(rigged.dbPath)) + (await sizeOf(`${rigged.dbPath}-wal`)); + const before = await diskSize(); + const next = await rig({ directory: rigged.directory }); + const history = next.history({ maxBytes: newest1200 }); + + await history.start(); + const kept = await history.replay({ sinceTs: 0 }); + // why: read while the file is open; closing it would drop the -wal and hide a missing checkpoint + const after = await diskSize(); + const walAfter = await sizeOf(`${rigged.dbPath}-wal`); + await history.stop(); + + expect(kept).toEqual(emitted.slice(-500)); + expect(after).toBeLessThan(before); + expect(walAfter).toBe(0); + }); + + it("the hourly prune counts the size of events written since start(), so it trims the oldest 1 000 at a time to maxBytes and the next hour trims nothing more", async () => { + const rigged = await rig({ capacity: 10 }); + const storedSize = (envelope: EventEnvelope): number => + Buffer.byteLength(JSON.stringify(envelope), "utf8"); + const probe = stopping(rigged.bus, "0000"); + const history = rigged.history({ maxBytes: storedSize(probe) * 1_200 }); + await history.start(); + const emitted: EventEnvelope[] = [probe]; + for (let index = 0; index < 2_500; index += 1) { + rigged.clock.advance(1); + emitted.push(stopping(rigged.bus, String(index).padStart(4, "0"))); + } + + rigged.clock.advance(HOUR); + await settle(); + const kept = await history.replay({ sinceTs: 0 }); + rigged.clock.advance(HOUR); + await settle(); + const keptAfterSecondHour = await history.replay({ sinceTs: 0 }); + await history.stop(); + + expect(kept).toEqual(emitted.slice(-501)); + expect(keptAfterSecondHour).toEqual(emitted.slice(-501)); + }); + + it("prune runs again one hour after start() on the clock, not before, and stop() cancels it", async () => { + const rigged = await rig(); + const history = rigged.history({ retention: HOUR / 2 }); + await history.start(); + const old = stopping(rigged.bus, "old"); + + rigged.clock.advance(HOUR - 1); + await settle(); + expect(await history.replay({ sinceTs: 0 })).toEqual([old]); + rigged.clock.advance(1); + await settle(); + expect(await history.replay({ sinceTs: 0 })).toEqual([]); + + const second = stopping(rigged.bus, "second"); + rigged.clock.advance(HOUR - 1); + await settle(); + expect(await history.replay({ sinceTs: 0 })).toEqual([second]); + rigged.clock.advance(1); + await settle(); + expect(await history.replay({ sinceTs: 0 })).toEqual([]); + + await history.stop(); + expect(rigged.clock.pendingTimerCount).toBe(0); + }); + + it("a prune step that throws logs one warning, writing goes on, and the next prune tries again", async () => { + const rigged = await rig(); + let attempts = 0; + const definition = plainProjection("failing-prune", () => ({ + apply: () => undefined, + prune: () => { + attempts += 1; + throw new Error("prune failed"); + }, + })); + const history = rigged.history({ projections: [definition] }); + await history.start(); + const attemptsAtStart = attempts; + const warningsAtStart = levelRecords(rigged, "warn"); + + rigged.clock.advance(HOUR); + await settle(); + const afterFirst = attempts - attemptsAtStart; + const warnings = levelRecords(rigged, "warn") - warningsAtStart; + const later = stopping(rigged.bus, "later"); + rigged.clock.advance(HOUR); + await settle(); + + expect(afterFirst).toBe(1); + expect(warnings).toBe(1); + expect(attempts - attemptsAtStart).toBe(2); + expect(history.status().writing).toBe(true); + expect((await history.replay({ sinceTs: 0 })).at(-1)).toEqual(later); + await history.stop(); + }); + + it("a prune that fails after its age delete committed leaves the stored size true, so the next prune keeps rows under maxBytes", async () => { + const rigged = await rig(); + const retention = 2 * HOUR; + const old = [1, 2, 3].map((index) => + relayed(rigged.bus, `evt_old${index}`, NOW - 3 * HOUR + index), + ); + const size = Buffer.byteLength(JSON.stringify(old[0]), "utf8"); + let attempts = 0; + const definition = plainProjection("fails-once", () => ({ + apply: () => undefined, + prune: () => { + attempts += 1; + if (attempts === 1) throw new Error("prune failed"); + }, + })); + const history = rigged.history({ maxBytes: size * 4, projections: [definition], retention }); + await history.start(); + const fresh = [1, 2, 3].map((index) => relayed(rigged.bus, `evt_new${index}`, NOW + index)); + + rigged.clock.advance(HOUR); + await settle(); + const kept = await history.replay({ sinceTs: 0 }); + await history.stop(); + + expect(attempts).toBe(2); + expect(kept).toEqual(fresh); + }); + + it("skips on replay a row whose id the envelope schema refuses", async () => { + const rigged = await rig(); + const history = rigged.history(); + await history.start(); + const good = relayed(rigged.bus, "evt_good", NOW); + relayed(rigged.bus, "not an id!", NOW + 1); + const after = relayed(rigged.bus, "evt_after", NOW + 2); + + const replayed = await history.replay({ sinceTs: 0 }); + await history.stop(); + expect(replayed).toEqual([good, after]); + }); +}); diff --git a/src/history/history-reader.test.ts b/src/history/history-reader.test.ts new file mode 100644 index 000000000..145ed84c4 --- /dev/null +++ b/src/history/history-reader.test.ts @@ -0,0 +1,83 @@ +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { NodeFilesystem, NodeSqliteOpener } from "../ports/index.js"; +import { HISTORY_DB_NAME, openHistoryReader } from "./index.js"; +import { ProbeSqliteOpener } from "./testing.js"; +import { JUNK, directories, dataDirectory, rig, stopping, readHistoryOf } from "./test-history.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("openHistoryReader", () => { + it("reads rows a writer committed while that writer is still open", async () => { + const rigged = await rig(); + const history = rigged.history(); + await history.start(); + const first = stopping(rigged.bus, "first"); + rigged.clock.advance(5); + const second = stopping(rigged.bus, "second"); + + const read = await readHistoryOf(rigged.directory); + const sinceFirst = await openHistoryReader({ + dataDirectory: rigged.directory, + filesystem: new NodeFilesystem(), + sqlite: new NodeSqliteOpener(), + }).read({ sinceTs: first.timestamp }); + await history.stop(); + + expect(read).toEqual([first, second]); + expect(sinceFirst).toEqual([second]); + }); + + it("opens history.db read-only", async () => { + const rigged = await rig(); + const history = rigged.history(); + await history.start(); + const sqlite = new ProbeSqliteOpener(); + + await openHistoryReader({ + dataDirectory: rigged.directory, + filesystem: new NodeFilesystem(), + sqlite, + }).read({ sinceTs: 0 }); + await history.stop(); + + expect(sqlite.connections.map((connection) => connection.readOnly)).toEqual([true]); + }); + + it("throws, and does not answer empty, when history.db cannot be examined for a reason other than being missing", async () => { + const directory = await dataDirectory(); + class DeniedFilesystem extends NodeFilesystem { + override async stat(): Promise { + throw Object.assign(new Error("permission denied"), { code: "EACCES" }); + } + } + + await expect( + openHistoryReader({ + dataDirectory: directory, + filesystem: new DeniedFilesystem(), + sqlite: new NodeSqliteOpener(), + }).read({ sinceTs: 0 }), + ).rejects.toThrow(/permission denied/); + }); + + it("answers empty when history.db is missing", async () => { + const directory = await dataDirectory(); + + expect(await readHistoryOf(directory)).toEqual([]); + }); + + it("throws when history.db is present and unreadable", async () => { + const directory = await dataDirectory(); + await writeFile(join(directory, HISTORY_DB_NAME), JUNK); + + await expect(readHistoryOf(directory)).rejects.toThrow(/not a database/); + }); +}); diff --git a/src/history/history-start-recovery.test.ts b/src/history/history-start-recovery.test.ts new file mode 100644 index 000000000..f6879b64c --- /dev/null +++ b/src/history/history-start-recovery.test.ts @@ -0,0 +1,346 @@ +import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { SqliteUnavailableError, type SqliteOpener } from "../ports/index.js"; +import { ProbeSqliteOpener } from "./testing.js"; +import { + NOW, + HOUR, + JUNK, + directories, + type Rig, + rig, + stopping, + levelRecords, + settle, + recordingProjection, + damagedNames, + readHistoryOf, + countRows, +} from "./test-history.js"; + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("createEventHistory", () => { + it("opens a new file with incremental auto_vacuum, WAL, synchronous NORMAL and a 5 s busy timeout", async () => { + const rigged = await rig(); + const sqlite = new ProbeSqliteOpener(); + const history = rigged.history({ sqlite }); + await history.start(); + + expect(sqlite.connections).toHaveLength(1); + const connection = sqlite.connections[0]!.database; + const settings = { + autoVacuum: connection.prepare("PRAGMA auto_vacuum").get(), + busyTimeout: connection.prepare("PRAGMA busy_timeout").get(), + journalMode: connection.prepare("PRAGMA journal_mode").get(), + synchronous: connection.prepare("PRAGMA synchronous").get(), + }; + await history.stop(); + expect(settings).toEqual({ + autoVacuum: { auto_vacuum: 2 }, + busyTimeout: { timeout: 5_000 }, + journalMode: { journal_mode: "wal" }, + synchronous: { synchronous: 1 }, + }); + }); + + it("the factory touches no file and status() is opened false; start() writes this run's earlier events once each, and later events as they come", async () => { + const rigged = await rig({ capacity: 3 }); + const history = rigged.history(); + for (const reason of ["a", "b", "c", "d", "e"]) stopping(rigged.bus, reason); + const ring = rigged.bus.replay(); + + expect(await readdir(rigged.directory)).toEqual([]); + expect(history.status()).toEqual({ opened: false, writing: false }); + + await history.start(); + const later = stopping(rigged.bus, "f"); + + expect(history.status()).toEqual({ opened: true, writing: true }); + expect(await history.replay({ sinceTs: 0 })).toEqual([...ring, later]); + await history.stop(); + }); + + it("stop() while start() awaits its open leaves no file open, logs no error and arms no timer, a later start() does nothing, and no prune runs an hour later", async () => { + const rigged = await rig(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const sqlite = new ProbeSqliteOpener({ beforeOpen: () => gate }); + const projection = recordingProjection(); + const history = rigged.history({ projections: [projection.definition], sqlite }); + + const starting = history.start(); + const stopped = history.stop(); + release(); + await Promise.all([starting, stopped]); + await history.start(); + rigged.clock.advance(HOUR); + await settle(); + + expect(sqlite.connections.every((connection) => connection.closed)).toBe(true); + expect(sqlite.opens).toBe(1); + expect(levelRecords(rigged, "error")).toBe(0); + expect(rigged.clock.pendingTimerCount).toBe(0); + expect(projection.pruned).toEqual([]); + }); + + it.each([ + { + kept: async (rigged: Rig, damaged: string) => { + expect(await readFile(`${rigged.dbPath}-wal.damaged-${NOW}`, "utf8")).toBe("wal bytes"); + expect(await readFile(`${rigged.dbPath}-shm.damaged-${NOW}`, "utf8")).toBe("shm bytes"); + expect((await stat(damaged)).isDirectory()).toBe(true); + expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual([ + "history.db-wal", + "history.db-shm", + "history.db", + ]); + }, + name: "a path that will not open", + prepare: async (rigged: Rig) => { + await mkdir(rigged.dbPath); + await writeFile(`${rigged.dbPath}-wal`, "wal bytes"); + await writeFile(`${rigged.dbPath}-shm`, "shm bytes"); + }, + sqlite: () => new ProbeSqliteOpener(), + }, + { + kept: async (rigged: Rig, damaged: string) => { + expect(await readFile(damaged, "utf8")).toBe(JUNK); + expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual(["history.db"]); + }, + name: "a file that is not a database", + prepare: async (rigged: Rig) => { + await writeFile(rigged.dbPath, JUNK); + }, + sqlite: () => new ProbeSqliteOpener(), + }, + { + kept: async (rigged: Rig, damaged: string) => { + expect(await countRows(damaged, "events")).toBe(1); + expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual(["history.db"]); + }, + name: "a file whose quick_check answers something but ok", + prepare: async (rigged: Rig) => { + const earlier = rigged.history(); + await earlier.start(); + stopping(rigged.bus, "earlier"); + await earlier.stop(); + }, + sqlite: () => + new ProbeSqliteOpener({ + answer: ({ connection, sql }) => + connection === 0 && /quick_check/i.test(sql) + ? [{ quick_check: "row 3 missing from index idx" }] + : undefined, + }), + }, + ])( + "$name is renamed to history.db.damaged- with its -wal and -shm, the main file last, a fresh file is written, and status() gives setAsideAt equal to now", + async ({ kept, prepare, sqlite }) => { + const rigged = await rig(); + await prepare(rigged); + const history = rigged.history({ sqlite: sqlite() }); + + await history.start(); + const fresh = stopping(rigged.bus, "fresh"); + + expect(history.status()).toEqual({ opened: true, setAsideAt: NOW, writing: true }); + expect(await history.replay({ sinceTs: 0 })).toEqual([fresh]); + await history.stop(); + await kept(rigged, join(rigged.directory, `history.db.damaged-${NOW}`)); + }, + ); + + it("stop() while the damaged file is being set aside starts no fresh file", async () => { + const rigged = await rig(); + await writeFile(rigged.dbPath, JUNK); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + rigged.filesystem.beforeRename = () => gate; + const sqlite = new ProbeSqliteOpener(); + const history = rigged.history({ sqlite }); + + const starting = history.start(); + await settle(); + const stopped = history.stop(); + release(); + await Promise.all([starting, stopped]); + + expect(sqlite.opens).toBe(1); + expect(await readdir(rigged.directory)).toEqual([`history.db.damaged-${NOW}`]); + }); + + it("a healthy file is not renamed, and keeps its rows", async () => { + const rigged = await rig(); + const earlier = rigged.history(); + await earlier.start(); + const kept = stopping(rigged.bus, "kept"); + await earlier.stop(); + const next = await rig({ directory: rigged.directory }); + const history = next.history(); + + await history.start(); + const replayed = await history.replay({ sinceTs: 0 }); + await history.stop(); + + expect(next.filesystem.renames).toEqual([]); + expect(history.status().setAsideAt).toBeUndefined(); + expect(replayed).toEqual([kept]); + expect(await damagedNames(rigged.directory)).toEqual([]); + }); + + it( + "a lock another connection holds past 5 s at open never sets the file aside, and leaves opened false", + { timeout: 30_000 }, + async () => { + const rigged = await rig(); + const earlier = rigged.history(); + await earlier.start(); + const kept = stopping(rigged.bus, "kept"); + await earlier.stop(); + const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); + holder.exec("PRAGMA locking_mode=EXCLUSIVE"); + holder.exec("BEGIN EXCLUSIVE"); + const next = await rig({ directory: rigged.directory }); + const history = next.history(); + + await history.start(); + holder.exec("COMMIT"); + holder.close(); + + expect(history.status().opened).toBe(false); + expect(next.filesystem.renames).toEqual([]); + expect(await damagedNames(rigged.directory)).toEqual([]); + expect(await readHistoryOf(rigged.directory)).toEqual([kept]); + }, + ); + + it("a lock another connection holds past 5 s at the check never sets the file aside, and leaves opened false", async () => { + const rigged = await rig(); + const earlier = rigged.history(); + await earlier.start(); + const kept = stopping(rigged.bus, "kept"); + await earlier.stop(); + const next = await rig({ directory: rigged.directory }); + const locked = new ProbeSqliteOpener({ + fault: (call) => { + if (call.kind === "prepare" && /quick_check/.test(call.sql)) { + throw Object.assign(new Error("database is locked"), { errcode: 5 }); + } + }, + }); + const history = next.history({ sqlite: locked }); + + await history.start(); + + expect(history.status().opened).toBe(false); + expect(next.filesystem.renames).toEqual([]); + expect(await damagedNames(rigged.directory)).toEqual([]); + expect(await readHistoryOf(rigged.directory)).toEqual([kept]); + }); + + it( + "a lock another connection holds past 5 s at a write never sets the file aside; writing goes false", + { timeout: 30_000 }, + async () => { + const rigged = await rig(); + const history = rigged.history(); + await history.start(); + const kept = stopping(rigged.bus, "kept"); + const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); + holder.exec("BEGIN IMMEDIATE"); + + stopping(rigged.bus, "blocked"); + holder.exec("ROLLBACK"); + holder.close(); + + expect(history.status().writing).toBe(false); + expect(levelRecords(rigged, "error")).toBe(1); + expect(rigged.filesystem.renames).toEqual([]); + await history.stop(); + expect(await damagedNames(rigged.directory)).toEqual([]); + expect(await readHistoryOf(rigged.directory)).toEqual([kept]); + }, + ); + + it( + "a lock another connection holds past 5 s at a prune never sets the file aside; it logs one warning and writing goes on", + { timeout: 30_000 }, + async () => { + const rigged = await rig(); + const history = rigged.history({ retention: HOUR / 2 }); + await history.start(); + stopping(rigged.bus, "old"); + const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); + holder.exec("BEGIN IMMEDIATE"); + + rigged.clock.advance(HOUR); + await settle(); + holder.exec("ROLLBACK"); + holder.close(); + const later = stopping(rigged.bus, "later"); + + expect(levelRecords(rigged, "warn")).toBe(1); + expect(history.status().writing).toBe(true); + expect(rigged.filesystem.renames).toEqual([]); + expect((await history.replay({ sinceTs: NOW })).at(-1)).toEqual(later); + await history.stop(); + }, + ); + + it("a failed load of node:sqlite leaves a healthy history.db byte-identical and in place, and opened false", async () => { + const rigged = await rig(); + const earlier = rigged.history(); + await earlier.start(); + stopping(rigged.bus, "kept"); + await earlier.stop(); + const before = await readFile(rigged.dbPath); + const unavailable: SqliteOpener = { + open: () => + Promise.reject(new SqliteUnavailableError("No such built-in module: node:sqlite")), + }; + const next = await rig({ directory: rigged.directory }); + const history = next.history({ sqlite: unavailable }); + + await history.start(); + + expect(history.status()).toEqual({ opened: false, writing: false }); + expect(next.filesystem.renames).toEqual([]); + expect(await readFile(rigged.dbPath)).toEqual(before); + expect(await damagedNames(rigged.directory)).toEqual([]); + }); + + it("when the fresh file cannot be opened either, opened and writing are false and replay answers the ring", async () => { + const rigged = await rig({ capacity: 2 }); + await writeFile(rigged.dbPath, JUNK); + const sqlite = new ProbeSqliteOpener({ + fault: (call) => { + if (call.kind === "open" && call.connection === 1) throw new Error("unable to open"); + }, + }); + const history = rigged.history({ sqlite }); + stopping(rigged.bus, "one"); + const two = stopping(rigged.bus, "two"); + const three = stopping(rigged.bus, "three"); + + await history.start(); + + expect(history.status().opened).toBe(false); + expect(history.status().writing).toBe(false); + expect(await history.replay({ sinceTs: 0 })).toEqual([two, three]); + expect(await readFile(join(rigged.directory, `history.db.damaged-${NOW}`), "utf8")).toBe(JUNK); + expect(rigged.filesystem.renames).toHaveLength(1); + }); +}); diff --git a/src/history/history.test.ts b/src/history/history.test.ts index 258ee547d..149c5dbba 100644 --- a/src/history/history.test.ts +++ b/src/history/history.test.ts @@ -1,30 +1,21 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { basename, join } from "node:path"; +import { rm, stat } from "node:fs/promises"; + import { afterEach, describe, expect, it } from "vitest"; -import { EventBus, type EventEnvelope } from "../bus/index.js"; -import { - FakeClock, - JsonLinesLogger, - MemoryLogSink, - NodeFilesystem, - NodeSqliteOpener, - SqliteUnavailableError, - type SqliteOpener, -} from "../ports/index.js"; -import { - createEventHistory, - HISTORY_DB_NAME, - type HistoryProjectionDefinition, - openHistoryReader, -} from "./index.js"; +import { NodeSqliteOpener } from "../ports/index.js"; import { plainProjection, ProbeSqliteOpener } from "./testing.js"; - -const NOW = 1_760_605_920_000; -const HOUR = 3_600_000; -const JUNK = "this is not a sqlite database ".repeat(20); -const directories: string[] = []; +import { + NOW, + directories, + rig, + stopping, + relayed, + levelRecords, + recordingProjection, + damagedNames, + countRows, + INSERTS_EVENT, +} from "./test-history.js"; afterEach(async () => { await Promise.all( @@ -32,151 +23,6 @@ afterEach(async () => { ); }); -class RecordingFilesystem extends NodeFilesystem { - readonly renames: Array<{ readonly from: string; readonly to: string }> = []; - beforeRename: (() => Promise) | undefined; - - override async rename(from: string, to: string): Promise { - this.renames.push({ from, to }); - await this.beforeRename?.(); - await super.rename(from, to); - } -} - -interface Rig { - readonly directory: string; - readonly dbPath: string; - readonly clock: FakeClock; - readonly bus: EventBus; - readonly logs: MemoryLogSink; - readonly filesystem: RecordingFilesystem; - readonly sqlite: SqliteOpener; - history(options?: { - readonly sqlite?: SqliteOpener; - readonly retention?: number; - readonly maxBytes?: number; - readonly projections?: readonly HistoryProjectionDefinition[]; - }): ReturnType; -} - -async function dataDirectory(): Promise { - const directory = await mkdtemp(join(tmpdir(), "simlock-history-")); - directories.push(directory); - return directory; -} - -async function rig( - options: { readonly capacity?: number; readonly directory?: string } = {}, -): Promise { - const directory = options.directory ?? (await dataDirectory()); - const clock = new FakeClock(NOW); - let nextId = 0; - const bus = new EventBus(clock, options.capacity ?? 1_000, undefined, { - generate: () => String((nextId += 1)).padStart(6, "0"), - }); - const logs = new MemoryLogSink(); - const filesystem = new RecordingFilesystem(); - const sqlite = new NodeSqliteOpener(); - return { - bus, - clock, - dbPath: join(directory, HISTORY_DB_NAME), - directory, - filesystem, - history: (overrides = {}) => - createEventHistory({ - bus, - clock, - config: { - maxBytes: overrides.maxBytes ?? 256 * 1024 * 1024, - retention: overrides.retention ?? 7 * 24 * HOUR, - }, - dataDirectory: directory, - filesystem, - logger: new JsonLinesLogger({ clock, sink: logs }), - projectionOptions: { fleet: false, metricsRetentionMs: 90 * 24 * HOUR }, - projections: overrides.projections ?? [], - sqlite: overrides.sqlite ?? sqlite, - }), - logs, - sqlite, - }; -} - -function stopping(bus: EventBus, reason: string): EventEnvelope { - return bus.emit("daemon.stopping", { reason }, "daemon"); -} - -function relayed(bus: EventBus, id: string, timestamp: number): EventEnvelope { - return bus.republish({ - event: "daemon.stopping", - id, - module: "daemon", - payload: { reason: id }, - timestamp, - }); -} - -function levelRecords(rigged: Rig, level: string): number { - return rigged.logs.records.filter((record) => record.level === level).length; -} - -async function settle(): Promise { - for (let turn = 0; turn < 5; turn += 1) await new Promise((resolve) => setImmediate(resolve)); -} - -function recordingProjection(): { - readonly definition: HistoryProjectionDefinition; - readonly applied: string[]; - readonly pruned: number[]; -} { - const applied: string[] = []; - const pruned: number[] = []; - return { - applied, - definition: plainProjection("recording", () => ({ - apply: (envelope) => { - applied.push(envelope.id); - }, - prune: (now) => { - pruned.push(now); - }, - })), - pruned, - }; -} - -async function sizeOf(path: string): Promise { - try { - return (await stat(path)).size; - } catch { - return 0; - } -} - -async function damagedNames(directory: string): Promise { - return (await readdir(directory)).filter((name) => name.includes(".damaged-")); -} - -async function readHistoryOf(directory: string): Promise { - return openHistoryReader({ - dataDirectory: directory, - filesystem: new NodeFilesystem(), - sqlite: new NodeSqliteOpener(), - }).read({ sinceTs: 0 }); -} - -async function countRows(path: string, table: string): Promise { - const database = await new NodeSqliteOpener().open(path, { readOnly: true }); - try { - return Number(database.prepare(`SELECT count(*) AS n FROM ${table}`).get()?.n); - } finally { - database.close(); - } -} - -const INSERTS_EVENT = /INSERT\s+(OR\s+IGNORE\s+)?INTO\s+events/i; - describe("createEventHistory", () => { it("creates history.db and its -wal and -shm files owner-only, mode 0o600", async () => { // why: #685 made the event store owner-only; a 022 umask would hide a 0644 default. @@ -592,560 +438,4 @@ describe("createEventHistory", () => { expect(indexed).toEqual([{ name: "events_worker" }]); expect(afterBadRow).toEqual({ events: [good], kind: "events" }); }); - - it("opens a new file with incremental auto_vacuum, WAL, synchronous NORMAL and a 5 s busy timeout", async () => { - const rigged = await rig(); - const sqlite = new ProbeSqliteOpener(); - const history = rigged.history({ sqlite }); - await history.start(); - - expect(sqlite.connections).toHaveLength(1); - const connection = sqlite.connections[0]!.database; - const settings = { - autoVacuum: connection.prepare("PRAGMA auto_vacuum").get(), - busyTimeout: connection.prepare("PRAGMA busy_timeout").get(), - journalMode: connection.prepare("PRAGMA journal_mode").get(), - synchronous: connection.prepare("PRAGMA synchronous").get(), - }; - await history.stop(); - expect(settings).toEqual({ - autoVacuum: { auto_vacuum: 2 }, - busyTimeout: { timeout: 5_000 }, - journalMode: { journal_mode: "wal" }, - synchronous: { synchronous: 1 }, - }); - }); - - it("the factory touches no file and status() is opened false; start() writes this run's earlier events once each, and later events as they come", async () => { - const rigged = await rig({ capacity: 3 }); - const history = rigged.history(); - for (const reason of ["a", "b", "c", "d", "e"]) stopping(rigged.bus, reason); - const ring = rigged.bus.replay(); - - expect(await readdir(rigged.directory)).toEqual([]); - expect(history.status()).toEqual({ opened: false, writing: false }); - - await history.start(); - const later = stopping(rigged.bus, "f"); - - expect(history.status()).toEqual({ opened: true, writing: true }); - expect(await history.replay({ sinceTs: 0 })).toEqual([...ring, later]); - await history.stop(); - }); - - it("stop() while start() awaits its open leaves no file open, logs no error and arms no timer, a later start() does nothing, and no prune runs an hour later", async () => { - const rigged = await rig(); - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const sqlite = new ProbeSqliteOpener({ beforeOpen: () => gate }); - const projection = recordingProjection(); - const history = rigged.history({ projections: [projection.definition], sqlite }); - - const starting = history.start(); - const stopped = history.stop(); - release(); - await Promise.all([starting, stopped]); - await history.start(); - rigged.clock.advance(HOUR); - await settle(); - - expect(sqlite.connections.every((connection) => connection.closed)).toBe(true); - expect(sqlite.opens).toBe(1); - expect(levelRecords(rigged, "error")).toBe(0); - expect(rigged.clock.pendingTimerCount).toBe(0); - expect(projection.pruned).toEqual([]); - }); - - it.each([ - { - kept: async (rigged: Rig, damaged: string) => { - expect(await readFile(`${rigged.dbPath}-wal.damaged-${NOW}`, "utf8")).toBe("wal bytes"); - expect(await readFile(`${rigged.dbPath}-shm.damaged-${NOW}`, "utf8")).toBe("shm bytes"); - expect((await stat(damaged)).isDirectory()).toBe(true); - expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual([ - "history.db-wal", - "history.db-shm", - "history.db", - ]); - }, - name: "a path that will not open", - prepare: async (rigged: Rig) => { - await mkdir(rigged.dbPath); - await writeFile(`${rigged.dbPath}-wal`, "wal bytes"); - await writeFile(`${rigged.dbPath}-shm`, "shm bytes"); - }, - sqlite: () => new ProbeSqliteOpener(), - }, - { - kept: async (rigged: Rig, damaged: string) => { - expect(await readFile(damaged, "utf8")).toBe(JUNK); - expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual(["history.db"]); - }, - name: "a file that is not a database", - prepare: async (rigged: Rig) => { - await writeFile(rigged.dbPath, JUNK); - }, - sqlite: () => new ProbeSqliteOpener(), - }, - { - kept: async (rigged: Rig, damaged: string) => { - expect(await countRows(damaged, "events")).toBe(1); - expect(rigged.filesystem.renames.map(({ from }) => basename(from))).toEqual(["history.db"]); - }, - name: "a file whose quick_check answers something but ok", - prepare: async (rigged: Rig) => { - const earlier = rigged.history(); - await earlier.start(); - stopping(rigged.bus, "earlier"); - await earlier.stop(); - }, - sqlite: () => - new ProbeSqliteOpener({ - answer: ({ connection, sql }) => - connection === 0 && /quick_check/i.test(sql) - ? [{ quick_check: "row 3 missing from index idx" }] - : undefined, - }), - }, - ])( - "$name is renamed to history.db.damaged- with its -wal and -shm, the main file last, a fresh file is written, and status() gives setAsideAt equal to now", - async ({ kept, prepare, sqlite }) => { - const rigged = await rig(); - await prepare(rigged); - const history = rigged.history({ sqlite: sqlite() }); - - await history.start(); - const fresh = stopping(rigged.bus, "fresh"); - - expect(history.status()).toEqual({ opened: true, setAsideAt: NOW, writing: true }); - expect(await history.replay({ sinceTs: 0 })).toEqual([fresh]); - await history.stop(); - await kept(rigged, join(rigged.directory, `history.db.damaged-${NOW}`)); - }, - ); - - it("stop() while the damaged file is being set aside starts no fresh file", async () => { - const rigged = await rig(); - await writeFile(rigged.dbPath, JUNK); - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - rigged.filesystem.beforeRename = () => gate; - const sqlite = new ProbeSqliteOpener(); - const history = rigged.history({ sqlite }); - - const starting = history.start(); - await settle(); - const stopped = history.stop(); - release(); - await Promise.all([starting, stopped]); - - expect(sqlite.opens).toBe(1); - expect(await readdir(rigged.directory)).toEqual([`history.db.damaged-${NOW}`]); - }); - - it("a healthy file is not renamed, and keeps its rows", async () => { - const rigged = await rig(); - const earlier = rigged.history(); - await earlier.start(); - const kept = stopping(rigged.bus, "kept"); - await earlier.stop(); - const next = await rig({ directory: rigged.directory }); - const history = next.history(); - - await history.start(); - const replayed = await history.replay({ sinceTs: 0 }); - await history.stop(); - - expect(next.filesystem.renames).toEqual([]); - expect(history.status().setAsideAt).toBeUndefined(); - expect(replayed).toEqual([kept]); - expect(await damagedNames(rigged.directory)).toEqual([]); - }); - - it( - "a lock another connection holds past 5 s at open never sets the file aside, and leaves opened false", - { timeout: 30_000 }, - async () => { - const rigged = await rig(); - const earlier = rigged.history(); - await earlier.start(); - const kept = stopping(rigged.bus, "kept"); - await earlier.stop(); - const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); - holder.exec("PRAGMA locking_mode=EXCLUSIVE"); - holder.exec("BEGIN EXCLUSIVE"); - const next = await rig({ directory: rigged.directory }); - const history = next.history(); - - await history.start(); - holder.exec("COMMIT"); - holder.close(); - - expect(history.status().opened).toBe(false); - expect(next.filesystem.renames).toEqual([]); - expect(await damagedNames(rigged.directory)).toEqual([]); - expect(await readHistoryOf(rigged.directory)).toEqual([kept]); - }, - ); - - it("a lock another connection holds past 5 s at the check never sets the file aside, and leaves opened false", async () => { - const rigged = await rig(); - const earlier = rigged.history(); - await earlier.start(); - const kept = stopping(rigged.bus, "kept"); - await earlier.stop(); - const next = await rig({ directory: rigged.directory }); - const locked = new ProbeSqliteOpener({ - fault: (call) => { - if (call.kind === "prepare" && /quick_check/.test(call.sql)) { - throw Object.assign(new Error("database is locked"), { errcode: 5 }); - } - }, - }); - const history = next.history({ sqlite: locked }); - - await history.start(); - - expect(history.status().opened).toBe(false); - expect(next.filesystem.renames).toEqual([]); - expect(await damagedNames(rigged.directory)).toEqual([]); - expect(await readHistoryOf(rigged.directory)).toEqual([kept]); - }); - - it( - "a lock another connection holds past 5 s at a write never sets the file aside; writing goes false", - { timeout: 30_000 }, - async () => { - const rigged = await rig(); - const history = rigged.history(); - await history.start(); - const kept = stopping(rigged.bus, "kept"); - const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); - holder.exec("BEGIN IMMEDIATE"); - - stopping(rigged.bus, "blocked"); - holder.exec("ROLLBACK"); - holder.close(); - - expect(history.status().writing).toBe(false); - expect(levelRecords(rigged, "error")).toBe(1); - expect(rigged.filesystem.renames).toEqual([]); - await history.stop(); - expect(await damagedNames(rigged.directory)).toEqual([]); - expect(await readHistoryOf(rigged.directory)).toEqual([kept]); - }, - ); - - it( - "a lock another connection holds past 5 s at a prune never sets the file aside; it logs one warning and writing goes on", - { timeout: 30_000 }, - async () => { - const rigged = await rig(); - const history = rigged.history({ retention: HOUR / 2 }); - await history.start(); - stopping(rigged.bus, "old"); - const holder = await rigged.sqlite.open(rigged.dbPath, { readOnly: false }); - holder.exec("BEGIN IMMEDIATE"); - - rigged.clock.advance(HOUR); - await settle(); - holder.exec("ROLLBACK"); - holder.close(); - const later = stopping(rigged.bus, "later"); - - expect(levelRecords(rigged, "warn")).toBe(1); - expect(history.status().writing).toBe(true); - expect(rigged.filesystem.renames).toEqual([]); - expect((await history.replay({ sinceTs: NOW })).at(-1)).toEqual(later); - await history.stop(); - }, - ); - - it("a failed load of node:sqlite leaves a healthy history.db byte-identical and in place, and opened false", async () => { - const rigged = await rig(); - const earlier = rigged.history(); - await earlier.start(); - stopping(rigged.bus, "kept"); - await earlier.stop(); - const before = await readFile(rigged.dbPath); - const unavailable: SqliteOpener = { - open: () => - Promise.reject(new SqliteUnavailableError("No such built-in module: node:sqlite")), - }; - const next = await rig({ directory: rigged.directory }); - const history = next.history({ sqlite: unavailable }); - - await history.start(); - - expect(history.status()).toEqual({ opened: false, writing: false }); - expect(next.filesystem.renames).toEqual([]); - expect(await readFile(rigged.dbPath)).toEqual(before); - expect(await damagedNames(rigged.directory)).toEqual([]); - }); - - it("when the fresh file cannot be opened either, opened and writing are false and replay answers the ring", async () => { - const rigged = await rig({ capacity: 2 }); - await writeFile(rigged.dbPath, JUNK); - const sqlite = new ProbeSqliteOpener({ - fault: (call) => { - if (call.kind === "open" && call.connection === 1) throw new Error("unable to open"); - }, - }); - const history = rigged.history({ sqlite }); - stopping(rigged.bus, "one"); - const two = stopping(rigged.bus, "two"); - const three = stopping(rigged.bus, "three"); - - await history.start(); - - expect(history.status().opened).toBe(false); - expect(history.status().writing).toBe(false); - expect(await history.replay({ sinceTs: 0 })).toEqual([two, three]); - expect(await readFile(join(rigged.directory, `history.db.damaged-${NOW}`), "utf8")).toBe(JUNK); - expect(rigged.filesystem.renames).toHaveLength(1); - }); - - it("prune deletes event rows older than retention and keeps one stamped exactly at the cutoff", async () => { - const rigged = await rig(); - const retention = 2 * HOUR; - const cutoff = NOW - retention; - relayed(rigged.bus, "evt_old", cutoff - 1); - const atCutoff = relayed(rigged.bus, "evt_cutoff", cutoff); - const newer = relayed(rigged.bus, "evt_newer", cutoff + 1); - const history = rigged.history({ retention }); - - await history.start(); - const replayed = await history.replay({ sinceTs: 0 }); - await history.stop(); - - expect(replayed).toEqual([atCutoff, newer]); - }); - - it("prune deletes the oldest event rows, 1 000 per step, until their stored size fits maxBytes, keeps the newest, and after the prune history.db plus its -wal is smaller than before it and the -wal is empty", async () => { - const rigged = await rig({ capacity: 10 }); - const writer = rigged.history(); - await writer.start(); - const padding = "é".repeat(1_000); - const emitted: EventEnvelope[] = []; - for (let index = 0; index < 2_500; index += 1) { - rigged.clock.advance(1); - emitted.push(stopping(rigged.bus, `${padding}${index}`)); - } - await writer.stop(); - const storedSize = (envelope: EventEnvelope): number => - Buffer.byteLength(JSON.stringify(envelope), "utf8"); - const newest1200 = emitted - .slice(-1_200) - .reduce((sum, envelope) => sum + storedSize(envelope), 0); - const diskSize = async (): Promise => - (await sizeOf(rigged.dbPath)) + (await sizeOf(`${rigged.dbPath}-wal`)); - const before = await diskSize(); - const next = await rig({ directory: rigged.directory }); - const history = next.history({ maxBytes: newest1200 }); - - await history.start(); - const kept = await history.replay({ sinceTs: 0 }); - // why: read while the file is open; closing it would drop the -wal and hide a missing checkpoint - const after = await diskSize(); - const walAfter = await sizeOf(`${rigged.dbPath}-wal`); - await history.stop(); - - expect(kept).toEqual(emitted.slice(-500)); - expect(after).toBeLessThan(before); - expect(walAfter).toBe(0); - }); - - it("the hourly prune counts the size of events written since start(), so it trims the oldest 1 000 at a time to maxBytes and the next hour trims nothing more", async () => { - const rigged = await rig({ capacity: 10 }); - const storedSize = (envelope: EventEnvelope): number => - Buffer.byteLength(JSON.stringify(envelope), "utf8"); - const probe = stopping(rigged.bus, "0000"); - const history = rigged.history({ maxBytes: storedSize(probe) * 1_200 }); - await history.start(); - const emitted: EventEnvelope[] = [probe]; - for (let index = 0; index < 2_500; index += 1) { - rigged.clock.advance(1); - emitted.push(stopping(rigged.bus, String(index).padStart(4, "0"))); - } - - rigged.clock.advance(HOUR); - await settle(); - const kept = await history.replay({ sinceTs: 0 }); - rigged.clock.advance(HOUR); - await settle(); - const keptAfterSecondHour = await history.replay({ sinceTs: 0 }); - await history.stop(); - - expect(kept).toEqual(emitted.slice(-501)); - expect(keptAfterSecondHour).toEqual(emitted.slice(-501)); - }); - - it("prune runs again one hour after start() on the clock, not before, and stop() cancels it", async () => { - const rigged = await rig(); - const history = rigged.history({ retention: HOUR / 2 }); - await history.start(); - const old = stopping(rigged.bus, "old"); - - rigged.clock.advance(HOUR - 1); - await settle(); - expect(await history.replay({ sinceTs: 0 })).toEqual([old]); - rigged.clock.advance(1); - await settle(); - expect(await history.replay({ sinceTs: 0 })).toEqual([]); - - const second = stopping(rigged.bus, "second"); - rigged.clock.advance(HOUR - 1); - await settle(); - expect(await history.replay({ sinceTs: 0 })).toEqual([second]); - rigged.clock.advance(1); - await settle(); - expect(await history.replay({ sinceTs: 0 })).toEqual([]); - - await history.stop(); - expect(rigged.clock.pendingTimerCount).toBe(0); - }); - - it("a prune step that throws logs one warning, writing goes on, and the next prune tries again", async () => { - const rigged = await rig(); - let attempts = 0; - const definition = plainProjection("failing-prune", () => ({ - apply: () => undefined, - prune: () => { - attempts += 1; - throw new Error("prune failed"); - }, - })); - const history = rigged.history({ projections: [definition] }); - await history.start(); - const attemptsAtStart = attempts; - const warningsAtStart = levelRecords(rigged, "warn"); - - rigged.clock.advance(HOUR); - await settle(); - const afterFirst = attempts - attemptsAtStart; - const warnings = levelRecords(rigged, "warn") - warningsAtStart; - const later = stopping(rigged.bus, "later"); - rigged.clock.advance(HOUR); - await settle(); - - expect(afterFirst).toBe(1); - expect(warnings).toBe(1); - expect(attempts - attemptsAtStart).toBe(2); - expect(history.status().writing).toBe(true); - expect((await history.replay({ sinceTs: 0 })).at(-1)).toEqual(later); - await history.stop(); - }); - - it("a prune that fails after its age delete committed leaves the stored size true, so the next prune keeps rows under maxBytes", async () => { - const rigged = await rig(); - const retention = 2 * HOUR; - const old = [1, 2, 3].map((index) => - relayed(rigged.bus, `evt_old${index}`, NOW - 3 * HOUR + index), - ); - const size = Buffer.byteLength(JSON.stringify(old[0]), "utf8"); - let attempts = 0; - const definition = plainProjection("fails-once", () => ({ - apply: () => undefined, - prune: () => { - attempts += 1; - if (attempts === 1) throw new Error("prune failed"); - }, - })); - const history = rigged.history({ maxBytes: size * 4, projections: [definition], retention }); - await history.start(); - const fresh = [1, 2, 3].map((index) => relayed(rigged.bus, `evt_new${index}`, NOW + index)); - - rigged.clock.advance(HOUR); - await settle(); - const kept = await history.replay({ sinceTs: 0 }); - await history.stop(); - - expect(attempts).toBe(2); - expect(kept).toEqual(fresh); - }); - - it("skips on replay a row whose id the envelope schema refuses", async () => { - const rigged = await rig(); - const history = rigged.history(); - await history.start(); - const good = relayed(rigged.bus, "evt_good", NOW); - relayed(rigged.bus, "not an id!", NOW + 1); - const after = relayed(rigged.bus, "evt_after", NOW + 2); - - const replayed = await history.replay({ sinceTs: 0 }); - await history.stop(); - expect(replayed).toEqual([good, after]); - }); -}); - -describe("openHistoryReader", () => { - it("reads rows a writer committed while that writer is still open", async () => { - const rigged = await rig(); - const history = rigged.history(); - await history.start(); - const first = stopping(rigged.bus, "first"); - rigged.clock.advance(5); - const second = stopping(rigged.bus, "second"); - - const read = await readHistoryOf(rigged.directory); - const sinceFirst = await openHistoryReader({ - dataDirectory: rigged.directory, - filesystem: new NodeFilesystem(), - sqlite: new NodeSqliteOpener(), - }).read({ sinceTs: first.timestamp }); - await history.stop(); - - expect(read).toEqual([first, second]); - expect(sinceFirst).toEqual([second]); - }); - - it("opens history.db read-only", async () => { - const rigged = await rig(); - const history = rigged.history(); - await history.start(); - const sqlite = new ProbeSqliteOpener(); - - await openHistoryReader({ - dataDirectory: rigged.directory, - filesystem: new NodeFilesystem(), - sqlite, - }).read({ sinceTs: 0 }); - await history.stop(); - - expect(sqlite.connections.map((connection) => connection.readOnly)).toEqual([true]); - }); - - it("throws, and does not answer empty, when history.db cannot be examined for a reason other than being missing", async () => { - const directory = await dataDirectory(); - class DeniedFilesystem extends NodeFilesystem { - override async stat(): Promise { - throw Object.assign(new Error("permission denied"), { code: "EACCES" }); - } - } - - await expect( - openHistoryReader({ - dataDirectory: directory, - filesystem: new DeniedFilesystem(), - sqlite: new NodeSqliteOpener(), - }).read({ sinceTs: 0 }), - ).rejects.toThrow(/permission denied/); - }); - - it("answers empty when history.db is missing", async () => { - const directory = await dataDirectory(); - - expect(await readHistoryOf(directory)).toEqual([]); - }); - - it("throws when history.db is present and unreadable", async () => { - const directory = await dataDirectory(); - await writeFile(join(directory, HISTORY_DB_NAME), JUNK); - - await expect(readHistoryOf(directory)).rejects.toThrow(/not a database/); - }); }); diff --git a/src/history/test-history.ts b/src/history/test-history.ts new file mode 100644 index 000000000..f61512891 --- /dev/null +++ b/src/history/test-history.ts @@ -0,0 +1,170 @@ +import { mkdtemp, readdir, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { EventBus, type EventEnvelope } from "../bus/index.js"; +import { + FakeClock, + JsonLinesLogger, + MemoryLogSink, + NodeFilesystem, + NodeSqliteOpener, + type SqliteOpener, +} from "../ports/index.js"; +import { + createEventHistory, + HISTORY_DB_NAME, + type HistoryProjectionDefinition, + openHistoryReader, +} from "./index.js"; +import { plainProjection } from "./testing.js"; + +export const NOW = 1_760_605_920_000; +export const HOUR = 3_600_000; +export const JUNK = "this is not a sqlite database ".repeat(20); +export const directories: string[] = []; + +export class RecordingFilesystem extends NodeFilesystem { + readonly renames: Array<{ readonly from: string; readonly to: string }> = []; + beforeRename: (() => Promise) | undefined; + + override async rename(from: string, to: string): Promise { + this.renames.push({ from, to }); + await this.beforeRename?.(); + await super.rename(from, to); + } +} + +export interface Rig { + readonly directory: string; + readonly dbPath: string; + readonly clock: FakeClock; + readonly bus: EventBus; + readonly logs: MemoryLogSink; + readonly filesystem: RecordingFilesystem; + readonly sqlite: SqliteOpener; + history(options?: { + readonly sqlite?: SqliteOpener; + readonly retention?: number; + readonly maxBytes?: number; + readonly projections?: readonly HistoryProjectionDefinition[]; + }): ReturnType; +} + +export async function dataDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "simlock-history-")); + directories.push(directory); + return directory; +} + +export async function rig( + options: { readonly capacity?: number; readonly directory?: string } = {}, +): Promise { + const directory = options.directory ?? (await dataDirectory()); + const clock = new FakeClock(NOW); + let nextId = 0; + const bus = new EventBus(clock, options.capacity ?? 1_000, undefined, { + generate: () => String((nextId += 1)).padStart(6, "0"), + }); + const logs = new MemoryLogSink(); + const filesystem = new RecordingFilesystem(); + const sqlite = new NodeSqliteOpener(); + return { + bus, + clock, + dbPath: join(directory, HISTORY_DB_NAME), + directory, + filesystem, + history: (overrides = {}) => + createEventHistory({ + bus, + clock, + config: { + maxBytes: overrides.maxBytes ?? 256 * 1024 * 1024, + retention: overrides.retention ?? 7 * 24 * HOUR, + }, + dataDirectory: directory, + filesystem, + logger: new JsonLinesLogger({ clock, sink: logs }), + projectionOptions: { fleet: false, metricsRetentionMs: 90 * 24 * HOUR }, + projections: overrides.projections ?? [], + sqlite: overrides.sqlite ?? sqlite, + }), + logs, + sqlite, + }; +} + +export function stopping(bus: EventBus, reason: string): EventEnvelope { + return bus.emit("daemon.stopping", { reason }, "daemon"); +} + +export function relayed(bus: EventBus, id: string, timestamp: number): EventEnvelope { + return bus.republish({ + event: "daemon.stopping", + id, + module: "daemon", + payload: { reason: id }, + timestamp, + }); +} + +export function levelRecords(rigged: Rig, level: string): number { + return rigged.logs.records.filter((record) => record.level === level).length; +} + +export async function settle(): Promise { + for (let turn = 0; turn < 5; turn += 1) await new Promise((resolve) => setImmediate(resolve)); +} + +export function recordingProjection(): { + readonly definition: HistoryProjectionDefinition; + readonly applied: string[]; + readonly pruned: number[]; +} { + const applied: string[] = []; + const pruned: number[] = []; + return { + applied, + definition: plainProjection("recording", () => ({ + apply: (envelope) => { + applied.push(envelope.id); + }, + prune: (now) => { + pruned.push(now); + }, + })), + pruned, + }; +} + +export async function sizeOf(path: string): Promise { + try { + return (await stat(path)).size; + } catch { + return 0; + } +} + +export async function damagedNames(directory: string): Promise { + return (await readdir(directory)).filter((name) => name.includes(".damaged-")); +} + +export async function readHistoryOf(directory: string): Promise { + return openHistoryReader({ + dataDirectory: directory, + filesystem: new NodeFilesystem(), + sqlite: new NodeSqliteOpener(), + }).read({ sinceTs: 0 }); +} + +export async function countRows(path: string, table: string): Promise { + const database = await new NodeSqliteOpener().open(path, { readOnly: true }); + try { + return Number(database.prepare(`SELECT count(*) AS n FROM ${table}`).get()?.n); + } finally { + database.close(); + } +} + +export const INSERTS_EVENT = /INSERT\s+(OR\s+IGNORE\s+)?INTO\s+events/i; From 34e022a7d0735f494a950108df8ef1e12d39fadc Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:33:45 +0100 Subject: [PATCH 67/83] test(agents): split spec-lint.test.ts by topic --- src/spec-lint-design.test.ts | 349 +++++++++++++++++ src/spec-lint-feature.test.ts | 162 ++++++++ src/spec-lint-refactor.test.ts | 73 ++++ src/spec-lint.test.ts | 664 +-------------------------------- src/test-spec-lint.ts | 100 +++++ 5 files changed, 686 insertions(+), 662 deletions(-) create mode 100644 src/spec-lint-design.test.ts create mode 100644 src/spec-lint-feature.test.ts create mode 100644 src/spec-lint-refactor.test.ts create mode 100644 src/test-spec-lint.ts diff --git a/src/spec-lint-design.test.ts b/src/spec-lint-design.test.ts new file mode 100644 index 000000000..f9a5250a8 --- /dev/null +++ b/src/spec-lint-design.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, it } from "vitest"; + +import { FENCE, FOOTER, SHA, lintFeature, lintTask, swap } from "./test-spec-lint.js"; + +const DESIGN_TASK = `Part of #50. + +## Scope + +An operator's config file is read. Gone wrong: the daemon starts with defaults. + +## Technical spec + +Commit ${SHA.slice(0, 9)}. + +### Rows + +Behaviours: B1, B2. Stored state: S1. Existing code: X1. + +### Facts + +none + +### Modules touched + +Searches: \`loadConfig\` (2 hits). + +- \`src/core/config.ts\`, from \`core/config.ts:1\`. + +### Rules in play + +- \`docs/internal/agent-rules/architecture.md\` rule 10: read once. + +### Tests + +Seam: \`loadConfig\`, modelled on \`src/diff-kind.test.ts\`. + +- B1: an operator's config is read: a number or an object accepted, a bad value refused. +- B2: an operator sees the path of a config that does not parse. + +## Done when + +- \`pnpm test:changed\` passes. + +## Out of scope + +## Depends on + +${FOOTER} +`; + +const design = (tasks: [string, string]) => `Request: none + +## Problem + +Agents wait. + +## Who it is for + +An operator. + +## Outcome + +They do not wait. + +## How it works for the user + +${FENCE}mermaid +flowchart LR + A --> B +${FENCE} + +## Examples + +## What could go wrong + +## Words used + +none + +## Non-goals + +## Completion conditions + +- The worker keeps the profile. + +## Open questions + +## Decisions + +- ADR 0001 — one: two. + +## Behaviours + +| Id | Behaviour (invoker, action, what it sees) | Seam | Cases | Task | +| -- | --- | --- | --- | --- | +| B1 | An operator's config is read | \`loadConfig\` | \`2\`; \`{}\`; \`0\` refused | ${tasks[0]} | +| B2 | An operator sees the path of a config that does not parse | \`loadConfig\` | bad JSON | ${tasks[0]} | +| B3 | A caller reads the profile over the socket | dispatcher | present; absent | ${tasks[1]} | + +## Stored state + +| Id | What and where | Created, changed, deleted by | Listed or swept by | A restart mid-step leaves | Schema step | Task | Used by | +| -- | --- | --- | --- | --- | --- | --- | --- | +| S1 | Table \`profiles\`, read in \`src/core/config.ts:1\` | created at start | doctor | nothing | \`registry +1\` | ${tasks[0]} | ${tasks[1]} | + +## Interfaces + +| Id | What | Shape and bounds | Errors and limits | Where | Task | Used by | +| -- | --- | --- | --- | --- | --- | --- | +| I1 | \`profile.get\` | \`name\`: 1 to 64 characters; bumps the protocol by one | \`UNKNOWN_PROFILE\`: \`no profile named ""\` | \`src/core/config.ts\` | ${tasks[1]} | — | + +## Existing code + +Commit ${SHA.slice(0, 9)}. + +Searches: \`loadConfig\` (2 hits). + +| Id | Code (path:line) | Today | With this feature | Handled by | +| -- | --- | --- | --- | --- | +| X1 | \`src/core/config.ts:1-2\` | reads the file | also reads profiles | S1 | + +## Tasks + +- ${tasks[0]} Keep the profile. +- ${tasks[1]} Read it over the socket. + +${FOOTER} +`; + +const DESIGN_FEATURE = design(["#100", "#102"]); + +const SOCKET_TASK = `### Rows + +Behaviours: B3. Interfaces: I1. + +### Tests + +Seam line. + +- B3: a caller reads the profile over the socket: present, absent. + +## Depends on + +- #100 +`; + +const DESIGN_WORLD = { + 200: { labels: ["feature:planned"], subIssues: [100, 102] }, + 100: { body: DESIGN_TASK, labels: ["task:ready"], parent: 50 }, + 102: { body: SOCKET_TASK, labels: ["task:ready"] }, +}; + +const fromRows = (body: string) => body.slice(body.indexOf("## Behaviours")); + +describe("spec-lint.mjs on a task that names its parent's design rows", () => { + it("passes when Rows holds every row the parent gives it and Tests has one line per Behaviours id", () => { + expect(lintTask(DESIGN_TASK, DESIGN_FEATURE)).toEqual({ + status: 0, + lines: ["spec-lint: 0 blocking, 0 notes"], + }); + }); + + it.each([ + [ + "a section of the old layout", + swap("### Facts\n", "### Failure modes\n\nnone\n\n### Facts\n"), + "N 100 Technical spec: `### Failure modes` is not a template section", + ], + [ + "no Rows section", + swap("### Rows\n\nBehaviours: B1, B2. Stored state: S1. Existing code: X1.\n\n", ""), + "B 100 Technical spec: `### Rows` is missing; add it", + ], + [ + "an id the parent does not have", + swap("Existing code: X1.", "Existing code: X9."), + "B 100 Rows: X9 is no row of the parent; fix the id", + ], + [ + "a row the parent gives it left out", + swap("Stored state: S1. ", ""), + "B 100 Rows: the parent gives S1 to this task; list it", + ], + [ + "a Tests line with no id", + swap("- B2: an operator sees", "- An operator sees"), + "B 100 Tests: line 2 does not start with its Behaviours id, as in `B3: ...`", + ], + [ + "a Tests line for another task's row", + swap("- B2: an operator sees", "- B3: an operator sees"), + "B 100 Tests: line 2 names B3, which the parent does not give to #100", + ], + [ + "two Tests lines for one row", + swap("- B2: an operator sees", "- B1: an operator sees"), + "B 100 Tests: two lines for B1; one line per row", + ], + [ + "a Behaviours row with no Tests line", + swap("- B2: an operator sees the path of a config that does not parse.\n", ""), + "B 100 Behaviours: #100 has no Tests line for B2; add it", + ], + ])("reports %s", (_name, change, line) => { + expect(lintTask(change(DESIGN_TASK), DESIGN_FEATURE).lines).toContain(line); + }); +}); + +describe("spec-lint.mjs on a feature with design tables", () => { + it("takes `none` under Existing code's Commit and Searches lines, and `none` or a dash in backticks in a cell", () => { + const sentinels = [ + swap( + "| Id | Code (path:line) | Today | With this feature | Handled by |\n| -- | --- | --- | --- | --- |\n| X1 | `src/core/config.ts:1-2` | reads the file | also reads profiles | S1 |", + "none", + ), + swap("| `registry +1` | #100 | #102 |", "| `none` | `#100` | #102 |"), + swap("| #102 | — |", "| #102 | `—` |"), + ].reduce((body, change) => change(body), DESIGN_FEATURE); + + expect(lintFeature(sentinels, DESIGN_WORLD)).toEqual({ + status: 0, + lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], + }); + }); + + it("passes when ids, steps and hand-offs agree with its tasks, and before any task is posted", () => { + expect(lintFeature(DESIGN_FEATURE, DESIGN_WORLD)).toEqual({ + status: 0, + lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], + }); + expect(lintFeature(design(["T1", "T2"]), { 200: { labels: ["feature:spec"] } })).toEqual({ + status: 0, + lines: ["spec-lint: 0 blocking, 0 notes"], + }); + }); + + it.each([ + [ + "a design section left out", + swap("## Interfaces\n", "## Ports\n"), + DESIGN_WORLD, + "B 200 body: `## Interfaces` is missing; add it", + ], + [ + "a design table with no Id column", + (body: string) => + body.replace(fromRows(body), fromRows(body).replace("| Id | What |", "| What |")), + DESIGN_WORLD, + "B 200 Interfaces: no table with an `Id` column; add the rows, or write `none`", + ], + [ + "a design section with no table whose prose only starts a line with none", + (body: string) => + body.replace( + /## Interfaces\n[\s\S]*?\n## Existing code/, + "## Interfaces\n\nThe socket adds `profile.get` for callers, and\nnone of the old calls change.\n\n## Existing code", + ), + DESIGN_WORLD, + "B 200 Interfaces: no table with an `Id` column; add the rows, or write `none`", + ], + [ + "Existing code that says none without its searches", + (body: string) => + body.replace( + /Searches: `loadConfig` \(2 hits\)\.\n\n\| Id \| Code[\s\S]*?\| S1 \|\n/, + "none\n", + ), + DESIGN_WORLD, + "B 200 Existing code: no `Searches:` line; name the searches that found the list", + ], + [ + "an id of another table's letter", + swap("| S1 | Table", "| B4 | Table"), + DESIGN_WORLD, + 'B 200 Stored state: "B4" is no id; number the rows S1, S2', + ], + [ + "an id used twice", + swap("| B2 | An operator sees", "| B1 | An operator sees"), + DESIGN_WORLD, + "B 200 Behaviours: B1 is used twice; renumber one", + ], + [ + "no Commit line under Existing code", + swap(`Commit ${SHA.slice(0, 9)}.\n\nSearches`, "Searches"), + DESIGN_WORLD, + "B 200 Existing code: no `Commit .` line, so no path:line can be checked; add the commit read", + ], + [ + "a path:line past the end of its file", + swap("read in `src/core/config.ts:1`", "read in `src/core/config.ts:99999`"), + DESIGN_WORLD, + expect.stringMatching(/^B 200 Stored state: `src\/core\/config\.ts:99999` is past the end/), + ], + [ + "Existing code with no searches", + swap("Searches: `loadConfig` (2 hits).\n\n| Id | Code", "| Id | Code"), + DESIGN_WORLD, + "B 200 Existing code: no `Searches:` line; name the searches that found the list", + ], + [ + "a protocol version under Interfaces", + swap("bumps the protocol by one", "protocol 78"), + DESIGN_WORLD, + "B 200 Interfaces: names a protocol version; say `bumps the protocol by one`", + ], + [ + "a Schema step that is a version", + swap("`registry +1`", "V2"), + DESIGN_WORLD, + 'B 200 Stored state: S1\'s Schema step "V2" is not ` +` or `none`', + ], + [ + "one schema step taken by two tasks", + swap( + "| `registry +1` | #100 | #102 |\n", + "| `registry +1` | #100 | #102 |\n| S2 | Column `devices.profile` | at provision | list | nothing | `registry +1` | #102 | — |\n", + ), + DESIGN_WORLD, + "B 200 Stored state: S1 and S2 both take `registry +1` in different tasks; give the later task the next step", + ], + [ + "a Stored state row of two tasks", + swap("| `registry +1` | #100 | #102 |", "| `registry +1` | #100, #102 | — |"), + DESIGN_WORLD, + 'B 200 Stored state: S1\'s Task cell "#100, #102" is not one task; name the task that adds it', + ], + [ + "a Task cell still `T1` once the tasks are posted", + (body: string) => body.replace(fromRows(body), fromRows(body).replaceAll("#102", "T2")), + DESIGN_WORLD, + 'B 200 Interfaces: I1\'s Task cell "T2" names no sub-issue; name the task', + ], + [ + "a Used by cell that names a module, not a task", + swap("| `registry +1` | #100 | #102 |", "| `registry +1` | #100 | the CLI |"), + DESIGN_WORLD, + 'B 200 Stored state: S1\'s Used by "the CLI" names no task; name the tasks, or write `—`', + ], + [ + "a task that uses a row without depending on the task that provides it", + (body: string) => body, + { ...DESIGN_WORLD, 102: { body: SOCKET_TASK.replace("- #100\n", "") } }, + "B 102 Depends on: #102 uses S1, which #100 provides, but does not depend on it; add #100", + ], + ])("reports %s", (_name, change, world, line) => { + expect(lintFeature(change(DESIGN_FEATURE), world).lines).toContainEqual(line); + }); +}); diff --git a/src/spec-lint-feature.test.ts b/src/spec-lint-feature.test.ts new file mode 100644 index 000000000..9a2d2105a --- /dev/null +++ b/src/spec-lint-feature.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; + +import { FENCE, FOOTER, WORLD, lintFeature, task } from "./test-spec-lint.js"; + +const FEATURE = `Request: none + +## Problem + +Agents wait. + +## Who it is for + +An operator. + +## Outcome + +They do not wait. + +## How it works for the user + +${FENCE}mermaid +flowchart LR + A --> B +${FENCE} + +## Examples + +## What could go wrong + +## Words used + +- profile: a file, e.g. \`p.json\`. + +## Non-goals + +## Completion conditions + +- The worker keeps the profile. + +## Open questions + +## Decisions + +- ADR 0001 — one: two. + +## Shared values + +none + +## Behaviours + +| Behaviour (invoker, action, what it sees) | Seam | Cases | Task | +| --- | --- | --- | --- | +| A worker restarts and keeps the profile | \`startDaemon\` | same bytes; new port | #101 | +| A caller cancels a waiting request by platform | dispatcher | given; absent | #102 | + +## Tasks + +- #101 Keep the profile. +- #102 Cancel by platform. + +${FOOTER} +`; + +describe("spec-lint.mjs on a feature", () => { + it("passes a feature whose rows match its tasks, and prints the Depends on chain", () => { + expect(lintFeature(FEATURE)).toEqual({ + status: 0, + lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], + }); + }); + + it.each([ + [ + "no Request line", + FEATURE.replace("Request: none", "From the inbox."), + WORLD, + "B 200 body: the first line is not `Request: #` or `Request: none`", + ], + [ + "open questions on a planned feature", + FEATURE.replace("## Open questions\n", "## Open questions\n\n- Which port?\n"), + WORLD, + "B 200 Open questions: not empty on a planned feature; settle them, or move the feature back to feature:spec", + ], + [ + "empty Words used", + FEATURE.replace("- profile: a file, e.g. `p.json`.", ""), + WORLD, + "B 200 Words used: empty; list the terms, or write `none`", + ], + [ + "no mermaid in How it works", + FEATURE.replace(`${FENCE}mermaid`, FENCE), + WORLD, + "B 200 How it works for the user: no ```mermaid block; add the user's flow", + ], + [ + "a task with more Tests lines than rows", + FEATURE, + { + ...WORLD, + 101: { + body: task("A worker restarts and keeps the profile: same bytes.\n- A second line."), + }, + }, + "B 101 Behaviours: #101 has 2 Tests lines and 1 Behaviours rows; one line per row", + ], + [ + "a row naming no sub-issue", + FEATURE.replace("| given; absent | #102 |", "| given; absent | #103 |"), + WORLD, + 'B 200 Behaviours: a row\'s Task cell "#103" names no sub-issue; name one', + ], + [ + "a sub-issue with no row", + FEATURE.replace("| given; absent | #102 |", "| given; absent | #101 |"), + WORLD, + "B 200 Behaviours: #102 has no row; add its behaviours", + ], + [ + "a Depends on number that does not exist", + FEATURE, + { + ...WORLD, + 102: { body: task("A caller cancels a waiting request by platform.", ["#999"]) }, + }, + "B 102 Depends on: #999 does not exist; fix the line", + ], + [ + "a cycle", + FEATURE, + { ...WORLD, 101: { body: task("A worker restarts and keeps the profile.", ["#102"]) } }, + "B 101 Depends on: a cycle: #101 → #102 → #101; drop one of its lines", + ], + [ + "a closed dependency", + FEATURE, + { + ...WORLD, + 102: { body: task("A caller cancels a waiting request by platform.", ["#300"]) }, + 300: { state: "closed" }, + }, + "N 102 Depends on: #300 is closed, so the line orders nothing; drop it", + ], + [ + "an open dependency of another feature", + FEATURE, + { + ...WORLD, + 102: { body: task("A caller cancels a waiting request by platform.", ["#301"]) }, + 301: { parent: 400 }, + }, + "N 102 Depends on: depends-outside #301 (open, feature #400)", + ], + ])("reports %s", (_name, body, world, expected) => { + const result = lintFeature(body, world); + + expect(result.lines).toContain(expected); + expect(result.status).toBe(expected.startsWith("B") ? 1 : 0); + }); +}); diff --git a/src/spec-lint-refactor.test.ts b/src/spec-lint-refactor.test.ts new file mode 100644 index 000000000..4258affce --- /dev/null +++ b/src/spec-lint-refactor.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; + +import { FOOTER, lint, swap, write } from "./test-spec-lint.js"; + +const REFACTOR = `## Why + +\`src/daemon/main.ts\` has 1,908 lines, over ADR 0063's cut-off of 1,400. + +## Target shape + +- \`composeWorker\` moves to \`src/daemon/worker-composition.ts\`. + +## Out of bounds + +- \`src/daemon/server.ts\`. + +## Done when + +- No file in \`src/daemon/\` has more than 1,400 lines. + +${FOOTER} +`; + +describe("spec-lint.mjs on a refactor", () => { + const lintRefactor = (body: string, labels = ["refactor:ready"]) => + lint([ + "100", + "--body-file", + write("refactor.md", body), + "--world", + write("world.json", JSON.stringify({ 100: { labels } })), + ]); + + it("passes Why, Target shape, Out of bounds and Done when with text, and asks for nothing a task has", () => { + expect(lintRefactor(REFACTOR)).toEqual({ + status: 0, + lines: ["spec-lint: 0 blocking, 0 notes"], + }); + }); + + it.each([ + [ + "a missing section", + swap("## Out of bounds\n\n- `src/daemon/server.ts`.\n\n", ""), + "B 100 body: `## Out of bounds` is missing; add it", + ], + [ + "an empty section", + swap("- `composeWorker` moves to `src/daemon/worker-composition.ts`.", ""), + "B 100 Target shape: empty; fill it in", + ], + [ + "sections out of order", + (body: string) => + body + .replace("## Why", "## Swap") + .replace("## Target shape", "## Why") + .replace("## Swap", "## Target shape"), + "B 100 body: `## Target shape` is out of template order; move it", + ], + ])("blocks %s", (_, change, expected) => { + const result = lintRefactor(change(REFACTOR)); + + expect(result.lines).toContain(expected); + expect(result.status).toBe(1); + }); + + it("reads a body with no refactor label as a refactor under --refactor", () => { + const result = lint(["--refactor", "--body-file", write("refactor.md", REFACTOR)]); + + expect(result).toEqual({ status: 0, lines: ["spec-lint: 0 blocking, 0 notes"] }); + }); +}); diff --git a/src/spec-lint.test.ts b/src/spec-lint.test.ts index a9d1c37b4..b46d4f5c0 100644 --- a/src/spec-lint.test.ts +++ b/src/spec-lint.test.ts @@ -1,16 +1,6 @@ -import { execFileSync } from "node:child_process"; -import { mkdtempSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - import { describe, expect, it } from "vitest"; -const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -const SCRIPT = join(ROOT, ".agents", "scripts", "spec-lint.mjs"); -const SHA = execFileSync("git", ["rev-parse", "HEAD"], { cwd: ROOT, encoding: "utf8" }).trim(); -const FOOTER = "*Written by an agent.*"; -const FENCE = "```"; +import { FENCE, FOOTER, SHA, lint, lintTask, swap, write } from "./test-spec-lint.js"; const TASK = `Part of #50. @@ -79,52 +69,8 @@ Seam: \`loadConfig\`, modelled on \`src/diff-kind.test.ts\`. ${FOOTER} `; -const PARENT = `Request: none - -## Behaviours - -| Behaviour (invoker, action, what it sees) | Seam | Cases | Task | -| --- | --- | --- | --- | -| An operator's config is read: number or object accepted, bad value refused | \`loadConfig\` | \`2\`; \`{}\`; \`0\` refused | #100 | - -${FOOTER} -`; - -function write(name: string, text: string): string { - const file = join(mkdtempSync(join(tmpdir(), "spec-lint-")), name); - writeFileSync(file, text); - return file; -} - -function lint(args: string[]) { - try { - const stdout = execFileSync(process.execPath, [SCRIPT, ...args], { - cwd: ROOT, - encoding: "utf8", - }); - return { status: 0, lines: stdout.split("\n").filter((l) => l !== "") }; - } catch (error) { - const failed = error as { status: number; stdout: string }; - return { status: failed.status, lines: failed.stdout.split("\n").filter((l) => l !== "") }; - } -} - -function lintTask(body: string, parent: string | null = PARENT) { - return lint([ - "100", - "--body-file", - write("task.md", body), - ...(parent === null ? [] : ["--parent-file", write("parent.md", parent)]), - "--world", - write("world.json", JSON.stringify({ 100: { parent: 50 } })), - ]); -} - -const swap = (from: string, to: string) => (body: string) => { - expect(body).toContain(from); - return body.replace(from, to); -}; const LINE = "- An operator's config is read: a number or an object accepted, a bad value refused."; + const tests = (n: number) => Array.from({ length: n }, (_, i) => `- An operator's config is read, case ${i}.`).join("\n"); @@ -400,609 +346,3 @@ describe("spec-lint.mjs on a task", () => { expect(result).toEqual({ status: 0, lines: ["spec-lint: 0 blocking, 0 notes"] }); }); }); - -const FEATURE = `Request: none - -## Problem - -Agents wait. - -## Who it is for - -An operator. - -## Outcome - -They do not wait. - -## How it works for the user - -${FENCE}mermaid -flowchart LR - A --> B -${FENCE} - -## Examples - -## What could go wrong - -## Words used - -- profile: a file, e.g. \`p.json\`. - -## Non-goals - -## Completion conditions - -- The worker keeps the profile. - -## Open questions - -## Decisions - -- ADR 0001 — one: two. - -## Shared values - -none - -## Behaviours - -| Behaviour (invoker, action, what it sees) | Seam | Cases | Task | -| --- | --- | --- | --- | -| A worker restarts and keeps the profile | \`startDaemon\` | same bytes; new port | #101 | -| A caller cancels a waiting request by platform | dispatcher | given; absent | #102 | - -## Tasks - -- #101 Keep the profile. -- #102 Cancel by platform. - -${FOOTER} -`; - -const task = (line: string, depends: string[] = []) => `### Tests - -Seam line. - -- ${line} - -## Depends on - -${depends.map((d) => `- ${d}`).join("\n")} -`; - -const WORLD = { - 200: { labels: ["feature:planned"], subIssues: [101, 102] }, - 101: { - body: task("A worker restarts and keeps the profile: same bytes, new port."), - labels: ["task:ready"], - }, - 102: { - body: task("A caller cancels a waiting request by platform: given, absent.", ["#101"]), - labels: ["task:ready"], - }, -}; - -function lintFeature(body: string, world: Record = WORLD) { - return lint([ - "200", - "--feature", - "--body-file", - write("feature.md", body), - "--world", - write("world.json", JSON.stringify(world)), - ]); -} - -describe("spec-lint.mjs on a feature", () => { - it("passes a feature whose rows match its tasks, and prints the Depends on chain", () => { - expect(lintFeature(FEATURE)).toEqual({ - status: 0, - lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], - }); - }); - - it.each([ - [ - "no Request line", - FEATURE.replace("Request: none", "From the inbox."), - WORLD, - "B 200 body: the first line is not `Request: #` or `Request: none`", - ], - [ - "open questions on a planned feature", - FEATURE.replace("## Open questions\n", "## Open questions\n\n- Which port?\n"), - WORLD, - "B 200 Open questions: not empty on a planned feature; settle them, or move the feature back to feature:spec", - ], - [ - "empty Words used", - FEATURE.replace("- profile: a file, e.g. `p.json`.", ""), - WORLD, - "B 200 Words used: empty; list the terms, or write `none`", - ], - [ - "no mermaid in How it works", - FEATURE.replace(`${FENCE}mermaid`, FENCE), - WORLD, - "B 200 How it works for the user: no ```mermaid block; add the user's flow", - ], - [ - "a task with more Tests lines than rows", - FEATURE, - { - ...WORLD, - 101: { - body: task("A worker restarts and keeps the profile: same bytes.\n- A second line."), - }, - }, - "B 101 Behaviours: #101 has 2 Tests lines and 1 Behaviours rows; one line per row", - ], - [ - "a row naming no sub-issue", - FEATURE.replace("| given; absent | #102 |", "| given; absent | #103 |"), - WORLD, - 'B 200 Behaviours: a row\'s Task cell "#103" names no sub-issue; name one', - ], - [ - "a sub-issue with no row", - FEATURE.replace("| given; absent | #102 |", "| given; absent | #101 |"), - WORLD, - "B 200 Behaviours: #102 has no row; add its behaviours", - ], - [ - "a Depends on number that does not exist", - FEATURE, - { - ...WORLD, - 102: { body: task("A caller cancels a waiting request by platform.", ["#999"]) }, - }, - "B 102 Depends on: #999 does not exist; fix the line", - ], - [ - "a cycle", - FEATURE, - { ...WORLD, 101: { body: task("A worker restarts and keeps the profile.", ["#102"]) } }, - "B 101 Depends on: a cycle: #101 → #102 → #101; drop one of its lines", - ], - [ - "a closed dependency", - FEATURE, - { - ...WORLD, - 102: { body: task("A caller cancels a waiting request by platform.", ["#300"]) }, - 300: { state: "closed" }, - }, - "N 102 Depends on: #300 is closed, so the line orders nothing; drop it", - ], - [ - "an open dependency of another feature", - FEATURE, - { - ...WORLD, - 102: { body: task("A caller cancels a waiting request by platform.", ["#301"]) }, - 301: { parent: 400 }, - }, - "N 102 Depends on: depends-outside #301 (open, feature #400)", - ], - ])("reports %s", (_name, body, world, expected) => { - const result = lintFeature(body, world); - - expect(result.lines).toContain(expected); - expect(result.status).toBe(expected.startsWith("B") ? 1 : 0); - }); -}); - -const REFACTOR = `## Why - -\`src/daemon/main.ts\` has 1,908 lines, over ADR 0063's cut-off of 1,400. - -## Target shape - -- \`composeWorker\` moves to \`src/daemon/worker-composition.ts\`. - -## Out of bounds - -- \`src/daemon/server.ts\`. - -## Done when - -- No file in \`src/daemon/\` has more than 1,400 lines. - -${FOOTER} -`; - -const DESIGN_TASK = `Part of #50. - -## Scope - -An operator's config file is read. Gone wrong: the daemon starts with defaults. - -## Technical spec - -Commit ${SHA.slice(0, 9)}. - -### Rows - -Behaviours: B1, B2. Stored state: S1. Existing code: X1. - -### Facts - -none - -### Modules touched - -Searches: \`loadConfig\` (2 hits). - -- \`src/core/config.ts\`, from \`core/config.ts:1\`. - -### Rules in play - -- \`docs/internal/agent-rules/architecture.md\` rule 10: read once. - -### Tests - -Seam: \`loadConfig\`, modelled on \`src/diff-kind.test.ts\`. - -- B1: an operator's config is read: a number or an object accepted, a bad value refused. -- B2: an operator sees the path of a config that does not parse. - -## Done when - -- \`pnpm test:changed\` passes. - -## Out of scope - -## Depends on - -${FOOTER} -`; - -const design = (tasks: [string, string]) => `Request: none - -## Problem - -Agents wait. - -## Who it is for - -An operator. - -## Outcome - -They do not wait. - -## How it works for the user - -${FENCE}mermaid -flowchart LR - A --> B -${FENCE} - -## Examples - -## What could go wrong - -## Words used - -none - -## Non-goals - -## Completion conditions - -- The worker keeps the profile. - -## Open questions - -## Decisions - -- ADR 0001 — one: two. - -## Behaviours - -| Id | Behaviour (invoker, action, what it sees) | Seam | Cases | Task | -| -- | --- | --- | --- | --- | -| B1 | An operator's config is read | \`loadConfig\` | \`2\`; \`{}\`; \`0\` refused | ${tasks[0]} | -| B2 | An operator sees the path of a config that does not parse | \`loadConfig\` | bad JSON | ${tasks[0]} | -| B3 | A caller reads the profile over the socket | dispatcher | present; absent | ${tasks[1]} | - -## Stored state - -| Id | What and where | Created, changed, deleted by | Listed or swept by | A restart mid-step leaves | Schema step | Task | Used by | -| -- | --- | --- | --- | --- | --- | --- | --- | -| S1 | Table \`profiles\`, read in \`src/core/config.ts:1\` | created at start | doctor | nothing | \`registry +1\` | ${tasks[0]} | ${tasks[1]} | - -## Interfaces - -| Id | What | Shape and bounds | Errors and limits | Where | Task | Used by | -| -- | --- | --- | --- | --- | --- | --- | -| I1 | \`profile.get\` | \`name\`: 1 to 64 characters; bumps the protocol by one | \`UNKNOWN_PROFILE\`: \`no profile named ""\` | \`src/core/config.ts\` | ${tasks[1]} | — | - -## Existing code - -Commit ${SHA.slice(0, 9)}. - -Searches: \`loadConfig\` (2 hits). - -| Id | Code (path:line) | Today | With this feature | Handled by | -| -- | --- | --- | --- | --- | -| X1 | \`src/core/config.ts:1-2\` | reads the file | also reads profiles | S1 | - -## Tasks - -- ${tasks[0]} Keep the profile. -- ${tasks[1]} Read it over the socket. - -${FOOTER} -`; - -const DESIGN_FEATURE = design(["#100", "#102"]); -const SOCKET_TASK = `### Rows - -Behaviours: B3. Interfaces: I1. - -### Tests - -Seam line. - -- B3: a caller reads the profile over the socket: present, absent. - -## Depends on - -- #100 -`; -const DESIGN_WORLD = { - 200: { labels: ["feature:planned"], subIssues: [100, 102] }, - 100: { body: DESIGN_TASK, labels: ["task:ready"], parent: 50 }, - 102: { body: SOCKET_TASK, labels: ["task:ready"] }, -}; -const fromRows = (body: string) => body.slice(body.indexOf("## Behaviours")); - -describe("spec-lint.mjs on a task that names its parent's design rows", () => { - it("passes when Rows holds every row the parent gives it and Tests has one line per Behaviours id", () => { - expect(lintTask(DESIGN_TASK, DESIGN_FEATURE)).toEqual({ - status: 0, - lines: ["spec-lint: 0 blocking, 0 notes"], - }); - }); - - it.each([ - [ - "a section of the old layout", - swap("### Facts\n", "### Failure modes\n\nnone\n\n### Facts\n"), - "N 100 Technical spec: `### Failure modes` is not a template section", - ], - [ - "no Rows section", - swap("### Rows\n\nBehaviours: B1, B2. Stored state: S1. Existing code: X1.\n\n", ""), - "B 100 Technical spec: `### Rows` is missing; add it", - ], - [ - "an id the parent does not have", - swap("Existing code: X1.", "Existing code: X9."), - "B 100 Rows: X9 is no row of the parent; fix the id", - ], - [ - "a row the parent gives it left out", - swap("Stored state: S1. ", ""), - "B 100 Rows: the parent gives S1 to this task; list it", - ], - [ - "a Tests line with no id", - swap("- B2: an operator sees", "- An operator sees"), - "B 100 Tests: line 2 does not start with its Behaviours id, as in `B3: ...`", - ], - [ - "a Tests line for another task's row", - swap("- B2: an operator sees", "- B3: an operator sees"), - "B 100 Tests: line 2 names B3, which the parent does not give to #100", - ], - [ - "two Tests lines for one row", - swap("- B2: an operator sees", "- B1: an operator sees"), - "B 100 Tests: two lines for B1; one line per row", - ], - [ - "a Behaviours row with no Tests line", - swap("- B2: an operator sees the path of a config that does not parse.\n", ""), - "B 100 Behaviours: #100 has no Tests line for B2; add it", - ], - ])("reports %s", (_name, change, line) => { - expect(lintTask(change(DESIGN_TASK), DESIGN_FEATURE).lines).toContain(line); - }); -}); - -describe("spec-lint.mjs on a feature with design tables", () => { - it("takes `none` under Existing code's Commit and Searches lines, and `none` or a dash in backticks in a cell", () => { - const sentinels = [ - swap( - "| Id | Code (path:line) | Today | With this feature | Handled by |\n| -- | --- | --- | --- | --- |\n| X1 | `src/core/config.ts:1-2` | reads the file | also reads profiles | S1 |", - "none", - ), - swap("| `registry +1` | #100 | #102 |", "| `none` | `#100` | #102 |"), - swap("| #102 | — |", "| #102 | `—` |"), - ].reduce((body, change) => change(body), DESIGN_FEATURE); - - expect(lintFeature(sentinels, DESIGN_WORLD)).toEqual({ - status: 0, - lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], - }); - }); - - it("passes when ids, steps and hand-offs agree with its tasks, and before any task is posted", () => { - expect(lintFeature(DESIGN_FEATURE, DESIGN_WORLD)).toEqual({ - status: 0, - lines: ["N 200 Depends on: chain 2 of 2 tasks", "spec-lint: 0 blocking, 1 notes"], - }); - expect(lintFeature(design(["T1", "T2"]), { 200: { labels: ["feature:spec"] } })).toEqual({ - status: 0, - lines: ["spec-lint: 0 blocking, 0 notes"], - }); - }); - - it.each([ - [ - "a design section left out", - swap("## Interfaces\n", "## Ports\n"), - DESIGN_WORLD, - "B 200 body: `## Interfaces` is missing; add it", - ], - [ - "a design table with no Id column", - (body: string) => - body.replace(fromRows(body), fromRows(body).replace("| Id | What |", "| What |")), - DESIGN_WORLD, - "B 200 Interfaces: no table with an `Id` column; add the rows, or write `none`", - ], - [ - "a design section with no table whose prose only starts a line with none", - (body: string) => - body.replace( - /## Interfaces\n[\s\S]*?\n## Existing code/, - "## Interfaces\n\nThe socket adds `profile.get` for callers, and\nnone of the old calls change.\n\n## Existing code", - ), - DESIGN_WORLD, - "B 200 Interfaces: no table with an `Id` column; add the rows, or write `none`", - ], - [ - "Existing code that says none without its searches", - (body: string) => - body.replace( - /Searches: `loadConfig` \(2 hits\)\.\n\n\| Id \| Code[\s\S]*?\| S1 \|\n/, - "none\n", - ), - DESIGN_WORLD, - "B 200 Existing code: no `Searches:` line; name the searches that found the list", - ], - [ - "an id of another table's letter", - swap("| S1 | Table", "| B4 | Table"), - DESIGN_WORLD, - 'B 200 Stored state: "B4" is no id; number the rows S1, S2', - ], - [ - "an id used twice", - swap("| B2 | An operator sees", "| B1 | An operator sees"), - DESIGN_WORLD, - "B 200 Behaviours: B1 is used twice; renumber one", - ], - [ - "no Commit line under Existing code", - swap(`Commit ${SHA.slice(0, 9)}.\n\nSearches`, "Searches"), - DESIGN_WORLD, - "B 200 Existing code: no `Commit .` line, so no path:line can be checked; add the commit read", - ], - [ - "a path:line past the end of its file", - swap("read in `src/core/config.ts:1`", "read in `src/core/config.ts:99999`"), - DESIGN_WORLD, - expect.stringMatching(/^B 200 Stored state: `src\/core\/config\.ts:99999` is past the end/), - ], - [ - "Existing code with no searches", - swap("Searches: `loadConfig` (2 hits).\n\n| Id | Code", "| Id | Code"), - DESIGN_WORLD, - "B 200 Existing code: no `Searches:` line; name the searches that found the list", - ], - [ - "a protocol version under Interfaces", - swap("bumps the protocol by one", "protocol 78"), - DESIGN_WORLD, - "B 200 Interfaces: names a protocol version; say `bumps the protocol by one`", - ], - [ - "a Schema step that is a version", - swap("`registry +1`", "V2"), - DESIGN_WORLD, - 'B 200 Stored state: S1\'s Schema step "V2" is not ` +` or `none`', - ], - [ - "one schema step taken by two tasks", - swap( - "| `registry +1` | #100 | #102 |\n", - "| `registry +1` | #100 | #102 |\n| S2 | Column `devices.profile` | at provision | list | nothing | `registry +1` | #102 | — |\n", - ), - DESIGN_WORLD, - "B 200 Stored state: S1 and S2 both take `registry +1` in different tasks; give the later task the next step", - ], - [ - "a Stored state row of two tasks", - swap("| `registry +1` | #100 | #102 |", "| `registry +1` | #100, #102 | — |"), - DESIGN_WORLD, - 'B 200 Stored state: S1\'s Task cell "#100, #102" is not one task; name the task that adds it', - ], - [ - "a Task cell still `T1` once the tasks are posted", - (body: string) => body.replace(fromRows(body), fromRows(body).replaceAll("#102", "T2")), - DESIGN_WORLD, - 'B 200 Interfaces: I1\'s Task cell "T2" names no sub-issue; name the task', - ], - [ - "a Used by cell that names a module, not a task", - swap("| `registry +1` | #100 | #102 |", "| `registry +1` | #100 | the CLI |"), - DESIGN_WORLD, - 'B 200 Stored state: S1\'s Used by "the CLI" names no task; name the tasks, or write `—`', - ], - [ - "a task that uses a row without depending on the task that provides it", - (body: string) => body, - { ...DESIGN_WORLD, 102: { body: SOCKET_TASK.replace("- #100\n", "") } }, - "B 102 Depends on: #102 uses S1, which #100 provides, but does not depend on it; add #100", - ], - ])("reports %s", (_name, change, world, line) => { - expect(lintFeature(change(DESIGN_FEATURE), world).lines).toContainEqual(line); - }); -}); - -describe("spec-lint.mjs on a refactor", () => { - const lintRefactor = (body: string, labels = ["refactor:ready"]) => - lint([ - "100", - "--body-file", - write("refactor.md", body), - "--world", - write("world.json", JSON.stringify({ 100: { labels } })), - ]); - - it("passes Why, Target shape, Out of bounds and Done when with text, and asks for nothing a task has", () => { - expect(lintRefactor(REFACTOR)).toEqual({ - status: 0, - lines: ["spec-lint: 0 blocking, 0 notes"], - }); - }); - - it.each([ - [ - "a missing section", - swap("## Out of bounds\n\n- `src/daemon/server.ts`.\n\n", ""), - "B 100 body: `## Out of bounds` is missing; add it", - ], - [ - "an empty section", - swap("- `composeWorker` moves to `src/daemon/worker-composition.ts`.", ""), - "B 100 Target shape: empty; fill it in", - ], - [ - "sections out of order", - (body: string) => - body - .replace("## Why", "## Swap") - .replace("## Target shape", "## Why") - .replace("## Swap", "## Target shape"), - "B 100 body: `## Target shape` is out of template order; move it", - ], - ])("blocks %s", (_, change, expected) => { - const result = lintRefactor(change(REFACTOR)); - - expect(result.lines).toContain(expected); - expect(result.status).toBe(1); - }); - - it("reads a body with no refactor label as a refactor under --refactor", () => { - const result = lint(["--refactor", "--body-file", write("refactor.md", REFACTOR)]); - - expect(result).toEqual({ status: 0, lines: ["spec-lint: 0 blocking, 0 notes"] }); - }); -}); diff --git a/src/test-spec-lint.ts b/src/test-spec-lint.ts new file mode 100644 index 000000000..8e94cdb35 --- /dev/null +++ b/src/test-spec-lint.ts @@ -0,0 +1,100 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect } from "vitest"; + +export const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); + +export const SCRIPT = join(ROOT, ".agents", "scripts", "spec-lint.mjs"); + +export const SHA = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: ROOT, + encoding: "utf8", +}).trim(); + +export const FOOTER = "*Written by an agent.*"; + +export const FENCE = "```"; + +export const PARENT = `Request: none + +## Behaviours + +| Behaviour (invoker, action, what it sees) | Seam | Cases | Task | +| --- | --- | --- | --- | +| An operator's config is read: number or object accepted, bad value refused | \`loadConfig\` | \`2\`; \`{}\`; \`0\` refused | #100 | + +${FOOTER} +`; + +export function write(name: string, text: string): string { + const file = join(mkdtempSync(join(tmpdir(), "spec-lint-")), name); + writeFileSync(file, text); + return file; +} + +export function lint(args: string[]) { + try { + const stdout = execFileSync(process.execPath, [SCRIPT, ...args], { + cwd: ROOT, + encoding: "utf8", + }); + return { status: 0, lines: stdout.split("\n").filter((l) => l !== "") }; + } catch (error) { + const failed = error as { status: number; stdout: string }; + return { status: failed.status, lines: failed.stdout.split("\n").filter((l) => l !== "") }; + } +} + +export function lintTask(body: string, parent: string | null = PARENT) { + return lint([ + "100", + "--body-file", + write("task.md", body), + ...(parent === null ? [] : ["--parent-file", write("parent.md", parent)]), + "--world", + write("world.json", JSON.stringify({ 100: { parent: 50 } })), + ]); +} + +export const swap = (from: string, to: string) => (body: string) => { + expect(body).toContain(from); + return body.replace(from, to); +}; + +export const task = (line: string, depends: string[] = []) => `### Tests + +Seam line. + +- ${line} + +## Depends on + +${depends.map((d) => `- ${d}`).join("\n")} +`; + +export const WORLD = { + 200: { labels: ["feature:planned"], subIssues: [101, 102] }, + 101: { + body: task("A worker restarts and keeps the profile: same bytes, new port."), + labels: ["task:ready"], + }, + 102: { + body: task("A caller cancels a waiting request by platform: given, absent.", ["#101"]), + labels: ["task:ready"], + }, +}; + +export function lintFeature(body: string, world: Record = WORLD) { + return lint([ + "200", + "--feature", + "--body-file", + write("feature.md", body), + "--world", + write("world.json", JSON.stringify(world)), + ]); +} From 3aa59f0e39be7bc763076f10ec72437fcc8018af Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:27:52 +0100 Subject: [PATCH 68/83] test(simlock-client): split client.test.ts by topic --- src/simlock-client/client-admin.test.ts | 61 ++ src/simlock-client/client-pushes.test.ts | 528 ++++++++++ .../client-request-lease.test.ts | 315 ++++++ src/simlock-client/client.test.ts | 932 +----------------- src/simlock-client/test-client.ts | 39 + 5 files changed, 944 insertions(+), 931 deletions(-) create mode 100644 src/simlock-client/client-admin.test.ts create mode 100644 src/simlock-client/client-pushes.test.ts create mode 100644 src/simlock-client/client-request-lease.test.ts create mode 100644 src/simlock-client/test-client.ts diff --git a/src/simlock-client/client-admin.test.ts b/src/simlock-client/client-admin.test.ts new file mode 100644 index 000000000..05df5b075 --- /dev/null +++ b/src/simlock-client/client-admin.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it, vi } from "vitest"; +import { connectSimlockAdmin } from "../admin/index.js"; +import { completeHello, ScriptedConnection } from "./test-support.js"; +import { flushMicrotasks, sampleGrant } from "./test-client.js"; + +describe("simlock/admin", () => { + it("exposes the admin operations and sends daemon.stop like any other request", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const admin = await connectPromise; + expect(admin.role).toBe("admin"); + + const stopPromise = admin.stopDaemon(); + await flushMicrotasks(); + const stopCall = connection.lastSentOf("daemon.stop")!; + connection.reply(stopCall.id, { stopping: true }); + await expect(stopPromise).resolves.toEqual({ stopping: true }); + }); + + it("releaseAllLeases clears every held lease id and connection death afterwards fires no onLeaseLost", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const admin = await connectPromise; + + const firstGrant = admin.requestLease({ model: "iPhone 17", platform: "ios" }); + await flushMicrotasks(); + connection.reply( + connection.lastSentOf("lease.request")!.id, + sampleGrant({ leaseId: "lease_a" }), + ); + await firstGrant; + + const secondGrant = admin.requestLease({ model: "iPhone 17", platform: "ios" }); + await flushMicrotasks(); + connection.reply( + connection.lastSentOf("lease.request")!.id, + sampleGrant({ leaseId: "lease_b" }), + ); + await secondGrant; + + const releaseAllPromise = admin.releaseAllLeases(); + await flushMicrotasks(); + const releaseAllCall = connection.lastSentOf("lease.release-all")!; + connection.reply(releaseAllCall.id, { leaseIds: ["lease_a", "lease_b"] }); + await expect(releaseAllPromise).resolves.toMatchObject({ + leaseIds: ["lease_a", "lease_b"], + }); + + // Both leases are no longer tracked as held -- a later connection death synthesizes no + // onLeaseLost for either of them. + const leaseLost = vi.fn(); + admin.onLeaseLost(leaseLost); + connection.simulateDeath(); + await flushMicrotasks(); + expect(leaseLost).not.toHaveBeenCalled(); + }); +}); diff --git a/src/simlock-client/client-pushes.test.ts b/src/simlock-client/client-pushes.test.ts new file mode 100644 index 000000000..6783a81c0 --- /dev/null +++ b/src/simlock-client/client-pushes.test.ts @@ -0,0 +1,528 @@ +import { describe, expect, it, vi } from "vitest"; +import { connectSimlockAdmin } from "../admin/index.js"; +import { connectSimlock } from "../client/index.js"; +import { completeHello, ScriptedConnection } from "./test-support.js"; +import { flushMicrotasks, sampleGrant } from "./test-client.js"; + +describe("pushes", () => { + it("routes progress to the originating call and drops it once the call is no longer tracked", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const onProgress = vi.fn(); + const grantPromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { onProgress }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + + // A push can arrive before the response to the request that caused it (ADR §8). + connection.push("progress", { + progress: { etaMs: 500, stage: "provisioning" }, + requestId: requestCall.id, + }); + connection.reply(requestCall.id, sampleGrant()); + await grantPromise; + + expect(onProgress).toHaveBeenCalledWith({ etaMs: 500, stage: "provisioning" }); + + // Dropped: no call is tracked under this id any more. + connection.push("progress", { + progress: { etaMs: 1, stage: "booting" }, + requestId: requestCall.id, + }); + expect(onProgress).toHaveBeenCalledTimes(1); + }); + + it("routes component-progress pushes to that installComponent call's onProgress, and stops at its reply", async () => { + // ADR 0010 §6: `component-progress` is request-scoped like `progress`. + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const onProgress = vi.fn(); + const installPromise = client.installComponent( + { platform: "android", version: "35" }, + { onProgress }, + ); + await flushMicrotasks(); + const installCall = connection.lastSentOf("component.install")!; + expect(installCall.payload).toEqual({ platform: "android", version: "35" }); + + connection.push("component-progress", { + progress: { stage: "waiting" }, + requestId: installCall.id, + }); + connection.push("component-progress", { + progress: { fraction: 0.41, stage: "downloading" }, + requestId: installCall.id, + }); + const result = { component: "35", outcome: "installed", platform: "android", version: "35" }; + connection.reply(installCall.id, result); + + await expect(installPromise).resolves.toEqual(result); + expect(onProgress.mock.calls.map(([progress]) => progress)).toEqual([ + { stage: "waiting" }, + { fraction: 0.41, stage: "downloading" }, + ]); + connection.push("component-progress", { + progress: { stage: "downloading" }, + requestId: installCall.id, + }); + expect(onProgress).toHaveBeenCalledTimes(2); + }); + + it("hands installComponentOnWorkers each relayed update with its workerId, and drops one that names no worker", async () => { + // ADR 0010 §7: a gateway sets `workerId` on every `component-progress` push it relays. + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const onProgress = vi.fn(); + const installPromise = client.installComponentOnWorkers( + { platform: "android", version: "35", workers: "all" }, + { onProgress }, + ); + await flushMicrotasks(); + const call = connection.lastSentOf("worker.install-component")!; + expect(call.payload).toEqual({ platform: "android", version: "35", workers: "all" }); + + connection.push("component-progress", { + progress: { stage: "waiting" }, + requestId: call.id, + workerId: "wrk_a", + }); + connection.push("component-progress", { progress: { stage: "waiting" }, requestId: call.id }); + connection.push("component-progress", { + progress: { fraction: 0.5, stage: "downloading" }, + requestId: call.id, + workerId: "wrk_b", + }); + const result = { results: [{ outcome: "installed", version: "35", workerId: "wrk_a" }] }; + connection.reply(call.id, result); + + await expect(installPromise).resolves.toEqual(result); + expect(onProgress.mock.calls.map(([progress]) => progress)).toEqual([ + { stage: "waiting", workerId: "wrk_a" }, + { fraction: 0.5, stage: "downloading", workerId: "wrk_b" }, + ]); + }); + + it("routes device.exec output chunks to that call's onOutput, in order, and stops at its reply", async () => { + // ADR 0005 §19a: `output` is request-scoped like `progress`, so it reaches the call that is + // still waiting on its frame id and nothing else. + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const onOutput = vi.fn(); + const execPromise = client.exec( + { args: ["list", "devices"], leaseId: "lease_1", tool: "simctl" }, + { onOutput }, + ); + await flushMicrotasks(); + const execCall = connection.lastSentOf("device.exec")!; + expect(execCall.payload).toEqual({ + args: ["list", "devices"], + leaseId: "lease_1", + tool: "simctl", + }); + + connection.push("output", { chunk: "one", requestId: execCall.id, stream: "stdout" }); + connection.push("output", { chunk: "two", requestId: execCall.id, stream: "stderr" }); + connection.reply(execCall.id, { exitCode: 7 }); + + await expect(execPromise).resolves.toEqual({ exitCode: 7 }); + expect(onOutput.mock.calls.map(([chunk]) => chunk)).toEqual([ + { chunk: "one", stream: "stdout" }, + { chunk: "two", stream: "stderr" }, + ]); + + // The call is no longer tracked, so a late chunk is dropped rather than delivered to a + // caller that has already been told the command finished. + connection.push("output", { chunk: "late", requestId: execCall.id, stream: "stdout" }); + expect(onOutput).toHaveBeenCalledTimes(2); + }); + + it("rotate sends device.rotate for the lease and answers its output, or throws the daemon's typed refusal", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const rotated = client.rotate("lease_1", "landscape_left"); + await flushMicrotasks(); + expect(connection.sent.map((frame) => frame.type)).toContain("device.rotate"); + const call = connection.lastSentOf("device.rotate")!; + expect(call.payload).toEqual({ leaseId: "lease_1", orientation: "landscape_left" }); + connection.reply(call.id, { leaseId: "lease_1", orientation: "landscape_left" }); + await expect(rotated).resolves.toEqual({ leaseId: "lease_1", orientation: "landscape_left" }); + + const refused = client.rotate("lease_2", "portrait", { requesterId: "agent-1" }); + await flushMicrotasks(); + const refusedCall = connection.lastSentOf("device.rotate")!; + expect(refusedCall.payload).toEqual({ + leaseId: "lease_2", + orientation: "portrait", + requesterId: "agent-1", + }); + const details = { class: "tv", leaseId: "lease_2", physical: false, platform: "ios" }; + connection.fail( + refusedCall.id, + "ROTATION_UNSUPPORTED", + "Lease lease_2 is on an Apple TV simulator; only an iOS simulator can be rotated", + details, + ); + await expect(refused).rejects.toMatchObject({ + code: "ROTATION_UNSUPPORTED", + details, + kind: "domain", + }); + }); + + it("routes a device.exec started push to that call's onStarted, once, before output, and not to another call in flight", async () => { + // ADR 0005 §19a: `started` is request-scoped like `output` -- the fact a transport chooses + // its response shape on. Round 5 review: the send site and this route could both be deleted + // with the whole suite green, because every test of this frame stopped at the coordinator's + // own callback. This one exercises the wire. + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const first = { onOutput: vi.fn(), onStarted: vi.fn() }; + const second = { onOutput: vi.fn(), onStarted: vi.fn() }; + const firstPromise = client.exec({ args: ["list"], leaseId: "lease_1", tool: "simctl" }, first); + const secondPromise = client.exec( + { args: ["devices"], leaseId: "lease_2", tool: "adb" }, + second, + ); + await flushMicrotasks(); + const calls = connection.sent.filter((frame) => frame.type === "device.exec"); + expect(calls).toHaveLength(2); + const [firstCall, secondCall] = calls as [(typeof calls)[0], (typeof calls)[0]]; + + connection.push("started", { requestId: firstCall.id }); + connection.push("output", { chunk: "one", requestId: firstCall.id, stream: "stdout" }); + + // Only the call it names, and before that call's own output. + expect(first.onStarted).toHaveBeenCalledTimes(1); + expect(second.onStarted).not.toHaveBeenCalled(); + expect(first.onStarted.mock.invocationCallOrder[0]).toBeLessThan( + first.onOutput.mock.invocationCallOrder[0] as number, + ); + + connection.reply(firstCall.id, { exitCode: 0 }); + connection.reply(secondCall.id, { exitCode: 0 }); + await expect(firstPromise).resolves.toEqual({ exitCode: 0 }); + await expect(secondPromise).resolves.toEqual({ exitCode: 0 }); + + // Settled calls are no longer tracked, so a late frame is dropped rather than delivered. + connection.push("started", { requestId: firstCall.id }); + expect(first.onStarted).toHaveBeenCalledTimes(1); + }); + + it("de-duplicates a repeated device-unhealthy push for the same lease (same state twice in a row)", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const unhealthy = vi.fn(); + client.onDeviceUnhealthy(unhealthy); + + connection.push("device-unhealthy", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "crashed", + }); + connection.push("device-unhealthy", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "crashed", + }); + + expect(unhealthy).toHaveBeenCalledTimes(1); + }); + + it("delivers a second device-unhealthy for the same lease once a device-recovered came between the two (S2: a second crash is not a duplicate)", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const unhealthy = vi.fn(); + const recovered = vi.fn(); + client.onDeviceUnhealthy(unhealthy); + client.onDeviceRecovered(recovered); + + connection.push("device-unhealthy", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "crashed", + }); + connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); + // A genuine second crash on the same lease -- `LeaseHealthMonitor` clears its recovery + // bookkeeping after a successful recovery, so this is a real, distinct fact, not a + // redundant re-push of the first one. + connection.push("device-unhealthy", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "crashed", + }); + + expect(unhealthy).toHaveBeenCalledTimes(2); + expect(recovered).toHaveBeenCalledTimes(1); + }); + + it("still de-duplicates a repeated device-recovered push for the same lease", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const recovered = vi.fn(); + client.onDeviceRecovered(recovered); + + connection.push("device-unhealthy", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "crashed", + }); + connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); + // The owner-routed fan-out re-delivering the same recovered fact must still be suppressed. + connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); + + expect(recovered).toHaveBeenCalledTimes(1); + }); + + it("de-duplicates a repeated lease-lost push for the same lease (permanent -- loss is terminal)", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const leaseLost = vi.fn(); + client.onLeaseLost(leaseLost); + + connection.push("lease-lost", { deviceId: "device_1", leaseId: "lease_1", reason: "crashed" }); + connection.push("lease-lost", { deviceId: "device_1", leaseId: "lease_1", reason: "crashed" }); + + expect(leaseLost).toHaveBeenCalledTimes(1); + }); + + it("a successful release never fires onLeaseLost, because the daemon suppresses the push at the source (ADR §8)", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const leaseLost = vi.fn(); + client.onLeaseLost(leaseLost); + + const grantPromise = client.requestLease({ model: "iPhone 17", platform: "ios" }); + await flushMicrotasks(); + connection.reply( + connection.lastSentOf("lease.request")!.id, + sampleGrant({ leaseId: "lease_1" }), + ); + await grantPromise; + + const releasePromise = client.releaseLease({ leaseId: "lease_1" }); + await flushMicrotasks(); + const releaseCall = connection.lastSentOf("lease.release")!; + // Unlike the real daemon (which never sends this push to the releasing connection), the + // scripted connection sends nothing here either -- this asserts the client does not + // itself need to remember and swallow a self-initiated release. + connection.reply(releaseCall.id, { leaseId: "lease_1" }); + await releasePromise; + + expect(leaseLost).not.toHaveBeenCalled(); + }); + + it("S3: a lease-lost push for a lease id whose earlier release attempt failed is still delivered, not permanently muted", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const leaseLost = vi.fn(); + client.onLeaseLost(leaseLost); + + const grantPromise = client.requestLease({ model: "iPhone 17", platform: "ios" }); + await flushMicrotasks(); + connection.reply( + connection.lastSentOf("lease.request")!.id, + sampleGrant({ leaseId: "lease_1" }), + ); + await grantPromise; + + // The release itself fails -- e.g. an agent-role client hitting FORBIDDEN. + const releasePromise = client.releaseLease({ leaseId: "lease_1" }); + await flushMicrotasks(); + const releaseCall = connection.lastSentOf("lease.release")!; + connection.fail(releaseCall.id, "FORBIDDEN", "not authorized"); + await expect(releasePromise).rejects.toMatchObject({ code: "FORBIDDEN" }); + + // The lease later genuinely expires/crashes -- this is a real loss the client must + // still surface, not residue from the earlier failed release attempt. + connection.push("lease-lost", { + deviceId: "device_1", + leaseId: "lease_1", + reason: "expired", + }); + + expect(leaseLost).toHaveBeenCalledWith({ + deviceId: "device_1", + leaseId: "lease_1", + reason: "expired", + }); + }); + + it("ignores an unknown push kind such as a protocol-3 daemon's lease.heartbeat", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + // Not reachable against a protocol-4 daemon (the ranges do not overlap, so `hello` fails + // first), but the wire must not answer a push it no longer understands. + connection.push("lease.heartbeat", { nonce: 12345 }); + await flushMicrotasks(); + + expect(connection.lastSentOf("lease.heartbeat")).toBeUndefined(); + await client.close(); + }); + + it("silently drops malformed lease-scoped and event pushes instead of throwing", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const leaseLost = vi.fn(); + const unhealthy = vi.fn(); + const recovered = vi.fn(); + client.onLeaseLost(leaseLost); + client.onDeviceUnhealthy(unhealthy); + client.onDeviceRecovered(recovered); + + expect(() => connection.push("lease-lost", { leaseId: "lease_1" })).not.toThrow(); + expect(() => connection.push("device-unhealthy", { leaseId: "lease_1" })).not.toThrow(); + expect(() => + connection.push("device-recovered", { deviceId: "device_1", leaseId: "lease_1" }), + ).not.toThrow(); + expect(() => connection.push("event", { not: "a valid event shape" })).not.toThrow(); + + expect(leaseLost).not.toHaveBeenCalled(); + expect(unhealthy).not.toHaveBeenCalled(); + expect(recovered).not.toHaveBeenCalled(); + }); +}); + +describe("event pushes", () => { + it("a worker push with a malformed id or timestamp never reaches the gateway's bus", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const listener = vi.fn(); + const subscribed = client.subscribeEvents(listener); + await flushMicrotasks(); + const subscribe = connection.lastSentOf("events.subscribe"); + if (subscribe === undefined) throw new Error("the client never sent events.subscribe"); + connection.reply(subscribe.id, { subscribed: true, subscriptionId: "sub_1" }); + await subscribed; + const valid = { + event: "lease.expired", + id: "evt_ok", + module: "leases", + payload: {}, + seq: 1, + timestamp: 5, + }; + + for (const bad of [ + { ...valid, id: "no-prefix" }, + { ...valid, id: `evt_${"a".repeat(65)}` }, + { ...valid, id: 7 }, + { ...valid, timestamp: null }, + { ...valid, timestamp: "5" }, + ]) { + connection.push("event", { subscriptionId: "sub_1", event: bad }); + } + // `1e999` is valid JSON that parses to Infinity, a value JSON.stringify cannot emit; the + // same raw channel carries the well-formed twin, which proves the line is delivered at all. + const rawPush = (timestamp: string) => + JSON.stringify({ + payload: { subscriptionId: "sub_1", event: { ...valid, timestamp: "__timestamp__" } }, + push: "event", + }).replace('"__timestamp__"', timestamp); + connection.receiveLine(rawPush("1e999")); + await flushMicrotasks(); + expect(listener).not.toHaveBeenCalled(); + + connection.receiveLine(rawPush("5")); + await flushMicrotasks(); + expect(listener).toHaveBeenCalledTimes(1); + await client.close(); + }); + + it("a worker push with a timestamp outside the date range never reaches the gateway's bus", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const listener = vi.fn(); + const subscribed = client.subscribeEvents(listener); + await flushMicrotasks(); + const subscribe = connection.lastSentOf("events.subscribe"); + if (subscribe === undefined) throw new Error("the client never sent events.subscribe"); + connection.reply(subscribe.id, { subscribed: true, subscriptionId: "sub_1" }); + await subscribed; + const push = (timestamp: number) => + connection.push("event", { + subscriptionId: "sub_1", + event: { + event: "lease.expired", + id: "evt_ok", + module: "leases", + payload: {}, + seq: 1, + timestamp, + }, + }); + + for (const outside of [8.64e15 + 2, -8.64e15 - 2, 1e300, -1e300]) push(outside); + await flushMicrotasks(); + expect(listener).not.toHaveBeenCalled(); + + push(8.64e15); + push(-8.64e15); + await flushMicrotasks(); + expect(listener).toHaveBeenCalledTimes(2); + await client.close(); + }); +}); diff --git a/src/simlock-client/client-request-lease.test.ts b/src/simlock-client/client-request-lease.test.ts new file mode 100644 index 000000000..21f57f9e6 --- /dev/null +++ b/src/simlock-client/client-request-lease.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it, vi } from "vitest"; +import { connectSimlock } from "../client/index.js"; +import { completeHello, ScriptedConnection } from "./test-support.js"; +import { flushMicrotasks, sampleGrant } from "./test-client.js"; + +describe("requestLease abort (ADR §10)", () => { + it("before the request is sent: rejects CANCELLED, nothing sent", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const controller = new AbortController(); + controller.abort(); + const before = connection.sent.length; + + await expect( + client.requestLease({ model: "iPhone 17", platform: "ios" }, { signal: controller.signal }), + ).rejects.toMatchObject({ code: "CANCELLED" }); + expect(connection.sent).toHaveLength(before); + }); + + it("while queued: sends lease.cancel, waits for the original to reject, surfaces CANCELLED", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + + controller.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + expect(cancelCall).toBeDefined(); + + connection.reply(cancelCall.id, { result: "cancelled" }); + await flushMicrotasks(); + connection.fail(requestCall.id, "QUEUE_TIMEOUT", "cancelled by request", { requestId: "r1" }); + + await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); + }); + + it("aborting one of two waiting requests of different platforms sends lease.cancel naming that request's platform and leaves the other waiting", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const iosController = new AbortController(); + const androidController = new AbortController(); + const iosLease = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: iosController.signal }, + ); + const androidLease = client.requestLease( + { model: "Pixel 9", platform: "android" }, + { signal: androidController.signal }, + ); + await flushMicrotasks(); + const requestOf = (platform: string) => + connection.sent + .filter((frame) => frame.type === "lease.request") + .find((frame) => (frame.payload as { platform: string }).platform === platform)!; + const androidCall = requestOf("android"); + + androidController.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + connection.reply(cancelCall.id, { result: "cancelled" }); + await flushMicrotasks(); + connection.fail(androidCall.id, "QUEUE_TIMEOUT", "cancelled by request", { requestId: "r2" }); + + await expect(androidLease).rejects.toMatchObject({ code: "CANCELLED" }); + expect(cancelCall.payload).toMatchObject({ platform: "android" }); + expect(connection.sent.filter((frame) => frame.type === "lease.cancel")).toHaveLength(1); + expect(requestOf("ios")).toBeDefined(); + const iosState = await Promise.race([iosLease.catch((e: unknown) => e), "waiting"]); + expect(iosState).toBe("waiting"); + }); + + it("while device work is in flight: releases a grant that still arrives, then rejects CANCELLED", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + + controller.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + connection.reply(cancelCall.id, { result: "not-cancellable" }); + await flushMicrotasks(); + + connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_abandoned" })); + await flushMicrotasks(); + const releaseCall = connection.lastSentOf("lease.release")!; + expect(releaseCall).toBeDefined(); + expect((releaseCall.payload as { leaseId: string }).leaseId).toBe("lease_abandoned"); + connection.reply(releaseCall.id, { leaseId: "lease_abandoned" }); + + await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); + }); + + it("after the grant resolved: abort is ignored", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_kept" })); + + const grant = await leasePromise; + expect(grant.lease.id).toBe("lease_kept"); + + const before = connection.sent.length; + controller.abort(); + await flushMicrotasks(); + + // No lease.cancel was ever sent for an already-resolved request. + expect(connection.sent).toHaveLength(before); + }); +}); + +describe("principal resolution (ADR §4, B3)", () => { + it('adopts the daemon-resolved principal from hello\'s reply, never defaulting to ""', async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + const hello = connection.lastSentOf("hello")!; + // No principal was supplied -- the daemon resolves its own default (a pid, in practice). + expect((hello.payload as { principal?: string }).principal).toBeUndefined(); + connection.reply(hello.id, { + daemonProtocolRange: { max: 3, min: 3 }, + principal: "48213", + protocolVersion: 3, + role: "agent", + version: "0.3.0", + }); + const client = await connectPromise; + + expect(client.principal).toBe("48213"); + }); +}); + +describe("requestLease abort: ADR §4 identity cases (B3)", () => { + it("no principal supplied: abort while queued cancels with the daemon-resolved principal, and leaves no lease held", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + const hello = connection.lastSentOf("hello")!; + connection.reply(hello.id, { + daemonProtocolRange: { max: 3, min: 3 }, + principal: "48213", + protocolVersion: 3, + role: "agent", + version: "0.3.0", + }); + const client = await connectPromise; + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + + controller.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + // Before this fix, this was always "" -- the daemon fixes the connection's principal to + // its own default, and the client had no way to learn it. Sent as the resolved principal + // now, never "". + expect((cancelCall.payload as { requesterId?: string }).requesterId).toBe("48213"); + + connection.reply(cancelCall.id, { result: "cancelled" }); + await flushMicrotasks(); + connection.fail(requestCall.id, "QUEUE_TIMEOUT", "cancelled by request"); + + await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); + + // No lease is held: connection death fires no onLeaseLost for anything. + const leaseLost = vi.fn(); + client.onLeaseLost(leaseLost); + connection.simulateDeath(); + await flushMicrotasks(); + expect(leaseLost).not.toHaveBeenCalled(); + }); + + it('ADR §4 proxy case (principal "host", requesterId "agent-7"): abort in flight releases the grant and leaves no lease held', async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection, principal: "host" }); + await flushMicrotasks(); + completeHello(connection, { principal: "host" }); + const client = await connectPromise; + expect(client.principal).toBe("host"); + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios", requesterId: "agent-7" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + const requestCall = connection.lastSentOf("lease.request")!; + + controller.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + // Before this fix, the daemon's `authorize` hook compared this `requesterId` straight to + // the principal and rejected FORBIDDEN outright -- exactly ADR §4's proxy case. The + // request is still sent with the proxied requesterId (attribution); the daemon now + // authorizes the cancel against the pending request's recorded *owner* instead. + expect((cancelCall.payload as { requesterId?: string }).requesterId).toBe("agent-7"); + connection.reply(cancelCall.id, { result: "not-cancellable" }); + await flushMicrotasks(); + + connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_proxy_abandoned" })); + await flushMicrotasks(); + const releaseCall = connection.lastSentOf("lease.release")!; + expect((releaseCall.payload as { leaseId: string }).leaseId).toBe("lease_proxy_abandoned"); + connection.reply(releaseCall.id, { leaseId: "lease_proxy_abandoned" }); + + await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); + + // Already released via the abandoned-grant path -- not tracked as held any more. + const leaseLost = vi.fn(); + client.onLeaseLost(leaseLost); + connection.simulateDeath(); + await flushMicrotasks(); + expect(leaseLost).not.toHaveBeenCalled(); + }); + + it("a FORBIDDEN from the abort's lease.cancel surfaces instead of being read as a dead connection", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const controller = new AbortController(); + const leasePromise = client.requestLease( + { model: "iPhone 17", platform: "ios" }, + { signal: controller.signal }, + ); + await flushMicrotasks(); + + controller.abort(); + await flushMicrotasks(); + const cancelCall = connection.lastSentOf("lease.cancel")!; + connection.fail(cancelCall.id, "FORBIDDEN", "not authorized"); + + await expect(leasePromise).rejects.toMatchObject({ code: "FORBIDDEN" }); + // Not misread as the connection dying -- it's still alive and usable for another call. + expect(connection.closed).toBe(false); + const anotherCall = client.cancelLease(); + await flushMicrotasks(); + const anotherCancelCall = connection.lastSentOf("lease.cancel")!; + expect(anotherCancelCall.id).not.toBe(cancelCall.id); + connection.reply(anotherCancelCall.id, { result: "not-found" }); + await expect(anotherCall).resolves.toMatchObject({ result: "not-found" }); + }); +}); + +describe("requestLease requires", () => { + it("sends requires on the lease.request frame and rejects with CAPABILITY_MISSING, its details naming the capabilities, when the worker lacks the capability", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlock({ connection }); + await flushMicrotasks(); + completeHello(connection); + const client = await connectPromise; + + const pending = client.requestLease({ + model: "iPhone 17", + platform: "ios", + requires: ["agent-device"], + }); + await flushMicrotasks(); + const call = connection.lastSentOf("lease.request")!; + connection.fail(call.id, "CAPABILITY_MISSING", "This worker cannot serve agent-device", { + capabilities: ["agent-device"], + }); + + expect(call.payload).toMatchObject({ requires: ["agent-device"] }); + await expect(pending).rejects.toMatchObject({ + code: "CAPABILITY_MISSING", + details: { capabilities: ["agent-device"] }, + kind: "domain", + }); + }); +}); diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index 826d3ecc2..51eb1f3e7 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -1,49 +1,10 @@ import { describe, expect, it, vi } from "vitest"; - import { connectSimlockAdmin } from "../admin/index.js"; import { connectSimlock } from "../client/index.js"; import type { SimlockAdminClient, SimlockClient } from "./client.js"; -import type { LeaseGrant } from "./types.js"; import { MAX_WIRE_LINE_CHARS } from "./wire.js"; import { completeHello, ScriptedConnection } from "./test-support.js"; - -async function flushMicrotasks(times = 10): Promise { - for (let index = 0; index < times; index += 1) await Promise.resolve(); -} - -function sampleGrant( - overrides: { readonly leaseId?: string; readonly deviceId?: string } = {}, -): LeaseGrant { - const leaseId = overrides.leaseId ?? "lease_1"; - const deviceId = overrides.deviceId ?? "device_1"; - return { - device: { - physical: false, - driverDeviceId: "sim-1", - id: deviceId, - spec: { physical: false, model: "iPhone 17", osVersion: "18.0", platform: "ios" }, - mode: "full", - }, - environment: {}, - lease: { - deviceId, - grantedAt: 0, - id: leaseId, - lastRenewedAt: 0, - idChosenByRequester: false, - ownerId: "agent-1", - requesterId: "agent-1", - ttlMs: 1_000, - ttlDeadline: 1_000, - }, - timing: { - estimatedBootMs: 1, - estimatedProvisionMs: 1, - estimatedReadyMs: 1, - estimatedReclaimMs: 1, - }, - }; -} +import { flushMicrotasks, sampleGrant } from "./test-client.js"; describe("connectSimlock: handshake", () => { it("a version mismatch leaves the connection open and usable for stopDaemon() only (ADR §6)", async () => { @@ -579,894 +540,3 @@ describe("wire framing", () => { await expect(pending).rejects.toMatchObject({ code: "DAEMON_CONNECTION_LOST" }); }); }); - -describe("pushes", () => { - it("routes progress to the originating call and drops it once the call is no longer tracked", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const onProgress = vi.fn(); - const grantPromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { onProgress }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - - // A push can arrive before the response to the request that caused it (ADR §8). - connection.push("progress", { - progress: { etaMs: 500, stage: "provisioning" }, - requestId: requestCall.id, - }); - connection.reply(requestCall.id, sampleGrant()); - await grantPromise; - - expect(onProgress).toHaveBeenCalledWith({ etaMs: 500, stage: "provisioning" }); - - // Dropped: no call is tracked under this id any more. - connection.push("progress", { - progress: { etaMs: 1, stage: "booting" }, - requestId: requestCall.id, - }); - expect(onProgress).toHaveBeenCalledTimes(1); - }); - - it("routes component-progress pushes to that installComponent call's onProgress, and stops at its reply", async () => { - // ADR 0010 §6: `component-progress` is request-scoped like `progress`. - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const onProgress = vi.fn(); - const installPromise = client.installComponent( - { platform: "android", version: "35" }, - { onProgress }, - ); - await flushMicrotasks(); - const installCall = connection.lastSentOf("component.install")!; - expect(installCall.payload).toEqual({ platform: "android", version: "35" }); - - connection.push("component-progress", { - progress: { stage: "waiting" }, - requestId: installCall.id, - }); - connection.push("component-progress", { - progress: { fraction: 0.41, stage: "downloading" }, - requestId: installCall.id, - }); - const result = { component: "35", outcome: "installed", platform: "android", version: "35" }; - connection.reply(installCall.id, result); - - await expect(installPromise).resolves.toEqual(result); - expect(onProgress.mock.calls.map(([progress]) => progress)).toEqual([ - { stage: "waiting" }, - { fraction: 0.41, stage: "downloading" }, - ]); - connection.push("component-progress", { - progress: { stage: "downloading" }, - requestId: installCall.id, - }); - expect(onProgress).toHaveBeenCalledTimes(2); - }); - - it("hands installComponentOnWorkers each relayed update with its workerId, and drops one that names no worker", async () => { - // ADR 0010 §7: a gateway sets `workerId` on every `component-progress` push it relays. - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const onProgress = vi.fn(); - const installPromise = client.installComponentOnWorkers( - { platform: "android", version: "35", workers: "all" }, - { onProgress }, - ); - await flushMicrotasks(); - const call = connection.lastSentOf("worker.install-component")!; - expect(call.payload).toEqual({ platform: "android", version: "35", workers: "all" }); - - connection.push("component-progress", { - progress: { stage: "waiting" }, - requestId: call.id, - workerId: "wrk_a", - }); - connection.push("component-progress", { progress: { stage: "waiting" }, requestId: call.id }); - connection.push("component-progress", { - progress: { fraction: 0.5, stage: "downloading" }, - requestId: call.id, - workerId: "wrk_b", - }); - const result = { results: [{ outcome: "installed", version: "35", workerId: "wrk_a" }] }; - connection.reply(call.id, result); - - await expect(installPromise).resolves.toEqual(result); - expect(onProgress.mock.calls.map(([progress]) => progress)).toEqual([ - { stage: "waiting", workerId: "wrk_a" }, - { fraction: 0.5, stage: "downloading", workerId: "wrk_b" }, - ]); - }); - - it("routes device.exec output chunks to that call's onOutput, in order, and stops at its reply", async () => { - // ADR 0005 §19a: `output` is request-scoped like `progress`, so it reaches the call that is - // still waiting on its frame id and nothing else. - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const onOutput = vi.fn(); - const execPromise = client.exec( - { args: ["list", "devices"], leaseId: "lease_1", tool: "simctl" }, - { onOutput }, - ); - await flushMicrotasks(); - const execCall = connection.lastSentOf("device.exec")!; - expect(execCall.payload).toEqual({ - args: ["list", "devices"], - leaseId: "lease_1", - tool: "simctl", - }); - - connection.push("output", { chunk: "one", requestId: execCall.id, stream: "stdout" }); - connection.push("output", { chunk: "two", requestId: execCall.id, stream: "stderr" }); - connection.reply(execCall.id, { exitCode: 7 }); - - await expect(execPromise).resolves.toEqual({ exitCode: 7 }); - expect(onOutput.mock.calls.map(([chunk]) => chunk)).toEqual([ - { chunk: "one", stream: "stdout" }, - { chunk: "two", stream: "stderr" }, - ]); - - // The call is no longer tracked, so a late chunk is dropped rather than delivered to a - // caller that has already been told the command finished. - connection.push("output", { chunk: "late", requestId: execCall.id, stream: "stdout" }); - expect(onOutput).toHaveBeenCalledTimes(2); - }); - - it("rotate sends device.rotate for the lease and answers its output, or throws the daemon's typed refusal", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const rotated = client.rotate("lease_1", "landscape_left"); - await flushMicrotasks(); - expect(connection.sent.map((frame) => frame.type)).toContain("device.rotate"); - const call = connection.lastSentOf("device.rotate")!; - expect(call.payload).toEqual({ leaseId: "lease_1", orientation: "landscape_left" }); - connection.reply(call.id, { leaseId: "lease_1", orientation: "landscape_left" }); - await expect(rotated).resolves.toEqual({ leaseId: "lease_1", orientation: "landscape_left" }); - - const refused = client.rotate("lease_2", "portrait", { requesterId: "agent-1" }); - await flushMicrotasks(); - const refusedCall = connection.lastSentOf("device.rotate")!; - expect(refusedCall.payload).toEqual({ - leaseId: "lease_2", - orientation: "portrait", - requesterId: "agent-1", - }); - const details = { class: "tv", leaseId: "lease_2", physical: false, platform: "ios" }; - connection.fail( - refusedCall.id, - "ROTATION_UNSUPPORTED", - "Lease lease_2 is on an Apple TV simulator; only an iOS simulator can be rotated", - details, - ); - await expect(refused).rejects.toMatchObject({ - code: "ROTATION_UNSUPPORTED", - details, - kind: "domain", - }); - }); - - it("routes a device.exec started push to that call's onStarted, once, before output, and not to another call in flight", async () => { - // ADR 0005 §19a: `started` is request-scoped like `output` -- the fact a transport chooses - // its response shape on. Round 5 review: the send site and this route could both be deleted - // with the whole suite green, because every test of this frame stopped at the coordinator's - // own callback. This one exercises the wire. - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const first = { onOutput: vi.fn(), onStarted: vi.fn() }; - const second = { onOutput: vi.fn(), onStarted: vi.fn() }; - const firstPromise = client.exec({ args: ["list"], leaseId: "lease_1", tool: "simctl" }, first); - const secondPromise = client.exec( - { args: ["devices"], leaseId: "lease_2", tool: "adb" }, - second, - ); - await flushMicrotasks(); - const calls = connection.sent.filter((frame) => frame.type === "device.exec"); - expect(calls).toHaveLength(2); - const [firstCall, secondCall] = calls as [(typeof calls)[0], (typeof calls)[0]]; - - connection.push("started", { requestId: firstCall.id }); - connection.push("output", { chunk: "one", requestId: firstCall.id, stream: "stdout" }); - - // Only the call it names, and before that call's own output. - expect(first.onStarted).toHaveBeenCalledTimes(1); - expect(second.onStarted).not.toHaveBeenCalled(); - expect(first.onStarted.mock.invocationCallOrder[0]).toBeLessThan( - first.onOutput.mock.invocationCallOrder[0] as number, - ); - - connection.reply(firstCall.id, { exitCode: 0 }); - connection.reply(secondCall.id, { exitCode: 0 }); - await expect(firstPromise).resolves.toEqual({ exitCode: 0 }); - await expect(secondPromise).resolves.toEqual({ exitCode: 0 }); - - // Settled calls are no longer tracked, so a late frame is dropped rather than delivered. - connection.push("started", { requestId: firstCall.id }); - expect(first.onStarted).toHaveBeenCalledTimes(1); - }); - - it("de-duplicates a repeated device-unhealthy push for the same lease (same state twice in a row)", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const unhealthy = vi.fn(); - client.onDeviceUnhealthy(unhealthy); - - connection.push("device-unhealthy", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "crashed", - }); - connection.push("device-unhealthy", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "crashed", - }); - - expect(unhealthy).toHaveBeenCalledTimes(1); - }); - - it("delivers a second device-unhealthy for the same lease once a device-recovered came between the two (S2: a second crash is not a duplicate)", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const unhealthy = vi.fn(); - const recovered = vi.fn(); - client.onDeviceUnhealthy(unhealthy); - client.onDeviceRecovered(recovered); - - connection.push("device-unhealthy", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "crashed", - }); - connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); - // A genuine second crash on the same lease -- `LeaseHealthMonitor` clears its recovery - // bookkeeping after a successful recovery, so this is a real, distinct fact, not a - // redundant re-push of the first one. - connection.push("device-unhealthy", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "crashed", - }); - - expect(unhealthy).toHaveBeenCalledTimes(2); - expect(recovered).toHaveBeenCalledTimes(1); - }); - - it("still de-duplicates a repeated device-recovered push for the same lease", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const recovered = vi.fn(); - client.onDeviceRecovered(recovered); - - connection.push("device-unhealthy", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "crashed", - }); - connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); - // The owner-routed fan-out re-delivering the same recovered fact must still be suppressed. - connection.push("device-recovered", { attempts: 1, deviceId: "device_1", leaseId: "lease_1" }); - - expect(recovered).toHaveBeenCalledTimes(1); - }); - - it("de-duplicates a repeated lease-lost push for the same lease (permanent -- loss is terminal)", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const leaseLost = vi.fn(); - client.onLeaseLost(leaseLost); - - connection.push("lease-lost", { deviceId: "device_1", leaseId: "lease_1", reason: "crashed" }); - connection.push("lease-lost", { deviceId: "device_1", leaseId: "lease_1", reason: "crashed" }); - - expect(leaseLost).toHaveBeenCalledTimes(1); - }); - - it("a successful release never fires onLeaseLost, because the daemon suppresses the push at the source (ADR §8)", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const leaseLost = vi.fn(); - client.onLeaseLost(leaseLost); - - const grantPromise = client.requestLease({ model: "iPhone 17", platform: "ios" }); - await flushMicrotasks(); - connection.reply( - connection.lastSentOf("lease.request")!.id, - sampleGrant({ leaseId: "lease_1" }), - ); - await grantPromise; - - const releasePromise = client.releaseLease({ leaseId: "lease_1" }); - await flushMicrotasks(); - const releaseCall = connection.lastSentOf("lease.release")!; - // Unlike the real daemon (which never sends this push to the releasing connection), the - // scripted connection sends nothing here either -- this asserts the client does not - // itself need to remember and swallow a self-initiated release. - connection.reply(releaseCall.id, { leaseId: "lease_1" }); - await releasePromise; - - expect(leaseLost).not.toHaveBeenCalled(); - }); - - it("S3: a lease-lost push for a lease id whose earlier release attempt failed is still delivered, not permanently muted", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const leaseLost = vi.fn(); - client.onLeaseLost(leaseLost); - - const grantPromise = client.requestLease({ model: "iPhone 17", platform: "ios" }); - await flushMicrotasks(); - connection.reply( - connection.lastSentOf("lease.request")!.id, - sampleGrant({ leaseId: "lease_1" }), - ); - await grantPromise; - - // The release itself fails -- e.g. an agent-role client hitting FORBIDDEN. - const releasePromise = client.releaseLease({ leaseId: "lease_1" }); - await flushMicrotasks(); - const releaseCall = connection.lastSentOf("lease.release")!; - connection.fail(releaseCall.id, "FORBIDDEN", "not authorized"); - await expect(releasePromise).rejects.toMatchObject({ code: "FORBIDDEN" }); - - // The lease later genuinely expires/crashes -- this is a real loss the client must - // still surface, not residue from the earlier failed release attempt. - connection.push("lease-lost", { - deviceId: "device_1", - leaseId: "lease_1", - reason: "expired", - }); - - expect(leaseLost).toHaveBeenCalledWith({ - deviceId: "device_1", - leaseId: "lease_1", - reason: "expired", - }); - }); - - it("ignores an unknown push kind such as a protocol-3 daemon's lease.heartbeat", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - // Not reachable against a protocol-4 daemon (the ranges do not overlap, so `hello` fails - // first), but the wire must not answer a push it no longer understands. - connection.push("lease.heartbeat", { nonce: 12345 }); - await flushMicrotasks(); - - expect(connection.lastSentOf("lease.heartbeat")).toBeUndefined(); - await client.close(); - }); - - it("silently drops malformed lease-scoped and event pushes instead of throwing", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const leaseLost = vi.fn(); - const unhealthy = vi.fn(); - const recovered = vi.fn(); - client.onLeaseLost(leaseLost); - client.onDeviceUnhealthy(unhealthy); - client.onDeviceRecovered(recovered); - - expect(() => connection.push("lease-lost", { leaseId: "lease_1" })).not.toThrow(); - expect(() => connection.push("device-unhealthy", { leaseId: "lease_1" })).not.toThrow(); - expect(() => - connection.push("device-recovered", { deviceId: "device_1", leaseId: "lease_1" }), - ).not.toThrow(); - expect(() => connection.push("event", { not: "a valid event shape" })).not.toThrow(); - - expect(leaseLost).not.toHaveBeenCalled(); - expect(unhealthy).not.toHaveBeenCalled(); - expect(recovered).not.toHaveBeenCalled(); - }); -}); - -describe("event pushes", () => { - it("a worker push with a malformed id or timestamp never reaches the gateway's bus", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); - await flushMicrotasks(); - completeHello(connection, { role: "admin" }); - const client = await connectPromise; - const listener = vi.fn(); - const subscribed = client.subscribeEvents(listener); - await flushMicrotasks(); - const subscribe = connection.lastSentOf("events.subscribe"); - if (subscribe === undefined) throw new Error("the client never sent events.subscribe"); - connection.reply(subscribe.id, { subscribed: true, subscriptionId: "sub_1" }); - await subscribed; - const valid = { - event: "lease.expired", - id: "evt_ok", - module: "leases", - payload: {}, - seq: 1, - timestamp: 5, - }; - - for (const bad of [ - { ...valid, id: "no-prefix" }, - { ...valid, id: `evt_${"a".repeat(65)}` }, - { ...valid, id: 7 }, - { ...valid, timestamp: null }, - { ...valid, timestamp: "5" }, - ]) { - connection.push("event", { subscriptionId: "sub_1", event: bad }); - } - // `1e999` is valid JSON that parses to Infinity, a value JSON.stringify cannot emit; the - // same raw channel carries the well-formed twin, which proves the line is delivered at all. - const rawPush = (timestamp: string) => - JSON.stringify({ - payload: { subscriptionId: "sub_1", event: { ...valid, timestamp: "__timestamp__" } }, - push: "event", - }).replace('"__timestamp__"', timestamp); - connection.receiveLine(rawPush("1e999")); - await flushMicrotasks(); - expect(listener).not.toHaveBeenCalled(); - - connection.receiveLine(rawPush("5")); - await flushMicrotasks(); - expect(listener).toHaveBeenCalledTimes(1); - await client.close(); - }); - - it("a worker push with a timestamp outside the date range never reaches the gateway's bus", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); - await flushMicrotasks(); - completeHello(connection, { role: "admin" }); - const client = await connectPromise; - const listener = vi.fn(); - const subscribed = client.subscribeEvents(listener); - await flushMicrotasks(); - const subscribe = connection.lastSentOf("events.subscribe"); - if (subscribe === undefined) throw new Error("the client never sent events.subscribe"); - connection.reply(subscribe.id, { subscribed: true, subscriptionId: "sub_1" }); - await subscribed; - const push = (timestamp: number) => - connection.push("event", { - subscriptionId: "sub_1", - event: { - event: "lease.expired", - id: "evt_ok", - module: "leases", - payload: {}, - seq: 1, - timestamp, - }, - }); - - for (const outside of [8.64e15 + 2, -8.64e15 - 2, 1e300, -1e300]) push(outside); - await flushMicrotasks(); - expect(listener).not.toHaveBeenCalled(); - - push(8.64e15); - push(-8.64e15); - await flushMicrotasks(); - expect(listener).toHaveBeenCalledTimes(2); - await client.close(); - }); -}); - -describe("requestLease abort (ADR §10)", () => { - it("before the request is sent: rejects CANCELLED, nothing sent", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const controller = new AbortController(); - controller.abort(); - const before = connection.sent.length; - - await expect( - client.requestLease({ model: "iPhone 17", platform: "ios" }, { signal: controller.signal }), - ).rejects.toMatchObject({ code: "CANCELLED" }); - expect(connection.sent).toHaveLength(before); - }); - - it("while queued: sends lease.cancel, waits for the original to reject, surfaces CANCELLED", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - - controller.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - expect(cancelCall).toBeDefined(); - - connection.reply(cancelCall.id, { result: "cancelled" }); - await flushMicrotasks(); - connection.fail(requestCall.id, "QUEUE_TIMEOUT", "cancelled by request", { requestId: "r1" }); - - await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); - }); - - it("aborting one of two waiting requests of different platforms sends lease.cancel naming that request's platform and leaves the other waiting", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const iosController = new AbortController(); - const androidController = new AbortController(); - const iosLease = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: iosController.signal }, - ); - const androidLease = client.requestLease( - { model: "Pixel 9", platform: "android" }, - { signal: androidController.signal }, - ); - await flushMicrotasks(); - const requestOf = (platform: string) => - connection.sent - .filter((frame) => frame.type === "lease.request") - .find((frame) => (frame.payload as { platform: string }).platform === platform)!; - const androidCall = requestOf("android"); - - androidController.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - connection.reply(cancelCall.id, { result: "cancelled" }); - await flushMicrotasks(); - connection.fail(androidCall.id, "QUEUE_TIMEOUT", "cancelled by request", { requestId: "r2" }); - - await expect(androidLease).rejects.toMatchObject({ code: "CANCELLED" }); - expect(cancelCall.payload).toMatchObject({ platform: "android" }); - expect(connection.sent.filter((frame) => frame.type === "lease.cancel")).toHaveLength(1); - expect(requestOf("ios")).toBeDefined(); - const iosState = await Promise.race([iosLease.catch((e: unknown) => e), "waiting"]); - expect(iosState).toBe("waiting"); - }); - - it("while device work is in flight: releases a grant that still arrives, then rejects CANCELLED", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - - controller.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - connection.reply(cancelCall.id, { result: "not-cancellable" }); - await flushMicrotasks(); - - connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_abandoned" })); - await flushMicrotasks(); - const releaseCall = connection.lastSentOf("lease.release")!; - expect(releaseCall).toBeDefined(); - expect((releaseCall.payload as { leaseId: string }).leaseId).toBe("lease_abandoned"); - connection.reply(releaseCall.id, { leaseId: "lease_abandoned" }); - - await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); - }); - - it("after the grant resolved: abort is ignored", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_kept" })); - - const grant = await leasePromise; - expect(grant.lease.id).toBe("lease_kept"); - - const before = connection.sent.length; - controller.abort(); - await flushMicrotasks(); - - // No lease.cancel was ever sent for an already-resolved request. - expect(connection.sent).toHaveLength(before); - }); -}); - -describe("principal resolution (ADR §4, B3)", () => { - it('adopts the daemon-resolved principal from hello\'s reply, never defaulting to ""', async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - const hello = connection.lastSentOf("hello")!; - // No principal was supplied -- the daemon resolves its own default (a pid, in practice). - expect((hello.payload as { principal?: string }).principal).toBeUndefined(); - connection.reply(hello.id, { - daemonProtocolRange: { max: 3, min: 3 }, - principal: "48213", - protocolVersion: 3, - role: "agent", - version: "0.3.0", - }); - const client = await connectPromise; - - expect(client.principal).toBe("48213"); - }); -}); - -describe("requestLease abort: ADR §4 identity cases (B3)", () => { - it("no principal supplied: abort while queued cancels with the daemon-resolved principal, and leaves no lease held", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - const hello = connection.lastSentOf("hello")!; - connection.reply(hello.id, { - daemonProtocolRange: { max: 3, min: 3 }, - principal: "48213", - protocolVersion: 3, - role: "agent", - version: "0.3.0", - }); - const client = await connectPromise; - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - - controller.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - // Before this fix, this was always "" -- the daemon fixes the connection's principal to - // its own default, and the client had no way to learn it. Sent as the resolved principal - // now, never "". - expect((cancelCall.payload as { requesterId?: string }).requesterId).toBe("48213"); - - connection.reply(cancelCall.id, { result: "cancelled" }); - await flushMicrotasks(); - connection.fail(requestCall.id, "QUEUE_TIMEOUT", "cancelled by request"); - - await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); - - // No lease is held: connection death fires no onLeaseLost for anything. - const leaseLost = vi.fn(); - client.onLeaseLost(leaseLost); - connection.simulateDeath(); - await flushMicrotasks(); - expect(leaseLost).not.toHaveBeenCalled(); - }); - - it('ADR §4 proxy case (principal "host", requesterId "agent-7"): abort in flight releases the grant and leaves no lease held', async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection, principal: "host" }); - await flushMicrotasks(); - completeHello(connection, { principal: "host" }); - const client = await connectPromise; - expect(client.principal).toBe("host"); - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios", requesterId: "agent-7" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - const requestCall = connection.lastSentOf("lease.request")!; - - controller.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - // Before this fix, the daemon's `authorize` hook compared this `requesterId` straight to - // the principal and rejected FORBIDDEN outright -- exactly ADR §4's proxy case. The - // request is still sent with the proxied requesterId (attribution); the daemon now - // authorizes the cancel against the pending request's recorded *owner* instead. - expect((cancelCall.payload as { requesterId?: string }).requesterId).toBe("agent-7"); - connection.reply(cancelCall.id, { result: "not-cancellable" }); - await flushMicrotasks(); - - connection.reply(requestCall.id, sampleGrant({ leaseId: "lease_proxy_abandoned" })); - await flushMicrotasks(); - const releaseCall = connection.lastSentOf("lease.release")!; - expect((releaseCall.payload as { leaseId: string }).leaseId).toBe("lease_proxy_abandoned"); - connection.reply(releaseCall.id, { leaseId: "lease_proxy_abandoned" }); - - await expect(leasePromise).rejects.toMatchObject({ code: "CANCELLED" }); - - // Already released via the abandoned-grant path -- not tracked as held any more. - const leaseLost = vi.fn(); - client.onLeaseLost(leaseLost); - connection.simulateDeath(); - await flushMicrotasks(); - expect(leaseLost).not.toHaveBeenCalled(); - }); - - it("a FORBIDDEN from the abort's lease.cancel surfaces instead of being read as a dead connection", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const controller = new AbortController(); - const leasePromise = client.requestLease( - { model: "iPhone 17", platform: "ios" }, - { signal: controller.signal }, - ); - await flushMicrotasks(); - - controller.abort(); - await flushMicrotasks(); - const cancelCall = connection.lastSentOf("lease.cancel")!; - connection.fail(cancelCall.id, "FORBIDDEN", "not authorized"); - - await expect(leasePromise).rejects.toMatchObject({ code: "FORBIDDEN" }); - // Not misread as the connection dying -- it's still alive and usable for another call. - expect(connection.closed).toBe(false); - const anotherCall = client.cancelLease(); - await flushMicrotasks(); - const anotherCancelCall = connection.lastSentOf("lease.cancel")!; - expect(anotherCancelCall.id).not.toBe(cancelCall.id); - connection.reply(anotherCancelCall.id, { result: "not-found" }); - await expect(anotherCall).resolves.toMatchObject({ result: "not-found" }); - }); -}); - -describe("requestLease requires", () => { - it("sends requires on the lease.request frame and rejects with CAPABILITY_MISSING, its details naming the capabilities, when the worker lacks the capability", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlock({ connection }); - await flushMicrotasks(); - completeHello(connection); - const client = await connectPromise; - - const pending = client.requestLease({ - model: "iPhone 17", - platform: "ios", - requires: ["agent-device"], - }); - await flushMicrotasks(); - const call = connection.lastSentOf("lease.request")!; - connection.fail(call.id, "CAPABILITY_MISSING", "This worker cannot serve agent-device", { - capabilities: ["agent-device"], - }); - - expect(call.payload).toMatchObject({ requires: ["agent-device"] }); - await expect(pending).rejects.toMatchObject({ - code: "CAPABILITY_MISSING", - details: { capabilities: ["agent-device"] }, - kind: "domain", - }); - }); -}); - -describe("simlock/admin", () => { - it("exposes the admin operations and sends daemon.stop like any other request", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); - await flushMicrotasks(); - completeHello(connection, { role: "admin" }); - const admin = await connectPromise; - expect(admin.role).toBe("admin"); - - const stopPromise = admin.stopDaemon(); - await flushMicrotasks(); - const stopCall = connection.lastSentOf("daemon.stop")!; - connection.reply(stopCall.id, { stopping: true }); - await expect(stopPromise).resolves.toEqual({ stopping: true }); - }); - - it("releaseAllLeases clears every held lease id and connection death afterwards fires no onLeaseLost", async () => { - const connection = new ScriptedConnection(); - const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); - await flushMicrotasks(); - completeHello(connection, { role: "admin" }); - const admin = await connectPromise; - - const firstGrant = admin.requestLease({ model: "iPhone 17", platform: "ios" }); - await flushMicrotasks(); - connection.reply( - connection.lastSentOf("lease.request")!.id, - sampleGrant({ leaseId: "lease_a" }), - ); - await firstGrant; - - const secondGrant = admin.requestLease({ model: "iPhone 17", platform: "ios" }); - await flushMicrotasks(); - connection.reply( - connection.lastSentOf("lease.request")!.id, - sampleGrant({ leaseId: "lease_b" }), - ); - await secondGrant; - - const releaseAllPromise = admin.releaseAllLeases(); - await flushMicrotasks(); - const releaseAllCall = connection.lastSentOf("lease.release-all")!; - connection.reply(releaseAllCall.id, { leaseIds: ["lease_a", "lease_b"] }); - await expect(releaseAllPromise).resolves.toMatchObject({ - leaseIds: ["lease_a", "lease_b"], - }); - - // Both leases are no longer tracked as held -- a later connection death synthesizes no - // onLeaseLost for either of them. - const leaseLost = vi.fn(); - admin.onLeaseLost(leaseLost); - connection.simulateDeath(); - await flushMicrotasks(); - expect(leaseLost).not.toHaveBeenCalled(); - }); -}); diff --git a/src/simlock-client/test-client.ts b/src/simlock-client/test-client.ts new file mode 100644 index 000000000..36c3f3cb0 --- /dev/null +++ b/src/simlock-client/test-client.ts @@ -0,0 +1,39 @@ +import type { LeaseGrant } from "./types.js"; + +export async function flushMicrotasks(times = 10): Promise { + for (let index = 0; index < times; index += 1) await Promise.resolve(); +} + +export function sampleGrant( + overrides: { readonly leaseId?: string; readonly deviceId?: string } = {}, +): LeaseGrant { + const leaseId = overrides.leaseId ?? "lease_1"; + const deviceId = overrides.deviceId ?? "device_1"; + return { + device: { + physical: false, + driverDeviceId: "sim-1", + id: deviceId, + spec: { physical: false, model: "iPhone 17", osVersion: "18.0", platform: "ios" }, + mode: "full", + }, + environment: {}, + lease: { + deviceId, + grantedAt: 0, + id: leaseId, + lastRenewedAt: 0, + idChosenByRequester: false, + ownerId: "agent-1", + requesterId: "agent-1", + ttlMs: 1_000, + ttlDeadline: 1_000, + }, + timing: { + estimatedBootMs: 1, + estimatedProvisionMs: 1, + estimatedReadyMs: 1, + estimatedReclaimMs: 1, + }, + }; +} From b4ae223ac77c2c2915d7be6473dc43e66bcf01e3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:32:00 +0100 Subject: [PATCH 69/83] test(cli): split tunnel.test.ts by topic --- src/cli/test-tunnel.ts | 140 +++ src/cli/tunnel-connections.test.ts | 499 +++++++++ src/cli/tunnel-forward-and-emulator.test.ts | 423 ++++++++ src/cli/tunnel.test.ts | 1040 +------------------ 4 files changed, 1064 insertions(+), 1038 deletions(-) create mode 100644 src/cli/test-tunnel.ts create mode 100644 src/cli/tunnel-connections.test.ts create mode 100644 src/cli/tunnel-forward-and-emulator.test.ts diff --git a/src/cli/test-tunnel.ts b/src/cli/test-tunnel.ts new file mode 100644 index 000000000..e0bf5078e --- /dev/null +++ b/src/cli/test-tunnel.ts @@ -0,0 +1,140 @@ +import { TUNNEL_TAG, decodeTunnelFrame, encodeTunnelFrame } from "../contract/index.js"; +import { buildCliEnvironment, runCli } from "./index.js"; +import { + FakeClock, + FakeDaemonLauncher, + FakeHttpJsonClient, + FakeSystemStats, + FakeTcpListenerFactory, + FakeTcpStreamConnector, + FakeWebSocketClientConnector, + MemoryFilesystem, + MemoryIpcTransport, + ProcessSpawnError, + ScriptedProcessRunner, + type ScriptedProcessExpectation, +} from "../ports/index.js"; + +type Listener = () => void; + +// why: the CLI's signal source, so a test sends Ctrl-C without signalling the test process. +class TestSignals { + readonly #listeners = new Map>(); + on(signal: string, listener: Listener): this { + const set = this.#listeners.get(signal) ?? new Set(); + set.add(listener); + this.#listeners.set(signal, set); + return this; + } + off(signal: string, listener: Listener): this { + this.#listeners.get(signal)?.delete(listener); + return this; + } + emit(signal: string): void { + for (const listener of this.#listeners.get(signal) ?? []) listener(); + } +} + +// why: stands in for the caller's machine: only the commands in `present` start, as a real runner throws ProcessSpawnError for an adb that is not there. +export class CallerAdb extends ScriptedProcessRunner { + constructor( + expectations: readonly ScriptedProcessExpectation[], + private readonly present: readonly string[] = ["adb"], + ) { + super(expectations); + } + + override spawn( + command: string, + args: readonly string[], + options?: Parameters[2], + ) { + if (!this.present.includes(command)) throw new ProcessSpawnError(command, args); + return super.spawn(command, args, options); + } +} + +export function command(env: NodeJS.ProcessEnv, processRunner: CallerAdb = new CallerAdb([])) { + const signals = new TestSignals(); + const out: string[] = []; + const err: string[] = []; + const webSocket = new FakeWebSocketClientConnector(); + const httpJson = new FakeHttpJsonClient(); + const clock = new FakeClock(0); + const tcpStreams = new FakeTcpStreamConnector(); + const tcpListeners = new FakeTcpListenerFactory(); + const environment = buildCliEnvironment( + { + clock, + daemonHome: { dataDirectory: "/unused", gatewayPresent: false, role: "worker" }, + filesystem: new MemoryFilesystem(), + homeDirectory: "/home/caller", + hostPlatform: "linux", + httpJson, + ipc: new MemoryIpcTransport(), + launcher: new FakeDaemonLauncher(), + processRunner, + runTool: async () => ({ outcome: "not-found" }), + signals, + stderr: { write: (value) => err.push(value) }, + stdout: { write: (value) => out.push(value) }, + systemStats: new FakeSystemStats({ cpuCount: 2, totalRamBytes: 2 ** 30 }), + tcpListeners, + tcpStreams, + webSocket, + workingDirectory: "/work", + }, + env, + ); + return { + clock, + err, + httpJson, + out, + run: (argv: readonly string[]) => runCli(argv, environment), + signals, + tcpListeners, + tcpStreams, + webSocket, + }; +} + +export const READY = encodeTunnelFrame(TUNNEL_TAG.ready, 0, "{}"); + +export const ENV = { SIMLOCK_TOKEN: "sk_agent_x", SIMLOCK_URL: "http://127.0.0.1:7300" }; + +export function open(conn: number, devicePort: number): Uint8Array { + return encodeTunnelFrame(TUNNEL_TAG.open, conn, JSON.stringify({ devicePort, kind: "reverse" })); +} + +export function sentFrames(cli: ReturnType) { + return (cli.webSocket.connections[0]?.peer.sent ?? []).flatMap((bytes) => { + const frame = decodeTunnelFrame(bytes); + return frame === undefined ? [] : [frame]; + }); +} + +// why: a failure reaches stderr through a chain of promises, so a check that nothing was written lets them run first. +export function settle(): Promise { + return new Promise((done) => setTimeout(done, 0)); +} + +export function refusedFrame(code: string, message: string): Uint8Array { + return encodeTunnelFrame(TUNNEL_TAG.refused, 0, JSON.stringify({ code, message })); +} + +export function errorBody(code: string, message: string) { + return { error: { code, message } }; +} + +export function line(cli: ReturnType): unknown { + return JSON.parse(cli.err.join("")); +} + +export function firstPeer(cli: ReturnType): Peer { + const peer = cli.webSocket.connections[0]?.peer; + if (peer === undefined) throw new Error("no socket"); + return peer; +} + +export type Peer = ReturnType["webSocket"]["connections"][number]["peer"]; diff --git a/src/cli/tunnel-connections.test.ts b/src/cli/tunnel-connections.test.ts new file mode 100644 index 000000000..62e8a61e2 --- /dev/null +++ b/src/cli/tunnel-connections.test.ts @@ -0,0 +1,499 @@ +import { describe, expect, it, vi } from "vitest"; +import { TUNNEL_MAX_DATA_BYTES, TUNNEL_TAG, encodeTunnelFrame } from "../contract/index.js"; +import { FakeTcpPeer, WebSocketRefusedError } from "../ports/index.js"; +import { + ENV, + READY, + command, + errorBody, + firstPeer, + line, + open, + refusedFrame, + sentFrames, + settle, +} from "./test-tunnel.js"; + +async function started(argv: readonly string[]) { + const cli = command(ENV); + const exit = cli.run(["tunnel", ...argv, "--lease", "L1"]); + await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); + const peer = cli.webSocket.connections[0]?.peer; + if (peer === undefined) throw new Error("no socket"); + peer.receive(READY); + return { cli, exit, peer }; +} + +async function warningsPerTunnel(): Promise { + const quiet = await started(["reverse", "5173:5174", "reverse", "6000"]); + quiet.cli.tcpStreams.refusedPorts.add(5174); + quiet.cli.tcpStreams.refusedPorts.add(6000); + const opening = async (conn: number, port: number) => { + quiet.peer.receive(open(conn, port)); + await vi.waitFor(() => + expect(sentFrames(quiet.cli).filter((f) => f.tag === TUNNEL_TAG.reset)).toHaveLength( + (conn + 1) / 2, + ), + ); + }; + const warnings = () => + quiet.cli.err + .join("") + .split("\n") + .filter((line) => line !== ""); + quiet.peer.receive(open(9, 7000)); + await opening(1, 5173); + expect(warnings()).toEqual(["reverse 5173: nothing listening on localhost:5174"]); + quiet.cli.clock.advance(9_999); + await opening(3, 5173); + expect(warnings()).toEqual(["reverse 5173: nothing listening on localhost:5174"]); + await opening(5, 6000); + expect(warnings()).toEqual([ + "reverse 5173: nothing listening on localhost:5174", + "reverse 6000: nothing listening on localhost:6000", + ]); + quiet.cli.clock.advance(1); + await opening(7, 5173); + expect(warnings()).toHaveLength(3); + expect(warnings()[2]).toBe("reverse 5173: nothing listening on localhost:5174"); + expect( + sentFrames(quiet.cli) + .filter((f) => f.tag === TUNNEL_TAG.reset) + .map((f) => f.conn), + ).toEqual([1, 3, 5, 7]); + expect(quiet.cli.tcpStreams.connections).toEqual([]); + expect(quiet.peer.closedWith).toBeUndefined(); + quiet.cli.signals.emit("SIGINT"); + expect(await quiet.exit).toBe(0); +} + +async function forwardResetWarnings(): Promise { + const cli = command(ENV); + const exit = cli.run(["tunnel", "forward", "9229:19229", "--lease", "L1"]); + await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); + const peer = firstPeer(cli); + peer.receive(READY); + const reason = "nothing on the lease's simulator listens on port 9229"; + const warnings = () => + cli.err + .join("") + .split("\n") + .filter((text) => text !== ""); + const resetByWorker = async (conn: number) => { + cli.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => expect(sentFrames(cli).at(-1)?.conn).toBe(conn)); + peer.receive(encodeTunnelFrame(TUNNEL_TAG.reset, conn, JSON.stringify({ reason }))); + await settle(); + }; + await resetByWorker(2); + expect(warnings()).toEqual([`warning: forward 9229: ${reason}`]); + cli.clock.advance(9_999); + await resetByWorker(4); + expect(warnings()).toHaveLength(1); + cli.clock.advance(1); + await resetByWorker(6); + expect(warnings()).toEqual([ + `warning: forward 9229: ${reason}`, + `warning: forward 9229: ${reason}`, + ]); + expect(peer.closedWith).toBeUndefined(); + cli.signals.emit("SIGINT"); + expect(await exit).toBe(0); +} + +async function responseThroughTheTunnel() { + const page = await started(["reverse", "5173:5174"]); + page.peer.receive(open(1, 5173)); + await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(1)); + expect(page.cli.tcpStreams.connections[0]?.port).toBe(5174); + const local = page.cli.tcpStreams.connections[0]?.peer; + if (local === undefined) throw new Error("no local connection"); + const response = Uint8Array.from({ length: 1_048_576 }, (_, index) => index % 251); + for (let at = 0; at < response.length; at += 131_072) + local.send(response.subarray(at, at + 131_072)); + local.end(); + const received: number[] = []; + let ended = false; + let before = 0; + const credit = (conn: number, size: number) => { + const granted = new Uint8Array(4); + new DataView(granted.buffer).setUint32(0, size); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.credit, conn, granted)); + }; + const drain = () => { + const frames = sentFrames(page.cli); + for (const frame of frames.slice(before)) { + if (frame.tag === TUNNEL_TAG.data) for (const byte of frame.payload) received.push(byte); + else if (frame.tag === TUNNEL_TAG.end) ended = true; + } + before = frames.length; + }; + await vi.waitFor(() => { + drain(); + expect(received.length).toBe(262_144); + }); + await new Promise((done) => setTimeout(done, 20)); + drain(); + expect(received).toHaveLength(262_144); + while (!ended) { + const have = received.length; + credit(1, 65_536); + await vi.waitFor(() => { + drain(); + expect(received.length > have || ended).toBe(true); + }); + } + expect(received).toHaveLength(1_048_576); + expect(Uint8Array.from(received)).toEqual(response); + const dataSizes = sentFrames(page.cli) + .filter((f) => f.tag === TUNNEL_TAG.data) + .map((f) => f.payload.length); + expect(Math.max(...dataSizes)).toBe(TUNNEL_MAX_DATA_BYTES); + expect(dataSizes.every((size) => size <= TUNNEL_MAX_DATA_BYTES)).toBe(true); + + return { local, page }; +} + +async function writesToTheCaller( + page: Awaited>, + local: FakeTcpPeer, +): Promise { + for (const size of [65_536, 30_000]) { + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 1, new Uint8Array(size))); + } + await vi.waitFor(() => + expect(local.written.map((chunk) => chunk.length)).toEqual([65_536, 30_000]), + ); + await vi.waitFor(() => { + const credits = sentFrames(page.cli) + .filter((f) => f.tag === TUNNEL_TAG.credit) + .map((f) => new DataView(f.payload.buffer, f.payload.byteOffset).getUint32(0)); + expect(credits).toEqual([65_536, 30_000]); + }); + expect(local.closed).toBe(false); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 1)); + await vi.waitFor(() => expect(local.writeEnded).toBe(true)); + await vi.waitFor(() => expect(local.closed).toBe(true)); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 1, new Uint8Array(5))); + await settle(); + expect(local.written.map((chunk) => chunk.length)).toEqual([65_536, 30_000]); + page.peer.receive(open(1, 5173)); + await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(2)); + const second = page.cli.tcpStreams.connections[1]?.peer; + expect(second?.closed).toBe(false); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 1)); + await vi.waitFor(() => expect(second?.writeEnded).toBe(true)); + await settle(); + expect(second?.closed).toBe(false); + second?.end(); + await vi.waitFor(() => expect(second?.closed).toBe(true)); + page.peer.receive(open(3, 5173)); + await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(3)); + const reset = page.cli.tcpStreams.connections[2]?.peer; + expect(reset?.closed).toBe(false); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.reset, 3)); + await vi.waitFor(() => expect(reset?.closed).toBe(true)); +} + +async function brokenWrite(page: Awaited>): Promise { + page.peer.receive(open(5, 5173)); + await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(4)); + const broken = page.cli.tcpStreams.connections[3]?.peer; + if (broken === undefined) throw new Error("no fourth connection"); + broken.writeFailure = new Error("write EPIPE"); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 5, new Uint8Array(10))); + page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 5)); + await vi.waitFor(() => + expect( + sentFrames(page.cli) + .filter((f) => f.tag === TUNNEL_TAG.reset) + .map((f) => f.conn), + ).toEqual([5]), + ); + await settle(); + expect(broken.closed).toBe(true); + expect(sentFrames(page.cli).filter((f) => f.tag === TUNNEL_TAG.credit && f.conn === 5)).toEqual( + [], + ); + expect(page.cli.err).toEqual([]); + page.peer.receive(open(7, 5173)); + await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(5)); + expect(page.cli.tcpStreams.connections[4]?.peer.closed).toBe(false); + page.cli.signals.emit("SIGINT"); + expect(await page.exit).toBe(0); +} + +describe("simlock tunnel carrying connections", () => { + it("warns once per 10 s per tunnel when nothing listens, or the worker resets a forward connection with its reason, keeps the tunnel open, and a 1 MiB response arrives whole, a connection whose both halves ended is closed and dropped, and one with a single half ended keeps relaying the other", async () => { + await warningsPerTunnel(); + await forwardResetWarnings(); + const { local, page } = await responseThroughTheTunnel(); + await writesToTheCaller(page, local); + await brokenWrite(page); + }); +}); + +async function withLocalConnection( + cli: ReturnType, + peer: { receive(bytes: Uint8Array): void }, + devicePort: number, +) { + const before = cli.tcpStreams.connections.length; + peer.receive(open(1 + 2 * before, devicePort)); + await vi.waitFor(() => expect(cli.tcpStreams.connections).toHaveLength(before + 1)); + const local = cli.tcpStreams.connections[before]?.peer; + if (local === undefined) throw new Error("no local connection"); + expect(local.closed).toBe(false); + return local; +} + +async function upgradeRefusals(): Promise { + const upgrades: readonly { + readonly refusal: WebSocketRefusedError; + readonly exit?: number; + readonly code: string; + readonly message: string; + }[] = [ + { + code: "UNAUTHENTICATED", + message: "No valid token", + refusal: new WebSocketRefusedError( + 401, + JSON.stringify(errorBody("UNAUTHENTICATED", "No valid token")), + ), + }, + { + code: "UNKNOWN_LEASE", + message: "Unknown lease L1", + refusal: new WebSocketRefusedError( + 404, + JSON.stringify(errorBody("UNKNOWN_LEASE", "Unknown lease L1")), + ), + }, + { + code: "TUNNEL_UNAVAILABLE", + message: "Tunnels are off", + refusal: new WebSocketRefusedError( + 503, + JSON.stringify(errorBody("TUNNEL_UNAVAILABLE", "Tunnels are off")), + ), + }, + { + code: "UNKNOWN_LEASE", + message: "Lease L1 is not known to this worker", + refusal: new WebSocketRefusedError(404, "Not Found"), + }, + { + code: "BAD_REQUEST", + exit: 2, + message: "Bad upgrade", + refusal: new WebSocketRefusedError( + 400, + JSON.stringify(errorBody("BAD_REQUEST", "Bad upgrade")), + ), + }, + { + code: "TUNNEL_DISCONNECTED", + message: "Lease L1: the worker answered 502", + refusal: new WebSocketRefusedError(502, "Bad Gateway"), + }, + ]; + for (const { code, exit, message, refusal } of upgrades) { + const cli = command(ENV); + cli.webSocket.failure = refusal; + expect(await cli.run(["tunnel", "reverse", "5173", "--lease", "L1"])).toBe(exit ?? 1); + expect(line(cli)).toEqual(errorBody(code, message)); + } +} + +async function lookupFailures(): Promise { + const lookups: readonly { + readonly identity?: { status: number; body: unknown }; + readonly code: string; + readonly message: string; + }[] = [ + { + code: "UNAUTHENTICATED", + identity: { body: errorBody("UNAUTHENTICATED", "No valid token"), status: 401 }, + message: "No valid token", + }, + { + code: "FORBIDDEN", + identity: { body: errorBody("FORBIDDEN", "Not allowed"), status: 403 }, + message: "Not allowed", + }, + { + code: "TUNNEL_DISCONNECTED", + identity: { body: undefined, status: 502 }, + message: "The worker answered 502", + }, + { + code: "TUNNEL_DISCONNECTED", + message: "No answer from http://127.0.0.1:7300", + }, + ]; + for (const { code, identity, message } of lookups) { + const cli = command(ENV); + if (identity !== undefined) + cli.httpJson.answers.set("http://127.0.0.1:7300/v1/identity", identity); + expect(await cli.run(["tunnel", "reverse", "5173"])).toBe(1); + expect(line(cli)).toEqual(errorBody(code, message)); + expect(cli.webSocket.connections).toEqual([]); + } +} + +async function refusalFrame(): Promise { + const refused = await started(["reverse", "80"]); + const refusedLocal = await withLocalConnection(refused.cli, refused.peer, 80); + refused.peer.receive(refusedFrame("TUNNEL_BAD_PORT", "Device port 80 is below 1024")); + expect(await refused.exit).toBe(1); + expect(refusedLocal.closed).toBe(true); + expect(refused.peer.closedWith?.code).toBe(1000); + expect(refused.cli.clock.pendingTimerCount).toBe(0); + expect(line(refused.cli)).toEqual( + errorBody("TUNNEL_BAD_PORT", "Lease L1: Device port 80 is below 1024"), + ); + const unknown = await started(["reverse", "5173"]); + unknown.peer.receive(refusedFrame("NOT_A_CODE", "something new")); + expect(await unknown.exit).toBe(1); + expect(line(unknown.cli)).toEqual(errorBody("TUNNEL_FAILED", "Lease L1: something new")); + const garbled = await started(["reverse", "5173"]); + garbled.peer.receive(open(1, 7000)); + garbled.peer.receive(encodeTunnelFrame(TUNNEL_TAG.open, 3, "not json")); + garbled.peer.receive(encodeTunnelFrame(TUNNEL_TAG.refused, 0, "not json")); + expect(await garbled.exit).toBe(1); + expect(line(garbled.cli)).toEqual( + errorBody("TUNNEL_FAILED", "Lease L1: the worker refused the tunnel"), + ); + expect(garbled.cli.tcpStreams.connections).toEqual([]); +} + +async function lossesOfTheSocket(): Promise { + const losses: readonly { + readonly drop: [number, string]; + readonly exit: number; + readonly error: unknown; + }[] = [ + { + drop: [4001, "device released"], + error: errorBody("TUNNEL_LEASE_ENDED", "Lease L1 ended: device released"), + exit: 14, + }, + ...[1011, 1001, 1006].map((code) => ({ + drop: [code, ""] as [number, string], + error: errorBody( + "TUNNEL_DISCONNECTED", + `Lease L1: the connection to ${ENV.SIMLOCK_URL} was lost`, + ), + exit: 1, + })), + ]; + for (const { drop, error, exit } of losses) { + const lost = await started(["reverse", "5173"]); + const lostLocal = await withLocalConnection(lost.cli, lost.peer, 5173); + lost.peer.dropped(...drop); + expect(await lost.exit).toBe(exit); + expect(lostLocal.closed).toBe(true); + expect(lost.cli.clock.pendingTimerCount).toBe(0); + expect(line(lost.cli)).toEqual(error); + } +} + +async function silentSocket(): Promise { + const silent = await started(["reverse", "5173"]); + silent.cli.clock.advance(30_000); + silent.peer.ping(); + silent.cli.clock.advance(44_999); + await settle(); + expect(silent.cli.err).toEqual([]); + const silentLocal = await withLocalConnection(silent.cli, silent.peer, 5173); + silent.cli.clock.advance(44_999); + await settle(); + expect(silent.cli.err).toEqual([]); + silent.cli.clock.advance(1); + await settle(); + expect(silent.cli.err).toHaveLength(1); + expect(await silent.exit).toBe(1); + expect(silentLocal.closed).toBe(true); + expect(silent.peer.closedWith?.code).toBe(1000); + expect(silent.peer.terminated).toBe(true); + expect(silent.cli.clock.pendingTimerCount).toBe(0); + expect(line(silent.cli)).toEqual( + errorBody("TUNNEL_DISCONNECTED", "Lease L1: nothing was heard from the worker for 45 s"), + ); +} + +async function noReadySocket(): Promise { + const noReady = command(ENV); + const noReadyExit = noReady.run(["tunnel", "reverse", "5173", "--lease", "L1"]); + await vi.waitFor(() => expect(noReady.webSocket.connections).toHaveLength(1)); + const quietPeer = noReady.webSocket.connections[0]?.peer; + for (let at = 0; at < 89_999; at += 15_000) { + noReady.clock.advance(Math.min(15_000, 89_999 - at)); + quietPeer?.ping(); + } + await settle(); + expect(noReady.err).toEqual([]); + noReady.clock.advance(1); + await settle(); + expect(noReady.err).toHaveLength(1); + expect(await noReadyExit).toBe(1); + expect(quietPeer?.closedWith?.code).toBe(1000); + expect(quietPeer?.terminated).toBe(true); + expect(noReady.clock.pendingTimerCount).toBe(0); + expect(line(noReady)).toEqual( + errorBody("TUNNEL_DISCONNECTED", "Lease L1: no READY from the worker within 90 s"), + ); +} + +async function ctrlC(): Promise { + const idle = await started(["reverse", "5173"]); + for (let at = 0; at < 90_000; at += 15_000) { + idle.cli.clock.advance(15_000); + idle.peer.ping(); + } + await settle(); + expect(idle.cli.err).toEqual([]); + expect(idle.peer.closedWith).toBeUndefined(); + const idleLocal = await withLocalConnection(idle.cli, idle.peer, 5173); + idle.peer.silentOnClose = true; + idle.cli.signals.emit("SIGINT"); + expect(idle.peer.closedWith?.code).toBe(1000); + let settled = false; + void idle.exit.then(() => { + settled = true; + }); + idle.cli.clock.advance(1_999); + await settle(); + expect(settled).toBe(false); + idle.cli.clock.advance(1); + await settle(); + expect(settled).toBe(true); + expect(await idle.exit).toBe(0); + expect(idleLocal.closed).toBe(true); + expect(idle.peer.terminated).toBe(true); + expect(idle.cli.clock.pendingTimerCount).toBe(0); +} + +async function sigterm(): Promise { + const term = await started(["reverse", "5173"]); + const termLocal = await withLocalConnection(term.cli, term.peer, 5173); + term.cli.signals.emit("SIGTERM"); + expect(term.peer.closedWith?.code).toBe(1000); + expect(await term.exit).toBe(0); + expect(termLocal.closed).toBe(true); + expect(term.cli.clock.pendingTimerCount).toBe(0); +} + +describe("simlock tunnel that is refused or loses its worker", () => { + it("exits with the worker's code and one JSON line, and a Ctrl-C closes and waits at most 2 s", async () => { + await upgradeRefusals(); + await lookupFailures(); + await refusalFrame(); + await lossesOfTheSocket(); + await silentSocket(); + await noReadySocket(); + await ctrlC(); + await sigterm(); + }); +}); diff --git a/src/cli/tunnel-forward-and-emulator.test.ts b/src/cli/tunnel-forward-and-emulator.test.ts new file mode 100644 index 000000000..3a3d7c8ed --- /dev/null +++ b/src/cli/tunnel-forward-and-emulator.test.ts @@ -0,0 +1,423 @@ +import { describe, expect, it, vi } from "vitest"; +import { TUNNEL_TAG, encodeTunnelFrame } from "../contract/index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import type { ScriptedProcessExpectation } from "../ports/index.js"; +import { + CallerAdb, + ENV, + Peer, + READY, + command, + errorBody, + firstPeer, + line, + open, + refusedFrame, + sentFrames, + settle, +} from "./test-tunnel.js"; + +describe("simlock tunnel forward listening on the caller's loopback", () => { + it("binds each caller port before HELLO, holds connections until READY, announces them with even ids and closes every listener however the command ends, and sends a device's OPEN to the reverse tunnel when a forward names the same device port", async () => { + const busy = command(ENV); + busy.tcpListeners.busy.add(9229); + expect(await busy.run(["tunnel", "forward", "9229", "--lease", "L1"])).toBe(1); + expect(line(busy)).toEqual( + errorBody("TUNNEL_CALLER_PORT_IN_USE", "forward 9229: localhost:9229 is already in use"), + ); + expect(busy.webSocket.connections).toEqual([]); + + const waiting = command(ENV); + const exit = waiting.run([ + "tunnel", + "forward", + "9229:19229", + "reverse", + "9229:5173", + "--lease", + "L1", + ]); + await vi.waitFor(() => expect(waiting.webSocket.connections).toHaveLength(1)); + const peer = firstPeer(waiting); + expect(waiting.tcpListeners.listeners.map((listener) => listener.port)).toEqual([19229]); + const early = waiting.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => + expect(sentFrames(waiting).map((f) => f.tag)).toEqual([TUNNEL_TAG.hello]), + ); + expect(early?.closed).toBe(false); + peer.receive(READY); + await vi.waitFor(() => + expect(sentFrames(waiting).map((f) => [f.tag, f.conn])).toEqual([ + [TUNNEL_TAG.hello, 0], + [TUNNEL_TAG.open, 2], + ]), + ); + expect(JSON.parse(new TextDecoder().decode(sentFrames(waiting)[1]?.payload))).toEqual({ + devicePort: 9229, + kind: "forward", + }); + waiting.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => expect(sentFrames(waiting).at(-1)?.conn).toBe(4)); + peer.receive(open(1, 9229)); + await vi.waitFor(() => expect(waiting.tcpStreams.connections).toHaveLength(1)); + expect(waiting.tcpStreams.connections[0]?.port).toBe(5173); + waiting.signals.emit("SIGINT"); + expect(await exit).toBe(0); + + const refusedEarly = command(ENV); + const refusedExit = refusedEarly.run(["tunnel", "forward", "9229", "--lease", "L1"]); + await vi.waitFor(() => expect(refusedEarly.webSocket.connections).toHaveLength(1)); + const queued = refusedEarly.tcpListeners.listeners[0]?.connect(); + refusedEarly.webSocket.connections[0]?.peer.receive( + refusedFrame("TUNNEL_BAD_PORT", "Device port 80 is below 1024"), + ); + expect(await refusedExit).toBe(1); + expect(queued?.closed).toBe(true); + expect(sentFrames(refusedEarly).map((f) => f.tag)).toEqual([TUNNEL_TAG.hello]); + + const slow = command(ENV); + let releaseLookup: () => void = () => undefined; + const lookup = new Promise((resolve) => { + releaseLookup = resolve; + }); + slow.httpJson.get = async () => { + await lookup; + throw new Error("the worker is unreachable"); + }; + const slowExit = slow.run(["tunnel", "forward", "9229"]); + await vi.waitFor(() => expect(slow.tcpListeners.listeners).toHaveLength(1)); + const beforeSocket = slow.tcpListeners.listeners[0]?.connect(); + expect(beforeSocket?.closed).toBe(false); + releaseLookup(); + expect(await slowExit).toBe(1); + expect(beforeSocket?.closed).toBe(true); + + const endings: readonly ((cli: ReturnType, peer: Peer) => void)[] = [ + (cli) => cli.signals.emit("SIGINT"), + (_cli, ended) => ended.receive(refusedFrame("TUNNEL_BAD_PORT", "no")), + (_cli, ended) => ended.dropped(4001, "device released"), + (_cli, ended) => ended.dropped(1006, ""), + ]; + for (const ending of endings) await listenersCloseAfter(ending); + }); +}); + +async function listenersCloseAfter( + ending: (cli: ReturnType, peer: Peer) => void, +): Promise { + const cli = command(ENV); + const run = cli.run(["tunnel", "forward", "9229", "forward", "9230", "--lease", "L1"]); + await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); + const connection = firstPeer(cli); + connection.silentOnClose = true; + expect(cli.tcpListeners.listeners.map((listener) => listener.closed)).toEqual([false, false]); + ending(cli, connection); + cli.clock.advance(2_000); + await run; + expect(cli.tcpListeners.listeners.map((listener) => listener.closed)).toEqual([true, true]); +} + +describe("simlock tunnel emulator adb", () => { + const emulatorOpen = JSON.stringify({ kind: "emulator", emulatorPort: "adb" }); + + // fallow-ignore-next-line complexity -- one Tests line: each adb outcome and each way the command ends are rows of its cases. + it("prints the tunnel line and connects the caller's own adb to the caller's port, which opens an emulator connection for each connection to it", async () => { + const adb = new CallerAdb([ + { + match: { args: ["connect", "127.0.0.1:49152"], command: "adb" }, + stdoutLines: ["connected to 127.0.0.1:49152"], + }, + ]); + const cli = command(ENV, adb); + const exit = cli.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); + await vi.waitFor(() => expect(sentFrames(cli)).toHaveLength(1)); + expect(new TextDecoder().decode(sentFrames(cli)[0]?.payload)).toBe( + '{"tunnels":[{"kind":"emulator","emulatorPort":"adb"}]}', + ); + expect(cli.out).toEqual([]); + + firstPeer(cli).receive(READY); + await vi.waitFor(() => + expect(cli.out.join("")).toBe( + "emulator localhost:49152 -> emulator:adb (lease L1)\nadb: connected to 127.0.0.1:49152\n", + ), + ); + expect(adb.calls.map(({ args, command: bin }) => [bin, ...args])).toEqual([ + ["adb", "connect", "127.0.0.1:49152"], + ]); + + cli.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => expect(sentFrames(cli).at(-1)?.conn).toBe(2)); + expect(sentFrames(cli).at(-1)?.tag).toBe(TUNNEL_TAG.open); + expect(new TextDecoder().decode(sentFrames(cli).at(-1)?.payload)).toBe(emulatorOpen); + + cli.signals.emit("SIGINT"); + cli.clock.advance(2_000); + expect(await exit).toBe(0); + + const target = "127.0.0.1:49152"; + const connect = ( + command: string, + output: Partial = {}, + ): ScriptedProcessExpectation => ({ + match: { args: ["connect", target], command }, + stdoutLines: [`connected to ${target}`], + ...output, + }); + const rerun = `run: adb connect ${target}`; + const outputs = [ + { + adb: new CallerAdb([ + connect("adb", { + stdoutLines: [], + stderrLines: ["* daemon not running; starting now"], + result: { + code: 0, + stderr: "", + stdout: `* daemon started successfully\nconnected to ${target}\n`, + }, + }), + ]), + err: "", + out: `adb: connected to ${target}\n`, + }, + { + adb: new CallerAdb([connect("adb", { stdoutLines: [`already connected to ${target}`] })]), + err: "", + out: `adb: connected to ${target}\n`, + }, + { + adb: new CallerAdb([ + connect("adb", { stdoutLines: [`failed to authenticate to ${target}`] }), + ]), + err: `warning: emulator adb: adb answered: failed to authenticate to ${target}; ${rerun}\n`, + out: "", + }, + { + adb: new CallerAdb([ + connect("adb", { + stdoutLines: [], + result: { + code: 1, + stderr: "", + stdout: `* daemon started successfully\nfailed to authenticate to ${target}\n`, + }, + }), + ]), + err: `warning: emulator adb: adb answered: failed to authenticate to ${target}; ${rerun}\n`, + out: "", + }, + { + adb: new CallerAdb([], []), + err: `warning: emulator adb: adb was not found; ${rerun}\n`, + out: "", + }, + { + adb: new CallerAdb([connect("/sdk/platform-tools/adb")], ["/sdk/platform-tools/adb"]), + env: { ANDROID_HOME: "/sdk" }, + err: "", + out: `adb: connected to ${target}\n`, + }, + { + adb: new CallerAdb( + [connect("/home/caller/Library/Android/sdk/platform-tools/adb")], + ["/home/caller/Library/Android/sdk/platform-tools/adb"], + ), + err: "", + out: `adb: connected to ${target}\n`, + }, + ]; + for (const expected of outputs) { + const other = command({ ...ENV, ...expected.env }, expected.adb); + const otherExit = other.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(other.webSocket.connections).toHaveLength(1)); + firstPeer(other).receive(READY); + await vi.waitFor(() => + expect(other.out.join("") + other.err.join("")).toContain( + expected.out === "" ? expected.err : expected.out, + ), + ); + await settle(); + expect(other.out.join("")).toBe( + `emulator localhost:49152 -> emulator:adb (lease L1)\n${expected.out}`, + ); + expect(other.err.join("")).toBe(expected.err); + other.signals.emit("SIGINT"); + other.clock.advance(2_000); + expect(await otherExit).toBe(0); + } + + const pinned = new CallerAdb([ + { + match: { args: ["connect", "127.0.0.1:6000"], command: "adb" }, + stdoutLines: ["connected to 127.0.0.1:6000"], + }, + ]); + const both = command(ENV, pinned); + const bothExit = both.run(["tunnel", "emulator", "adb:6000", "--lease", "L1"]); + await vi.waitFor(() => expect(both.webSocket.connections).toHaveLength(1)); + expect(both.tcpListeners.listeners.map((listener) => listener.port)).toEqual([6000]); + firstPeer(both).receive(READY); + await vi.waitFor(() => + expect(both.out.join("")).toBe( + "emulator localhost:6000 -> emulator:adb (lease L1)\nadb: connected to 127.0.0.1:6000\n", + ), + ); + both.signals.emit("SIGINT"); + both.clock.advance(2_000); + expect(await bothExit).toBe(0); + + const busy = command(ENV, new CallerAdb([])); + busy.tcpListeners.busy.add(6000); + expect(await busy.run(["tunnel", "emulator", "adb:6000", "--lease", "L1"])).toBe(1); + expect(line(busy)).toEqual( + errorBody("TUNNEL_CALLER_PORT_IN_USE", "emulator adb: localhost:6000 is already in use"), + ); + + const mixedAdb = new CallerAdb([connect("adb")]); + const mixed = command(ENV, mixedAdb); + const mixedExit = mixed.run(["tunnel", "reverse", "8081", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(sentFrames(mixed)).toHaveLength(1)); + expect(new TextDecoder().decode(sentFrames(mixed)[0]?.payload)).toBe( + '{"tunnels":[{"devicePort":8081,"kind":"reverse"},{"kind":"emulator","emulatorPort":"adb"}]}', + ); + firstPeer(mixed).receive(READY); + await vi.waitFor(() => + expect(mixed.out.join("")).toBe( + `reverse device:8081 -> localhost:8081 (lease L1)\nemulator localhost:49152 -> emulator:adb (lease L1)\nadb: connected to ${target}\n`, + ), + ); + firstPeer(mixed).receive(open(1, 8081)); + await vi.waitFor(() => + expect(mixed.tcpStreams.connections.map(({ port }) => port)).toEqual([8081]), + ); + + mixed.tcpListeners.listeners[0]?.connect(); + mixed.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => + expect(sentFrames(mixed).filter((f) => f.tag === TUNNEL_TAG.open)).toHaveLength(2), + ); + const reset = (conn: number, reason: string) => + encodeTunnelFrame(TUNNEL_TAG.reset, conn, JSON.stringify({ reason })); + firstPeer(mixed).receive(reset(2, "closed")); + firstPeer(mixed).receive(reset(4, "timeout")); + await vi.waitFor(() => expect(mixed.err.join("")).toBe("warning: emulator adb: closed\n")); + mixed.clock.advance(9_999); + mixed.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => expect(sentFrames(mixed).at(-1)?.conn).toBe(6)); + firstPeer(mixed).receive(reset(6, "timeout")); + await settle(); + expect(mixed.err.join("")).toBe("warning: emulator adb: closed\n"); + mixed.clock.advance(1); + mixed.tcpListeners.listeners[0]?.connect(); + await vi.waitFor(() => expect(sentFrames(mixed).at(-1)?.conn).toBe(8)); + firstPeer(mixed).receive(reset(8, "timeout")); + await vi.waitFor(() => + expect(mixed.err.join("")).toBe( + "warning: emulator adb: closed\nwarning: emulator adb: timeout\n", + ), + ); + mixed.signals.emit("SIGINT"); + mixed.clock.advance(2_000); + expect(await mixedExit).toBe(0); + + const disconnect = ( + output: Partial = {}, + ): ScriptedProcessExpectation => ({ + match: { args: ["disconnect", target], command: "adb" }, + stdoutLines: [`disconnected ${target}`], + ...output, + }); + const calls = (runner: CallerAdb) => runner.calls.map(({ args }) => args.join(" ")); + const endings: readonly { + readonly end: (cli: ReturnType, peer: Peer) => void; + readonly exit: number; + }[] = [ + { end: (cli) => cli.signals.emit("SIGINT"), exit: 0 }, + { end: (_cli, ended) => ended.receive(refusedFrame("TUNNEL_BAD_PORT", "no")), exit: 1 }, + { end: (_cli, ended) => ended.dropped(4001, "device released"), exit: 14 }, + { end: (_cli, ended) => ended.dropped(1006, ""), exit: 1 }, + ]; + for (const ending of endings) { + const runner = new CallerAdb([connect("adb"), disconnect()]); + const leaving = command(ENV, runner); + const leavingExit = leaving.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(leaving.webSocket.connections).toHaveLength(1)); + firstPeer(leaving).receive(READY); + await vi.waitFor(() => expect(leaving.out.join("")).toContain("adb: connected")); + ending.end(leaving, firstPeer(leaving)); + expect(await leavingExit).toBe(ending.exit); + expect(calls(runner)).toEqual([`connect ${target}`, `disconnect ${target}`]); + } + + const failing = new CallerAdb([ + connect("adb"), + disconnect({ result: { code: 1, stderr: "error: no such device", stdout: "" } }), + ]); + const failed = command(ENV, failing); + const failedExit = failed.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(failed.webSocket.connections).toHaveLength(1)); + firstPeer(failed).receive(READY); + await vi.waitFor(() => expect(failed.out.join("")).toContain("adb: connected")); + firstPeer(failed).dropped(4001, "device released"); + expect(await failedExit).toBe(14); + expect(calls(failing)).toEqual([`connect ${target}`, `disconnect ${target}`]); + + const stuckAdb = new CallerAdb([connect("adb"), disconnect({ hangs: true })]); + const stuck = command(ENV, stuckAdb); + const stuckExit = stuck.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(stuck.webSocket.connections).toHaveLength(1)); + firstPeer(stuck).receive(READY); + await vi.waitFor(() => expect(stuck.out.join("")).toContain("adb: connected")); + stuck.signals.emit("SIGINT"); + await vi.waitFor(() => expect(stuckAdb.calls).toHaveLength(2)); + const stuckState = promiseState(stuckExit); + stuck.clock.advance(1_999); + await settle(); + expect(stuckState.state).toBe("pending"); + stuck.clock.advance(1); + await settle(); + expect(stuckState.state).toBe("fulfilled"); + expect(await stuckExit).toBe(0); + expect((await stuckAdb.handles[1]?.wait())?.code).toBeNull(); + + const early = new CallerAdb([connect("adb", { hangs: true }), disconnect()]); + const interrupted = command(ENV, early); + const interruptedExit = interrupted.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(interrupted.webSocket.connections).toHaveLength(1)); + firstPeer(interrupted).receive(READY); + await vi.waitFor(() => expect(early.calls).toHaveLength(1)); + interrupted.signals.emit("SIGINT"); + const interruptedState = promiseState(interruptedExit); + await settle(); + expect(interruptedState.state).toBe("fulfilled"); + expect(await interruptedExit).toBe(0); + expect(calls(early)).toEqual([`connect ${target}`, `disconnect ${target}`]); + const killed = promiseState(Promise.resolve(early.handles[0]?.wait())); + await settle(); + expect(killed.state).toBe("fulfilled"); + expect((await early.handles[0]?.wait())?.code).toBeNull(); + expect(interrupted.err).toEqual([]); + + const hung = new CallerAdb([connect("adb", { hangs: true })]); + const slow = command(ENV, hung); + const slowExit = slow.run(["tunnel", "emulator", "adb", "--lease", "L1"]); + await vi.waitFor(() => expect(slow.webSocket.connections).toHaveLength(1)); + firstPeer(slow).receive(READY); + await vi.waitFor(() => expect(hung.calls).toHaveLength(1)); + slow.clock.advance(9_999); + await settle(); + expect(slow.err).toEqual([]); + slow.clock.advance(1); + await vi.waitFor(() => + expect(slow.err.join("")).toBe( + `warning: emulator adb: adb did not answer in 10 s; ${rerun}\n`, + ), + ); + expect((await hung.handles[0]?.wait())?.code).toBeNull(); + slow.signals.emit("SIGINT"); + slow.clock.advance(2_000); + expect(await slowExit).toBe(0); + }); +}); diff --git a/src/cli/tunnel.test.ts b/src/cli/tunnel.test.ts index 8a01d66c1..a285b0b0c 100644 --- a/src/cli/tunnel.test.ts +++ b/src/cli/tunnel.test.ts @@ -1,117 +1,6 @@ import { describe, expect, it, vi } from "vitest"; - -import { - TUNNEL_MAX_DATA_BYTES, - TUNNEL_TAG, - decodeTunnelFrame, - encodeTunnelFrame, -} from "../contract/index.js"; -import { promiseState } from "../test-support/promise-state.js"; -import { buildCliEnvironment, runCli } from "./index.js"; -import { - FakeClock, - FakeDaemonLauncher, - FakeHttpJsonClient, - FakeSystemStats, - FakeTcpListenerFactory, - FakeTcpPeer, - FakeTcpStreamConnector, - FakeWebSocketClientConnector, - WebSocketRefusedError, - MemoryFilesystem, - MemoryIpcTransport, - ProcessSpawnError, - ScriptedProcessRunner, - type ScriptedProcessExpectation, -} from "../ports/index.js"; - -type Listener = () => void; - -// why: the CLI's signal source, so a test sends Ctrl-C without signalling the test process. -class TestSignals { - readonly #listeners = new Map>(); - on(signal: string, listener: Listener): this { - const set = this.#listeners.get(signal) ?? new Set(); - set.add(listener); - this.#listeners.set(signal, set); - return this; - } - off(signal: string, listener: Listener): this { - this.#listeners.get(signal)?.delete(listener); - return this; - } - emit(signal: string): void { - for (const listener of this.#listeners.get(signal) ?? []) listener(); - } -} - -// why: stands in for the caller's machine: only the commands in `present` start, as a real runner throws ProcessSpawnError for an adb that is not there. -class CallerAdb extends ScriptedProcessRunner { - constructor( - expectations: readonly ScriptedProcessExpectation[], - private readonly present: readonly string[] = ["adb"], - ) { - super(expectations); - } - - override spawn( - command: string, - args: readonly string[], - options?: Parameters[2], - ) { - if (!this.present.includes(command)) throw new ProcessSpawnError(command, args); - return super.spawn(command, args, options); - } -} - -function command(env: NodeJS.ProcessEnv, processRunner: CallerAdb = new CallerAdb([])) { - const signals = new TestSignals(); - const out: string[] = []; - const err: string[] = []; - const webSocket = new FakeWebSocketClientConnector(); - const httpJson = new FakeHttpJsonClient(); - const clock = new FakeClock(0); - const tcpStreams = new FakeTcpStreamConnector(); - const tcpListeners = new FakeTcpListenerFactory(); - const environment = buildCliEnvironment( - { - clock, - daemonHome: { dataDirectory: "/unused", gatewayPresent: false, role: "worker" }, - filesystem: new MemoryFilesystem(), - homeDirectory: "/home/caller", - hostPlatform: "linux", - httpJson, - ipc: new MemoryIpcTransport(), - launcher: new FakeDaemonLauncher(), - processRunner, - runTool: async () => ({ outcome: "not-found" }), - signals, - stderr: { write: (value) => err.push(value) }, - stdout: { write: (value) => out.push(value) }, - systemStats: new FakeSystemStats({ cpuCount: 2, totalRamBytes: 2 ** 30 }), - tcpListeners, - tcpStreams, - webSocket, - workingDirectory: "/work", - }, - env, - ); - return { - clock, - err, - httpJson, - out, - run: (argv: readonly string[]) => runCli(argv, environment), - signals, - tcpListeners, - tcpStreams, - webSocket, - }; -} - -const READY = encodeTunnelFrame(TUNNEL_TAG.ready, 0, "{}"); - -const ENV = { SIMLOCK_TOKEN: "sk_agent_x", SIMLOCK_URL: "http://127.0.0.1:7300" }; +import { TUNNEL_TAG, decodeTunnelFrame } from "../contract/index.js"; +import { ENV, READY, command } from "./test-tunnel.js"; describe("simlock tunnel that cannot run", () => { it("exits 2 with a USAGE line, before it connects anywhere, for each bad invocation", async () => { @@ -421,928 +310,3 @@ describe("simlock tunnel that runs", () => { } }); }); - -function open(conn: number, devicePort: number): Uint8Array { - return encodeTunnelFrame(TUNNEL_TAG.open, conn, JSON.stringify({ devicePort, kind: "reverse" })); -} - -function sentFrames(cli: ReturnType) { - return (cli.webSocket.connections[0]?.peer.sent ?? []).flatMap((bytes) => { - const frame = decodeTunnelFrame(bytes); - return frame === undefined ? [] : [frame]; - }); -} - -async function started(argv: readonly string[]) { - const cli = command(ENV); - const exit = cli.run(["tunnel", ...argv, "--lease", "L1"]); - await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); - const peer = cli.webSocket.connections[0]?.peer; - if (peer === undefined) throw new Error("no socket"); - peer.receive(READY); - return { cli, exit, peer }; -} - -async function warningsPerTunnel(): Promise { - const quiet = await started(["reverse", "5173:5174", "reverse", "6000"]); - quiet.cli.tcpStreams.refusedPorts.add(5174); - quiet.cli.tcpStreams.refusedPorts.add(6000); - const opening = async (conn: number, port: number) => { - quiet.peer.receive(open(conn, port)); - await vi.waitFor(() => - expect(sentFrames(quiet.cli).filter((f) => f.tag === TUNNEL_TAG.reset)).toHaveLength( - (conn + 1) / 2, - ), - ); - }; - const warnings = () => - quiet.cli.err - .join("") - .split("\n") - .filter((line) => line !== ""); - quiet.peer.receive(open(9, 7000)); - await opening(1, 5173); - expect(warnings()).toEqual(["reverse 5173: nothing listening on localhost:5174"]); - quiet.cli.clock.advance(9_999); - await opening(3, 5173); - expect(warnings()).toEqual(["reverse 5173: nothing listening on localhost:5174"]); - await opening(5, 6000); - expect(warnings()).toEqual([ - "reverse 5173: nothing listening on localhost:5174", - "reverse 6000: nothing listening on localhost:6000", - ]); - quiet.cli.clock.advance(1); - await opening(7, 5173); - expect(warnings()).toHaveLength(3); - expect(warnings()[2]).toBe("reverse 5173: nothing listening on localhost:5174"); - expect( - sentFrames(quiet.cli) - .filter((f) => f.tag === TUNNEL_TAG.reset) - .map((f) => f.conn), - ).toEqual([1, 3, 5, 7]); - expect(quiet.cli.tcpStreams.connections).toEqual([]); - expect(quiet.peer.closedWith).toBeUndefined(); - quiet.cli.signals.emit("SIGINT"); - expect(await quiet.exit).toBe(0); -} - -async function forwardResetWarnings(): Promise { - const cli = command(ENV); - const exit = cli.run(["tunnel", "forward", "9229:19229", "--lease", "L1"]); - await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); - const peer = firstPeer(cli); - peer.receive(READY); - const reason = "nothing on the lease's simulator listens on port 9229"; - const warnings = () => - cli.err - .join("") - .split("\n") - .filter((text) => text !== ""); - const resetByWorker = async (conn: number) => { - cli.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => expect(sentFrames(cli).at(-1)?.conn).toBe(conn)); - peer.receive(encodeTunnelFrame(TUNNEL_TAG.reset, conn, JSON.stringify({ reason }))); - await settle(); - }; - await resetByWorker(2); - expect(warnings()).toEqual([`warning: forward 9229: ${reason}`]); - cli.clock.advance(9_999); - await resetByWorker(4); - expect(warnings()).toHaveLength(1); - cli.clock.advance(1); - await resetByWorker(6); - expect(warnings()).toEqual([ - `warning: forward 9229: ${reason}`, - `warning: forward 9229: ${reason}`, - ]); - expect(peer.closedWith).toBeUndefined(); - cli.signals.emit("SIGINT"); - expect(await exit).toBe(0); -} - -async function responseThroughTheTunnel() { - const page = await started(["reverse", "5173:5174"]); - page.peer.receive(open(1, 5173)); - await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(1)); - expect(page.cli.tcpStreams.connections[0]?.port).toBe(5174); - const local = page.cli.tcpStreams.connections[0]?.peer; - if (local === undefined) throw new Error("no local connection"); - const response = Uint8Array.from({ length: 1_048_576 }, (_, index) => index % 251); - for (let at = 0; at < response.length; at += 131_072) - local.send(response.subarray(at, at + 131_072)); - local.end(); - const received: number[] = []; - let ended = false; - let before = 0; - const credit = (conn: number, size: number) => { - const granted = new Uint8Array(4); - new DataView(granted.buffer).setUint32(0, size); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.credit, conn, granted)); - }; - const drain = () => { - const frames = sentFrames(page.cli); - for (const frame of frames.slice(before)) { - if (frame.tag === TUNNEL_TAG.data) for (const byte of frame.payload) received.push(byte); - else if (frame.tag === TUNNEL_TAG.end) ended = true; - } - before = frames.length; - }; - await vi.waitFor(() => { - drain(); - expect(received.length).toBe(262_144); - }); - await new Promise((done) => setTimeout(done, 20)); - drain(); - expect(received).toHaveLength(262_144); - while (!ended) { - const have = received.length; - credit(1, 65_536); - await vi.waitFor(() => { - drain(); - expect(received.length > have || ended).toBe(true); - }); - } - expect(received).toHaveLength(1_048_576); - expect(Uint8Array.from(received)).toEqual(response); - const dataSizes = sentFrames(page.cli) - .filter((f) => f.tag === TUNNEL_TAG.data) - .map((f) => f.payload.length); - expect(Math.max(...dataSizes)).toBe(TUNNEL_MAX_DATA_BYTES); - expect(dataSizes.every((size) => size <= TUNNEL_MAX_DATA_BYTES)).toBe(true); - - return { local, page }; -} - -async function writesToTheCaller( - page: Awaited>, - local: FakeTcpPeer, -): Promise { - for (const size of [65_536, 30_000]) { - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 1, new Uint8Array(size))); - } - await vi.waitFor(() => - expect(local.written.map((chunk) => chunk.length)).toEqual([65_536, 30_000]), - ); - await vi.waitFor(() => { - const credits = sentFrames(page.cli) - .filter((f) => f.tag === TUNNEL_TAG.credit) - .map((f) => new DataView(f.payload.buffer, f.payload.byteOffset).getUint32(0)); - expect(credits).toEqual([65_536, 30_000]); - }); - expect(local.closed).toBe(false); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 1)); - await vi.waitFor(() => expect(local.writeEnded).toBe(true)); - await vi.waitFor(() => expect(local.closed).toBe(true)); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 1, new Uint8Array(5))); - await settle(); - expect(local.written.map((chunk) => chunk.length)).toEqual([65_536, 30_000]); - page.peer.receive(open(1, 5173)); - await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(2)); - const second = page.cli.tcpStreams.connections[1]?.peer; - expect(second?.closed).toBe(false); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 1)); - await vi.waitFor(() => expect(second?.writeEnded).toBe(true)); - await settle(); - expect(second?.closed).toBe(false); - second?.end(); - await vi.waitFor(() => expect(second?.closed).toBe(true)); - page.peer.receive(open(3, 5173)); - await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(3)); - const reset = page.cli.tcpStreams.connections[2]?.peer; - expect(reset?.closed).toBe(false); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.reset, 3)); - await vi.waitFor(() => expect(reset?.closed).toBe(true)); -} - -async function brokenWrite(page: Awaited>): Promise { - page.peer.receive(open(5, 5173)); - await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(4)); - const broken = page.cli.tcpStreams.connections[3]?.peer; - if (broken === undefined) throw new Error("no fourth connection"); - broken.writeFailure = new Error("write EPIPE"); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.data, 5, new Uint8Array(10))); - page.peer.receive(encodeTunnelFrame(TUNNEL_TAG.end, 5)); - await vi.waitFor(() => - expect( - sentFrames(page.cli) - .filter((f) => f.tag === TUNNEL_TAG.reset) - .map((f) => f.conn), - ).toEqual([5]), - ); - await settle(); - expect(broken.closed).toBe(true); - expect(sentFrames(page.cli).filter((f) => f.tag === TUNNEL_TAG.credit && f.conn === 5)).toEqual( - [], - ); - expect(page.cli.err).toEqual([]); - page.peer.receive(open(7, 5173)); - await vi.waitFor(() => expect(page.cli.tcpStreams.connections).toHaveLength(5)); - expect(page.cli.tcpStreams.connections[4]?.peer.closed).toBe(false); - page.cli.signals.emit("SIGINT"); - expect(await page.exit).toBe(0); -} - -describe("simlock tunnel carrying connections", () => { - it("warns once per 10 s per tunnel when nothing listens, or the worker resets a forward connection with its reason, keeps the tunnel open, and a 1 MiB response arrives whole, a connection whose both halves ended is closed and dropped, and one with a single half ended keeps relaying the other", async () => { - await warningsPerTunnel(); - await forwardResetWarnings(); - const { local, page } = await responseThroughTheTunnel(); - await writesToTheCaller(page, local); - await brokenWrite(page); - }); -}); - -// why: a failure reaches stderr through a chain of promises, so a check that nothing was written lets them run first. -function settle(): Promise { - return new Promise((done) => setTimeout(done, 0)); -} - -async function withLocalConnection( - cli: ReturnType, - peer: { receive(bytes: Uint8Array): void }, - devicePort: number, -) { - const before = cli.tcpStreams.connections.length; - peer.receive(open(1 + 2 * before, devicePort)); - await vi.waitFor(() => expect(cli.tcpStreams.connections).toHaveLength(before + 1)); - const local = cli.tcpStreams.connections[before]?.peer; - if (local === undefined) throw new Error("no local connection"); - expect(local.closed).toBe(false); - return local; -} - -function refusedFrame(code: string, message: string): Uint8Array { - return encodeTunnelFrame(TUNNEL_TAG.refused, 0, JSON.stringify({ code, message })); -} - -function errorBody(code: string, message: string) { - return { error: { code, message } }; -} - -function line(cli: ReturnType): unknown { - return JSON.parse(cli.err.join("")); -} - -async function upgradeRefusals(): Promise { - const upgrades: readonly { - readonly refusal: WebSocketRefusedError; - readonly exit?: number; - readonly code: string; - readonly message: string; - }[] = [ - { - code: "UNAUTHENTICATED", - message: "No valid token", - refusal: new WebSocketRefusedError( - 401, - JSON.stringify(errorBody("UNAUTHENTICATED", "No valid token")), - ), - }, - { - code: "UNKNOWN_LEASE", - message: "Unknown lease L1", - refusal: new WebSocketRefusedError( - 404, - JSON.stringify(errorBody("UNKNOWN_LEASE", "Unknown lease L1")), - ), - }, - { - code: "TUNNEL_UNAVAILABLE", - message: "Tunnels are off", - refusal: new WebSocketRefusedError( - 503, - JSON.stringify(errorBody("TUNNEL_UNAVAILABLE", "Tunnels are off")), - ), - }, - { - code: "UNKNOWN_LEASE", - message: "Lease L1 is not known to this worker", - refusal: new WebSocketRefusedError(404, "Not Found"), - }, - { - code: "BAD_REQUEST", - exit: 2, - message: "Bad upgrade", - refusal: new WebSocketRefusedError( - 400, - JSON.stringify(errorBody("BAD_REQUEST", "Bad upgrade")), - ), - }, - { - code: "TUNNEL_DISCONNECTED", - message: "Lease L1: the worker answered 502", - refusal: new WebSocketRefusedError(502, "Bad Gateway"), - }, - ]; - for (const { code, exit, message, refusal } of upgrades) { - const cli = command(ENV); - cli.webSocket.failure = refusal; - expect(await cli.run(["tunnel", "reverse", "5173", "--lease", "L1"])).toBe(exit ?? 1); - expect(line(cli)).toEqual(errorBody(code, message)); - } -} - -async function lookupFailures(): Promise { - const lookups: readonly { - readonly identity?: { status: number; body: unknown }; - readonly code: string; - readonly message: string; - }[] = [ - { - code: "UNAUTHENTICATED", - identity: { body: errorBody("UNAUTHENTICATED", "No valid token"), status: 401 }, - message: "No valid token", - }, - { - code: "FORBIDDEN", - identity: { body: errorBody("FORBIDDEN", "Not allowed"), status: 403 }, - message: "Not allowed", - }, - { - code: "TUNNEL_DISCONNECTED", - identity: { body: undefined, status: 502 }, - message: "The worker answered 502", - }, - { - code: "TUNNEL_DISCONNECTED", - message: "No answer from http://127.0.0.1:7300", - }, - ]; - for (const { code, identity, message } of lookups) { - const cli = command(ENV); - if (identity !== undefined) - cli.httpJson.answers.set("http://127.0.0.1:7300/v1/identity", identity); - expect(await cli.run(["tunnel", "reverse", "5173"])).toBe(1); - expect(line(cli)).toEqual(errorBody(code, message)); - expect(cli.webSocket.connections).toEqual([]); - } -} - -async function refusalFrame(): Promise { - const refused = await started(["reverse", "80"]); - const refusedLocal = await withLocalConnection(refused.cli, refused.peer, 80); - refused.peer.receive(refusedFrame("TUNNEL_BAD_PORT", "Device port 80 is below 1024")); - expect(await refused.exit).toBe(1); - expect(refusedLocal.closed).toBe(true); - expect(refused.peer.closedWith?.code).toBe(1000); - expect(refused.cli.clock.pendingTimerCount).toBe(0); - expect(line(refused.cli)).toEqual( - errorBody("TUNNEL_BAD_PORT", "Lease L1: Device port 80 is below 1024"), - ); - const unknown = await started(["reverse", "5173"]); - unknown.peer.receive(refusedFrame("NOT_A_CODE", "something new")); - expect(await unknown.exit).toBe(1); - expect(line(unknown.cli)).toEqual(errorBody("TUNNEL_FAILED", "Lease L1: something new")); - const garbled = await started(["reverse", "5173"]); - garbled.peer.receive(open(1, 7000)); - garbled.peer.receive(encodeTunnelFrame(TUNNEL_TAG.open, 3, "not json")); - garbled.peer.receive(encodeTunnelFrame(TUNNEL_TAG.refused, 0, "not json")); - expect(await garbled.exit).toBe(1); - expect(line(garbled.cli)).toEqual( - errorBody("TUNNEL_FAILED", "Lease L1: the worker refused the tunnel"), - ); - expect(garbled.cli.tcpStreams.connections).toEqual([]); -} - -async function lossesOfTheSocket(): Promise { - const losses: readonly { - readonly drop: [number, string]; - readonly exit: number; - readonly error: unknown; - }[] = [ - { - drop: [4001, "device released"], - error: errorBody("TUNNEL_LEASE_ENDED", "Lease L1 ended: device released"), - exit: 14, - }, - ...[1011, 1001, 1006].map((code) => ({ - drop: [code, ""] as [number, string], - error: errorBody( - "TUNNEL_DISCONNECTED", - `Lease L1: the connection to ${ENV.SIMLOCK_URL} was lost`, - ), - exit: 1, - })), - ]; - for (const { drop, error, exit } of losses) { - const lost = await started(["reverse", "5173"]); - const lostLocal = await withLocalConnection(lost.cli, lost.peer, 5173); - lost.peer.dropped(...drop); - expect(await lost.exit).toBe(exit); - expect(lostLocal.closed).toBe(true); - expect(lost.cli.clock.pendingTimerCount).toBe(0); - expect(line(lost.cli)).toEqual(error); - } -} - -async function silentSocket(): Promise { - const silent = await started(["reverse", "5173"]); - silent.cli.clock.advance(30_000); - silent.peer.ping(); - silent.cli.clock.advance(44_999); - await settle(); - expect(silent.cli.err).toEqual([]); - const silentLocal = await withLocalConnection(silent.cli, silent.peer, 5173); - silent.cli.clock.advance(44_999); - await settle(); - expect(silent.cli.err).toEqual([]); - silent.cli.clock.advance(1); - await settle(); - expect(silent.cli.err).toHaveLength(1); - expect(await silent.exit).toBe(1); - expect(silentLocal.closed).toBe(true); - expect(silent.peer.closedWith?.code).toBe(1000); - expect(silent.peer.terminated).toBe(true); - expect(silent.cli.clock.pendingTimerCount).toBe(0); - expect(line(silent.cli)).toEqual( - errorBody("TUNNEL_DISCONNECTED", "Lease L1: nothing was heard from the worker for 45 s"), - ); -} - -async function noReadySocket(): Promise { - const noReady = command(ENV); - const noReadyExit = noReady.run(["tunnel", "reverse", "5173", "--lease", "L1"]); - await vi.waitFor(() => expect(noReady.webSocket.connections).toHaveLength(1)); - const quietPeer = noReady.webSocket.connections[0]?.peer; - for (let at = 0; at < 89_999; at += 15_000) { - noReady.clock.advance(Math.min(15_000, 89_999 - at)); - quietPeer?.ping(); - } - await settle(); - expect(noReady.err).toEqual([]); - noReady.clock.advance(1); - await settle(); - expect(noReady.err).toHaveLength(1); - expect(await noReadyExit).toBe(1); - expect(quietPeer?.closedWith?.code).toBe(1000); - expect(quietPeer?.terminated).toBe(true); - expect(noReady.clock.pendingTimerCount).toBe(0); - expect(line(noReady)).toEqual( - errorBody("TUNNEL_DISCONNECTED", "Lease L1: no READY from the worker within 90 s"), - ); -} - -async function ctrlC(): Promise { - const idle = await started(["reverse", "5173"]); - for (let at = 0; at < 90_000; at += 15_000) { - idle.cli.clock.advance(15_000); - idle.peer.ping(); - } - await settle(); - expect(idle.cli.err).toEqual([]); - expect(idle.peer.closedWith).toBeUndefined(); - const idleLocal = await withLocalConnection(idle.cli, idle.peer, 5173); - idle.peer.silentOnClose = true; - idle.cli.signals.emit("SIGINT"); - expect(idle.peer.closedWith?.code).toBe(1000); - let settled = false; - void idle.exit.then(() => { - settled = true; - }); - idle.cli.clock.advance(1_999); - await settle(); - expect(settled).toBe(false); - idle.cli.clock.advance(1); - await settle(); - expect(settled).toBe(true); - expect(await idle.exit).toBe(0); - expect(idleLocal.closed).toBe(true); - expect(idle.peer.terminated).toBe(true); - expect(idle.cli.clock.pendingTimerCount).toBe(0); -} - -async function sigterm(): Promise { - const term = await started(["reverse", "5173"]); - const termLocal = await withLocalConnection(term.cli, term.peer, 5173); - term.cli.signals.emit("SIGTERM"); - expect(term.peer.closedWith?.code).toBe(1000); - expect(await term.exit).toBe(0); - expect(termLocal.closed).toBe(true); - expect(term.cli.clock.pendingTimerCount).toBe(0); -} - -describe("simlock tunnel that is refused or loses its worker", () => { - it("exits with the worker's code and one JSON line, and a Ctrl-C closes and waits at most 2 s", async () => { - await upgradeRefusals(); - await lookupFailures(); - await refusalFrame(); - await lossesOfTheSocket(); - await silentSocket(); - await noReadySocket(); - await ctrlC(); - await sigterm(); - }); -}); - -describe("simlock tunnel forward listening on the caller's loopback", () => { - it("binds each caller port before HELLO, holds connections until READY, announces them with even ids and closes every listener however the command ends, and sends a device's OPEN to the reverse tunnel when a forward names the same device port", async () => { - const busy = command(ENV); - busy.tcpListeners.busy.add(9229); - expect(await busy.run(["tunnel", "forward", "9229", "--lease", "L1"])).toBe(1); - expect(line(busy)).toEqual( - errorBody("TUNNEL_CALLER_PORT_IN_USE", "forward 9229: localhost:9229 is already in use"), - ); - expect(busy.webSocket.connections).toEqual([]); - - const waiting = command(ENV); - const exit = waiting.run([ - "tunnel", - "forward", - "9229:19229", - "reverse", - "9229:5173", - "--lease", - "L1", - ]); - await vi.waitFor(() => expect(waiting.webSocket.connections).toHaveLength(1)); - const peer = firstPeer(waiting); - expect(waiting.tcpListeners.listeners.map((listener) => listener.port)).toEqual([19229]); - const early = waiting.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => - expect(sentFrames(waiting).map((f) => f.tag)).toEqual([TUNNEL_TAG.hello]), - ); - expect(early?.closed).toBe(false); - peer.receive(READY); - await vi.waitFor(() => - expect(sentFrames(waiting).map((f) => [f.tag, f.conn])).toEqual([ - [TUNNEL_TAG.hello, 0], - [TUNNEL_TAG.open, 2], - ]), - ); - expect(JSON.parse(new TextDecoder().decode(sentFrames(waiting)[1]?.payload))).toEqual({ - devicePort: 9229, - kind: "forward", - }); - waiting.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => expect(sentFrames(waiting).at(-1)?.conn).toBe(4)); - peer.receive(open(1, 9229)); - await vi.waitFor(() => expect(waiting.tcpStreams.connections).toHaveLength(1)); - expect(waiting.tcpStreams.connections[0]?.port).toBe(5173); - waiting.signals.emit("SIGINT"); - expect(await exit).toBe(0); - - const refusedEarly = command(ENV); - const refusedExit = refusedEarly.run(["tunnel", "forward", "9229", "--lease", "L1"]); - await vi.waitFor(() => expect(refusedEarly.webSocket.connections).toHaveLength(1)); - const queued = refusedEarly.tcpListeners.listeners[0]?.connect(); - refusedEarly.webSocket.connections[0]?.peer.receive( - refusedFrame("TUNNEL_BAD_PORT", "Device port 80 is below 1024"), - ); - expect(await refusedExit).toBe(1); - expect(queued?.closed).toBe(true); - expect(sentFrames(refusedEarly).map((f) => f.tag)).toEqual([TUNNEL_TAG.hello]); - - const slow = command(ENV); - let releaseLookup: () => void = () => undefined; - const lookup = new Promise((resolve) => { - releaseLookup = resolve; - }); - slow.httpJson.get = async () => { - await lookup; - throw new Error("the worker is unreachable"); - }; - const slowExit = slow.run(["tunnel", "forward", "9229"]); - await vi.waitFor(() => expect(slow.tcpListeners.listeners).toHaveLength(1)); - const beforeSocket = slow.tcpListeners.listeners[0]?.connect(); - expect(beforeSocket?.closed).toBe(false); - releaseLookup(); - expect(await slowExit).toBe(1); - expect(beforeSocket?.closed).toBe(true); - - const endings: readonly ((cli: ReturnType, peer: Peer) => void)[] = [ - (cli) => cli.signals.emit("SIGINT"), - (_cli, ended) => ended.receive(refusedFrame("TUNNEL_BAD_PORT", "no")), - (_cli, ended) => ended.dropped(4001, "device released"), - (_cli, ended) => ended.dropped(1006, ""), - ]; - for (const ending of endings) await listenersCloseAfter(ending); - }); -}); - -function firstPeer(cli: ReturnType): Peer { - const peer = cli.webSocket.connections[0]?.peer; - if (peer === undefined) throw new Error("no socket"); - return peer; -} - -async function listenersCloseAfter( - ending: (cli: ReturnType, peer: Peer) => void, -): Promise { - const cli = command(ENV); - const run = cli.run(["tunnel", "forward", "9229", "forward", "9230", "--lease", "L1"]); - await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); - const connection = firstPeer(cli); - connection.silentOnClose = true; - expect(cli.tcpListeners.listeners.map((listener) => listener.closed)).toEqual([false, false]); - ending(cli, connection); - cli.clock.advance(2_000); - await run; - expect(cli.tcpListeners.listeners.map((listener) => listener.closed)).toEqual([true, true]); -} - -type Peer = ReturnType["webSocket"]["connections"][number]["peer"]; - -describe("simlock tunnel emulator adb", () => { - const emulatorOpen = JSON.stringify({ kind: "emulator", emulatorPort: "adb" }); - - // fallow-ignore-next-line complexity -- one Tests line: each adb outcome and each way the command ends are rows of its cases. - it("prints the tunnel line and connects the caller's own adb to the caller's port, which opens an emulator connection for each connection to it", async () => { - const adb = new CallerAdb([ - { - match: { args: ["connect", "127.0.0.1:49152"], command: "adb" }, - stdoutLines: ["connected to 127.0.0.1:49152"], - }, - ]); - const cli = command(ENV, adb); - const exit = cli.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(cli.webSocket.connections).toHaveLength(1)); - await vi.waitFor(() => expect(sentFrames(cli)).toHaveLength(1)); - expect(new TextDecoder().decode(sentFrames(cli)[0]?.payload)).toBe( - '{"tunnels":[{"kind":"emulator","emulatorPort":"adb"}]}', - ); - expect(cli.out).toEqual([]); - - firstPeer(cli).receive(READY); - await vi.waitFor(() => - expect(cli.out.join("")).toBe( - "emulator localhost:49152 -> emulator:adb (lease L1)\nadb: connected to 127.0.0.1:49152\n", - ), - ); - expect(adb.calls.map(({ args, command: bin }) => [bin, ...args])).toEqual([ - ["adb", "connect", "127.0.0.1:49152"], - ]); - - cli.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => expect(sentFrames(cli).at(-1)?.conn).toBe(2)); - expect(sentFrames(cli).at(-1)?.tag).toBe(TUNNEL_TAG.open); - expect(new TextDecoder().decode(sentFrames(cli).at(-1)?.payload)).toBe(emulatorOpen); - - cli.signals.emit("SIGINT"); - cli.clock.advance(2_000); - expect(await exit).toBe(0); - - const target = "127.0.0.1:49152"; - const connect = ( - command: string, - output: Partial = {}, - ): ScriptedProcessExpectation => ({ - match: { args: ["connect", target], command }, - stdoutLines: [`connected to ${target}`], - ...output, - }); - const rerun = `run: adb connect ${target}`; - const outputs = [ - { - adb: new CallerAdb([ - connect("adb", { - stdoutLines: [], - stderrLines: ["* daemon not running; starting now"], - result: { - code: 0, - stderr: "", - stdout: `* daemon started successfully\nconnected to ${target}\n`, - }, - }), - ]), - err: "", - out: `adb: connected to ${target}\n`, - }, - { - adb: new CallerAdb([connect("adb", { stdoutLines: [`already connected to ${target}`] })]), - err: "", - out: `adb: connected to ${target}\n`, - }, - { - adb: new CallerAdb([ - connect("adb", { stdoutLines: [`failed to authenticate to ${target}`] }), - ]), - err: `warning: emulator adb: adb answered: failed to authenticate to ${target}; ${rerun}\n`, - out: "", - }, - { - adb: new CallerAdb([ - connect("adb", { - stdoutLines: [], - result: { - code: 1, - stderr: "", - stdout: `* daemon started successfully\nfailed to authenticate to ${target}\n`, - }, - }), - ]), - err: `warning: emulator adb: adb answered: failed to authenticate to ${target}; ${rerun}\n`, - out: "", - }, - { - adb: new CallerAdb([], []), - err: `warning: emulator adb: adb was not found; ${rerun}\n`, - out: "", - }, - { - adb: new CallerAdb([connect("/sdk/platform-tools/adb")], ["/sdk/platform-tools/adb"]), - env: { ANDROID_HOME: "/sdk" }, - err: "", - out: `adb: connected to ${target}\n`, - }, - { - adb: new CallerAdb( - [connect("/home/caller/Library/Android/sdk/platform-tools/adb")], - ["/home/caller/Library/Android/sdk/platform-tools/adb"], - ), - err: "", - out: `adb: connected to ${target}\n`, - }, - ]; - for (const expected of outputs) { - const other = command({ ...ENV, ...expected.env }, expected.adb); - const otherExit = other.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(other.webSocket.connections).toHaveLength(1)); - firstPeer(other).receive(READY); - await vi.waitFor(() => - expect(other.out.join("") + other.err.join("")).toContain( - expected.out === "" ? expected.err : expected.out, - ), - ); - await settle(); - expect(other.out.join("")).toBe( - `emulator localhost:49152 -> emulator:adb (lease L1)\n${expected.out}`, - ); - expect(other.err.join("")).toBe(expected.err); - other.signals.emit("SIGINT"); - other.clock.advance(2_000); - expect(await otherExit).toBe(0); - } - - const pinned = new CallerAdb([ - { - match: { args: ["connect", "127.0.0.1:6000"], command: "adb" }, - stdoutLines: ["connected to 127.0.0.1:6000"], - }, - ]); - const both = command(ENV, pinned); - const bothExit = both.run(["tunnel", "emulator", "adb:6000", "--lease", "L1"]); - await vi.waitFor(() => expect(both.webSocket.connections).toHaveLength(1)); - expect(both.tcpListeners.listeners.map((listener) => listener.port)).toEqual([6000]); - firstPeer(both).receive(READY); - await vi.waitFor(() => - expect(both.out.join("")).toBe( - "emulator localhost:6000 -> emulator:adb (lease L1)\nadb: connected to 127.0.0.1:6000\n", - ), - ); - both.signals.emit("SIGINT"); - both.clock.advance(2_000); - expect(await bothExit).toBe(0); - - const busy = command(ENV, new CallerAdb([])); - busy.tcpListeners.busy.add(6000); - expect(await busy.run(["tunnel", "emulator", "adb:6000", "--lease", "L1"])).toBe(1); - expect(line(busy)).toEqual( - errorBody("TUNNEL_CALLER_PORT_IN_USE", "emulator adb: localhost:6000 is already in use"), - ); - - const mixedAdb = new CallerAdb([connect("adb")]); - const mixed = command(ENV, mixedAdb); - const mixedExit = mixed.run(["tunnel", "reverse", "8081", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(sentFrames(mixed)).toHaveLength(1)); - expect(new TextDecoder().decode(sentFrames(mixed)[0]?.payload)).toBe( - '{"tunnels":[{"devicePort":8081,"kind":"reverse"},{"kind":"emulator","emulatorPort":"adb"}]}', - ); - firstPeer(mixed).receive(READY); - await vi.waitFor(() => - expect(mixed.out.join("")).toBe( - `reverse device:8081 -> localhost:8081 (lease L1)\nemulator localhost:49152 -> emulator:adb (lease L1)\nadb: connected to ${target}\n`, - ), - ); - firstPeer(mixed).receive(open(1, 8081)); - await vi.waitFor(() => - expect(mixed.tcpStreams.connections.map(({ port }) => port)).toEqual([8081]), - ); - - mixed.tcpListeners.listeners[0]?.connect(); - mixed.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => - expect(sentFrames(mixed).filter((f) => f.tag === TUNNEL_TAG.open)).toHaveLength(2), - ); - const reset = (conn: number, reason: string) => - encodeTunnelFrame(TUNNEL_TAG.reset, conn, JSON.stringify({ reason })); - firstPeer(mixed).receive(reset(2, "closed")); - firstPeer(mixed).receive(reset(4, "timeout")); - await vi.waitFor(() => expect(mixed.err.join("")).toBe("warning: emulator adb: closed\n")); - mixed.clock.advance(9_999); - mixed.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => expect(sentFrames(mixed).at(-1)?.conn).toBe(6)); - firstPeer(mixed).receive(reset(6, "timeout")); - await settle(); - expect(mixed.err.join("")).toBe("warning: emulator adb: closed\n"); - mixed.clock.advance(1); - mixed.tcpListeners.listeners[0]?.connect(); - await vi.waitFor(() => expect(sentFrames(mixed).at(-1)?.conn).toBe(8)); - firstPeer(mixed).receive(reset(8, "timeout")); - await vi.waitFor(() => - expect(mixed.err.join("")).toBe( - "warning: emulator adb: closed\nwarning: emulator adb: timeout\n", - ), - ); - mixed.signals.emit("SIGINT"); - mixed.clock.advance(2_000); - expect(await mixedExit).toBe(0); - - const disconnect = ( - output: Partial = {}, - ): ScriptedProcessExpectation => ({ - match: { args: ["disconnect", target], command: "adb" }, - stdoutLines: [`disconnected ${target}`], - ...output, - }); - const calls = (runner: CallerAdb) => runner.calls.map(({ args }) => args.join(" ")); - const endings: readonly { - readonly end: (cli: ReturnType, peer: Peer) => void; - readonly exit: number; - }[] = [ - { end: (cli) => cli.signals.emit("SIGINT"), exit: 0 }, - { end: (_cli, ended) => ended.receive(refusedFrame("TUNNEL_BAD_PORT", "no")), exit: 1 }, - { end: (_cli, ended) => ended.dropped(4001, "device released"), exit: 14 }, - { end: (_cli, ended) => ended.dropped(1006, ""), exit: 1 }, - ]; - for (const ending of endings) { - const runner = new CallerAdb([connect("adb"), disconnect()]); - const leaving = command(ENV, runner); - const leavingExit = leaving.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(leaving.webSocket.connections).toHaveLength(1)); - firstPeer(leaving).receive(READY); - await vi.waitFor(() => expect(leaving.out.join("")).toContain("adb: connected")); - ending.end(leaving, firstPeer(leaving)); - expect(await leavingExit).toBe(ending.exit); - expect(calls(runner)).toEqual([`connect ${target}`, `disconnect ${target}`]); - } - - const failing = new CallerAdb([ - connect("adb"), - disconnect({ result: { code: 1, stderr: "error: no such device", stdout: "" } }), - ]); - const failed = command(ENV, failing); - const failedExit = failed.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(failed.webSocket.connections).toHaveLength(1)); - firstPeer(failed).receive(READY); - await vi.waitFor(() => expect(failed.out.join("")).toContain("adb: connected")); - firstPeer(failed).dropped(4001, "device released"); - expect(await failedExit).toBe(14); - expect(calls(failing)).toEqual([`connect ${target}`, `disconnect ${target}`]); - - const stuckAdb = new CallerAdb([connect("adb"), disconnect({ hangs: true })]); - const stuck = command(ENV, stuckAdb); - const stuckExit = stuck.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(stuck.webSocket.connections).toHaveLength(1)); - firstPeer(stuck).receive(READY); - await vi.waitFor(() => expect(stuck.out.join("")).toContain("adb: connected")); - stuck.signals.emit("SIGINT"); - await vi.waitFor(() => expect(stuckAdb.calls).toHaveLength(2)); - const stuckState = promiseState(stuckExit); - stuck.clock.advance(1_999); - await settle(); - expect(stuckState.state).toBe("pending"); - stuck.clock.advance(1); - await settle(); - expect(stuckState.state).toBe("fulfilled"); - expect(await stuckExit).toBe(0); - expect((await stuckAdb.handles[1]?.wait())?.code).toBeNull(); - - const early = new CallerAdb([connect("adb", { hangs: true }), disconnect()]); - const interrupted = command(ENV, early); - const interruptedExit = interrupted.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(interrupted.webSocket.connections).toHaveLength(1)); - firstPeer(interrupted).receive(READY); - await vi.waitFor(() => expect(early.calls).toHaveLength(1)); - interrupted.signals.emit("SIGINT"); - const interruptedState = promiseState(interruptedExit); - await settle(); - expect(interruptedState.state).toBe("fulfilled"); - expect(await interruptedExit).toBe(0); - expect(calls(early)).toEqual([`connect ${target}`, `disconnect ${target}`]); - const killed = promiseState(Promise.resolve(early.handles[0]?.wait())); - await settle(); - expect(killed.state).toBe("fulfilled"); - expect((await early.handles[0]?.wait())?.code).toBeNull(); - expect(interrupted.err).toEqual([]); - - const hung = new CallerAdb([connect("adb", { hangs: true })]); - const slow = command(ENV, hung); - const slowExit = slow.run(["tunnel", "emulator", "adb", "--lease", "L1"]); - await vi.waitFor(() => expect(slow.webSocket.connections).toHaveLength(1)); - firstPeer(slow).receive(READY); - await vi.waitFor(() => expect(hung.calls).toHaveLength(1)); - slow.clock.advance(9_999); - await settle(); - expect(slow.err).toEqual([]); - slow.clock.advance(1); - await vi.waitFor(() => - expect(slow.err.join("")).toBe( - `warning: emulator adb: adb did not answer in 10 s; ${rerun}\n`, - ), - ); - expect((await hung.handles[0]?.wait())?.code).toBeNull(); - slow.signals.emit("SIGINT"); - slow.clock.advance(2_000); - expect(await slowExit).toBe(0); - }); -}); From 210796aee6b3284a74b63dd35fe9ecea86dbcebb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:34:19 +0100 Subject: [PATCH 70/83] test(mcp): split server.test.ts by topic --- src/mcp/server-devices.test.ts | 261 +++++++++++ src/mcp/server-run-on-device.test.ts | 340 ++++++++++++++ src/mcp/server.test.ts | 638 +-------------------------- src/mcp/test-server.ts | 44 ++ 4 files changed, 647 insertions(+), 636 deletions(-) create mode 100644 src/mcp/server-devices.test.ts create mode 100644 src/mcp/server-run-on-device.test.ts create mode 100644 src/mcp/test-server.ts diff --git a/src/mcp/server-devices.test.ts b/src/mcp/server-devices.test.ts new file mode 100644 index 000000000..a5e1a7da2 --- /dev/null +++ b/src/mcp/server-devices.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError, type StatusGetOutput } from "../client/index.js"; +import { FakeSimlockClient, sampleGrant, startingStatus } from "./test-support.js"; +import { call, connectedServer, text } from "./test-server.js"; + +type StatusDevice = NonNullable[number]; +type WorkerView = NonNullable[number]; + +describe("MCP list_devices physicalDevices", () => { + const phone = { + physical: true, + platform: "ios", + model: "iPhone 15", + osVersion: "26.4", + class: "phone", + } as const; + const listed = { + platform: "ios", + model: "iPhone 15", + osVersion: "26.4", + class: "phone", + } as const; + const cases: ReadonlyArray<{ + readonly name: string; + readonly devices: StatusDevice[] | undefined; + readonly workers?: WorkerView[]; + readonly expected: unknown[]; + }> = [ + { + name: "a ready phone", + devices: [statusDevice({ ...phone, id: "p1", state: "ready" })], + expected: [{ ...listed, state: "ready" }], + }, + { + name: "a phone through a gateway, with its worker", + devices: [statusDevice({ ...phone, id: "p1", state: "ready", workerId: "mac-mini-2" })], + workers: [workerView("mac-mini-2", {})], + expected: [{ ...listed, workerId: "mac-mini-2", state: "ready" }], + }, + { + name: "phones on a lost, an incompatible and a drained worker are left out", + devices: [ + statusDevice({ ...phone, id: "p1", state: "ready", workerId: "lost" }), + statusDevice({ ...phone, id: "p2", state: "ready", workerId: "old" }), + statusDevice({ ...phone, id: "p3", state: "ready", workerId: "drained" }), + statusDevice({ ...phone, id: "p4", model: "iPhone 16", state: "ready", workerId: "ok" }), + ], + workers: [ + workerView("lost", { connection: "disconnected" }), + workerView("old", { connection: "incompatible" }), + workerView("drained", { drained: true }), + workerView("ok", {}), + ], + expected: [{ ...listed, model: "iPhone 16", workerId: "ok", state: "ready" }], + }, + { + name: "simulators and emulators are left out", + devices: [ + statusDevice({ + id: "s1", + physical: false, + platform: "ios", + model: "iPhone 17", + osVersion: "26.5", + state: "ready", + }), + statusDevice({ + id: "e1", + physical: false, + platform: "android", + model: "Pixel 8", + osVersion: "35", + state: "leased", + }), + ], + expected: [], + }, + { name: "no phone enrolled", devices: [], expected: [] }, + { name: "a daemon still starting, with no device list", devices: undefined, expected: [] }, + { + name: "a removed phone is left out", + devices: [ + statusDevice({ ...phone, id: "p1", state: "deleted" }), + statusDevice({ ...phone, id: "p2", model: "iPhone 16", state: "ready" }), + ], + expected: [{ ...listed, model: "iPhone 16", state: "ready" }], + }, + { + name: "a leased phone, with no holder field", + devices: [statusDevice({ ...phone, id: "p1", state: "leased" })], + expected: [{ ...listed, state: "leased" }], + }, + ...(["locked", "untrusted", "not-present"] as const).map((absentReason) => ({ + name: `an absent phone, reason ${absentReason}`, + devices: [statusDevice({ ...phone, id: "p1", state: "absent", absentReason })], + expected: [{ ...listed, state: "absent", absentReason }], + })), + ]; + + it("lists the catalog plus each enrolled phone's worker, platform, model, OS version, class, state and absent reason, and no holder", async () => { + for (const { name, devices, workers, expected } of cases) { + const client = new FakeSimlockClient(); + client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + client.getStatusImpl = () => Promise.resolve(statusOutput(devices, workers)); + const { mcpClient, close } = await connectedServer(client); + try { + const result = await call(mcpClient, "list_devices", {}); + expect(result.isError, name).not.toBe(true); + expect(result.structuredContent, name).toEqual({ + platforms: [], + physicalDevices: expected, + }); + } finally { + await close(); + } + } + + const failing = new FakeSimlockClient(); + failing.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + failing.getStatusImpl = () => + Promise.reject(new SimlockError("FORBIDDEN", "protocol", "not allowed", {})); + const { mcpClient, close } = await connectedServer(failing); + try { + const result = await call(mcpClient, "list_devices", {}); + expect(result.isError).toBe(true); + expect(JSON.parse(text(result))).toMatchObject({ code: "FORBIDDEN" }); + } finally { + await close(); + } + }); +}); + +describe("MCP rotate_device", () => { + it("rotates the session's held lease and returns its id and the canonical orientation", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); + client.rotateImpl = (leaseId, orientation) => Promise.resolve({ leaseId, orientation }); + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + const { mcpClient, close } = await connectedServer(client); + try { + const never = await call(mcpClient, "rotate_device", { orientation: "left" }); + expect(never.isError).toBe(true); + expect(JSON.parse(text(never)).code).toBe("NO_LEASE"); + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const result = await call(mcpClient, "rotate_device", { orientation: "landscape-left" }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toEqual({ + leaseId: "lease-1", + orientation: "landscape_left", + }); + expect(client.calls.filter((entry) => entry.method === "rotate")).toEqual([ + { input: { leaseId: "lease-1", orientation: "landscape_left" }, method: "rotate" }, + ]); + + for (const [spelling, canonical] of [ + ["LEFT", "landscape_left"], + [" upside-down ", "portrait_upside_down"], + ]) { + const spelled = await call(mcpClient, "rotate_device", { orientation: spelling }); + expect(spelled.isError).not.toBe(true); + expect(spelled.structuredContent).toEqual({ leaseId: "lease-1", orientation: canonical }); + } + for (const [orientation, bound] of [ + ["", "expected string to have >=1 characters"], + ["x".repeat(65), "expected string to have <=64 characters"], + ] as const) { + const refused = await call(mcpClient, "rotate_device", { orientation }); + expect(refused.isError).toBe(true); + expect(text(refused)).toContain(`Invalid arguments for tool rotate_device: Too `); + expect(text(refused)).toContain(bound); + } + const longest = await call(mcpClient, "rotate_device", { orientation: "x".repeat(64) }); + expect(JSON.parse(text(longest)).code).toBe("BAD_REQUEST"); + expect(client.calls.filter((entry) => entry.method === "rotate")).toHaveLength(3); + const unknown = await call(mcpClient, "rotate_device", { orientation: "sideways" }); + expect(unknown.isError).toBe(true); + expect(JSON.parse(text(unknown))).toEqual({ + code: "BAD_REQUEST", + message: + "Unknown orientation sideways; use portrait, portrait-upside-down, landscape-left or landscape-right", + }); + const unsupported = new SimlockError( + "ROTATION_UNSUPPORTED", + "domain", + "Lease lease-1 is on an Android emulator; only an iOS simulator can be rotated", + { class: "emulator", leaseId: "lease-1", physical: false, platform: "android" }, + ); + client.rotateImpl = () => Promise.reject(unsupported); + const refusal = await call(mcpClient, "rotate_device", { orientation: "left" }); + expect(refusal.isError).toBe(true); + expect(JSON.parse(text(refusal))).toEqual({ + code: "ROTATION_UNSUPPORTED", + message: unsupported.message, + }); + client.rotateImpl = (leaseId, orientation) => Promise.resolve({ leaseId, orientation }); + + await call(mcpClient, "release_simulator", { leaseId: "lease-1" }); + const released = await call(mcpClient, "rotate_device", { orientation: "left" }); + expect(released.isError).toBe(true); + expect(JSON.parse(text(released))).toEqual({ + code: "NO_LEASE", + message: "This MCP session holds no lease. Call lease_simulator first.", + }); + } finally { + await close(); + } + }); +}); + +interface StatusDeviceFixture { + readonly id: string; + readonly physical: boolean; + readonly platform: "ios" | "android"; + readonly model: string; + readonly osVersion: string; + readonly class?: "phone" | "tablet"; + readonly state: StatusDevice["state"]; + readonly absentReason?: StatusDevice["absentReason"]; + readonly workerId?: string; +} + +function statusDevice(fixture: StatusDeviceFixture): StatusDevice { + return { + id: fixture.id, + spec: { + physical: fixture.physical, + platform: fixture.platform, + model: fixture.model, + osVersion: fixture.osVersion, + }, + state: fixture.state, + physical: fixture.physical, + servesDefaultMode: false, + ...(fixture.class === undefined ? {} : { class: fixture.class }), + ...(fixture.absentReason === undefined ? {} : { absentReason: fixture.absentReason }), + ...(fixture.workerId === undefined ? {} : { workerId: fixture.workerId }), + }; +} + +function statusOutput( + devices: StatusDevice[] | undefined, + workers?: WorkerView[], +): StatusGetOutput { + return { + ...startingStatus(), + ...(devices === undefined ? {} : { devices }), + ...(workers === undefined ? {} : { workers }), + }; +} + +function workerView( + id: string, + state: { readonly connection?: WorkerView["connection"]; readonly drained?: boolean }, +): WorkerView { + return { + id, + connection: state.connection ?? "connected", + drained: state.drained ?? false, + lastSeenAt: 0, + }; +} diff --git a/src/mcp/server-run-on-device.test.ts b/src/mcp/server-run-on-device.test.ts new file mode 100644 index 000000000..cc617bd98 --- /dev/null +++ b/src/mcp/server-run-on-device.test.ts @@ -0,0 +1,340 @@ +import { CallToolResultSchema } from "@modelcontextprotocol/sdk/types.js"; +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../client/index.js"; +import { FakeSimlockClient, sampleGrant } from "./test-support.js"; +import { call, connectedServer, text, waitFor } from "./test-server.js"; + +describe("MCP run_on_device", () => { + it("returns the command's stdout, stderr and exit code, run on the session's lease", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); + const { mcpClient, close } = await connectedServer(client); + try { + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const cases = [ + { + input: { args: ["listapps", "booted"], tool: "simctl" }, + chunks: [{ chunk: "com.x\n", stream: "stdout" as const }], + exitCode: 0, + expected: { exitCode: 0, stderr: "", stdout: "com.x\n", truncated: false }, + }, + { + input: { args: ["install", "/b/App.app"], tool: "simctl" }, + chunks: [{ chunk: "no such file", stream: "stderr" as const }], + exitCode: 3, + expected: { exitCode: 3, stderr: "no such file", stdout: "", truncated: false }, + }, + { + input: { args: ["shell", "cat"], stdin: "hello", tool: "adb" }, + chunks: [], + exitCode: 0, + expected: { exitCode: 0, stderr: "", stdout: "", truncated: false }, + }, + ]; + for (const entry of cases) { + client.execImpl = (_input, options) => { + for (const chunk of entry.chunks) options.onOutput?.(chunk); + return Promise.resolve({ exitCode: entry.exitCode }); + }; + const result = await call(mcpClient, "run_on_device", entry.input); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toEqual(entry.expected); + expect(client.calls.filter((c) => c.method === "exec").at(-1)?.input).toEqual({ + ...entry.input, + leaseId: "lease-1", + }); + } + } finally { + await close(); + } + }); +}); + +describe("MCP run_on_device output cap", () => { + it("keeps the last 32768 characters of each stream and sets truncated when either lost any", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); + const { mcpClient, close } = await connectedServer(client); + try { + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const cap = 32_768; + const cases = [ + { + chunks: [ + { chunk: "a".repeat(cap), stream: "stdout" as const }, + { chunk: "tail", stream: "stdout" as const }, + ], + expected: { + stderr: "", + stdout: "a".repeat(cap - 4) + "tail", + truncated: true, + }, + }, + { + chunks: [ + { chunk: "out", stream: "stdout" as const }, + { chunk: "b".repeat(cap + 1), stream: "stderr" as const }, + ], + expected: { stderr: "b".repeat(cap), stdout: "out", truncated: true }, + }, + { + chunks: [{ chunk: "c".repeat(cap), stream: "stdout" as const }], + expected: { stderr: "", stdout: "c".repeat(cap), truncated: false }, + }, + ]; + for (const entry of cases) { + client.execImpl = (_input, options) => { + for (const chunk of entry.chunks) options.onOutput?.(chunk); + return Promise.resolve({ exitCode: 0 }); + }; + const result = await call(mcpClient, "run_on_device", { args: [], tool: "adb" }); + expect(result.structuredContent).toEqual({ exitCode: 0, ...entry.expected }); + } + } finally { + await close(); + } + }); +}); + +describe("MCP run_on_device without a lease", () => { + it("answers NO_LEASE with no daemon call when the session holds no lease", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + client.execImpl = () => Promise.resolve({ exitCode: 0 }); + const { mcpClient, close } = await connectedServer(client); + const noLease = { + code: "NO_LEASE", + message: "This MCP session holds no lease. Call lease_simulator first.", + }; + const run = () => call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); + try { + const never = await run(); + expect(never.isError).toBe(true); + expect(JSON.parse(text(never))).toEqual(noLease); + + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + expect((await run()).isError).not.toBe(true); + + await call(mcpClient, "release_simulator", { leaseId: "lease-1" }); + expect(JSON.parse(text(await run()))).toEqual(noLease); + + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + expect((await run()).isError).not.toBe(true); + client.emitLeaseLost({ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }); + expect(JSON.parse(text(await run()))).toEqual(noLease); + + expect(client.calls.filter((entry) => entry.method === "exec")).toHaveLength(2); + } finally { + await close(); + } + }); +}); + +describe("MCP run_on_device daemon errors", () => { + it("comes back as isError with the daemon's code and message", async () => { + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); + const { mcpClient, close } = await connectedServer(client); + try { + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const refusal = new SimlockError( + "PASSTHROUGH_REFUSED", + "domain", + "Refusing `simctl boot booted`: it names device booted.", + { tool: "simctl" }, + ); + const cases = [ + refusal, + new SimlockError("UNKNOWN_PASSTHROUGH_TOOL", "domain", "No driver wraps rm.", { + tool: "rm", + }), + new SimlockError("EXEC_TIMEOUT", "domain", "The command ran past its time limit.", {}), + ]; + for (const error of cases) { + client.execImpl = () => Promise.reject(error); + const result = await call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); + expect(result.isError).toBe(true); + expect(JSON.parse(text(result))).toEqual({ code: error.code, message: error.message }); + } + } finally { + await close(); + } + }); +}); + +describe("MCP run_on_device progress", () => { + it("sends progress at most once per second while output arrives and every 10 s while none does, only with a progress token", async () => { + const client = new FakeSimlockClient(); + const grant = sampleGrant(); + client.requestLeaseImpl = () => + Promise.resolve({ + ...grant, + lease: { ...grant.lease, ttlDeadline: 3_600_000, ttlMs: 3_600_000 }, + }); + const { clock, mcpClient, close } = await connectedServer(client); + try { + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const install = ["install", "/b/App.app"]; + const cases = [ + { + args: install, + message: "Running simctl install", + script: [ + { at: 0, chunk: true }, + { at: 400, chunk: true }, + { at: 900, chunk: true }, + ], + token: true, + notifications: 1, + }, + { + args: install, + message: "Running simctl install", + script: [ + { at: 0, chunk: true }, + { at: 1500, chunk: true }, + ], + token: true, + notifications: 2, + }, + { + args: install, + message: "Running simctl install", + script: [{ at: 25_000, chunk: false }], + token: true, + notifications: 2, + }, + { + args: install, + message: "Running simctl install", + script: [ + { at: 0, chunk: true }, + { at: 1000, chunk: true }, + ], + token: true, + notifications: 2, + }, + { + args: install, + message: "Running simctl install", + script: [{ at: 9_999, chunk: false }], + token: true, + notifications: 0, + }, + { + args: install, + message: "Running simctl install", + script: [{ at: 10_000, chunk: false }], + token: true, + notifications: 1, + }, + { + args: install, + message: "Running simctl install", + script: [ + { at: 0, chunk: true }, + { at: 5000, chunk: true }, + { at: 12_000, chunk: false }, + ], + token: true, + notifications: 2, + }, + { + args: [], + message: "Running simctl", + script: [{ at: 0, chunk: true }], + token: true, + notifications: 1, + }, + { + args: install, + message: "", + script: [ + { at: 0, chunk: true }, + { at: 1500, chunk: true }, + { at: 25_000, chunk: false }, + ], + token: false, + notifications: 0, + }, + ]; + for (const entry of cases) { + let emit: ((chunk: { stream: "stdout"; chunk: string }) => void) | undefined; + let finish: (() => void) | undefined; + client.execImpl = (_input, options) => + new Promise((resolve) => { + emit = (chunk) => options.onOutput?.(chunk); + finish = () => resolve({ exitCode: 0 }); + }); + const events: Array<{ + message?: string | undefined; + progress: number; + total?: number | undefined; + }> = []; + const unexpected: Error[] = []; + mcpClient.onerror = (error) => unexpected.push(error); + const pending = mcpClient.request( + { + method: "tools/call", + params: { arguments: { args: entry.args, tool: "simctl" }, name: "run_on_device" }, + }, + CallToolResultSchema, + entry.token ? { onprogress: (progress) => events.push(progress) } : undefined, + ); + await waitFor(() => emit !== undefined); + let elapsed = 0; + for (const step of entry.script) { + clock.advance(step.at - elapsed); + elapsed = step.at; + if (step.chunk) emit?.({ chunk: "x", stream: "stdout" }); + } + finish?.(); + await pending; + // why: a notification the server sends without a token, or after the call ended, reaches + // the client as an error for an unknown token. + clock.advance(30_000); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(unexpected).toEqual([]); + expect(events.map((event) => event.progress)).toEqual( + Array.from({ length: entry.notifications }, (_, index) => index + 1), + ); + for (const event of events) { + expect(event.message).toBe(entry.message); + expect(event.total).toBeUndefined(); + } + } + } finally { + await close(); + } + }); +}); + +describe("MCP run_on_device alongside other tools", () => { + it("lets lease_status answer while the command is still running", async () => { + const client = new FakeSimlockClient(); + const grant = sampleGrant(); + client.requestLeaseImpl = () => Promise.resolve(grant); + client.listLeasesImpl = () => + Promise.resolve({ + leases: [{ ...grant.lease, deviceId: grant.device.id }], + }); + let finish: (() => void) | undefined; + client.execImpl = () => + new Promise((resolve) => { + finish = () => resolve({ exitCode: 0 }); + }); + const { mcpClient, close } = await connectedServer(client); + try { + await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); + const running = call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); + await waitFor(() => finish !== undefined); + const status = await call(mcpClient, "lease_status", {}); + expect(status.structuredContent).toMatchObject({ held: true, id: "lease-1" }); + finish?.(); + expect((await running).structuredContent).toMatchObject({ exitCode: 0 }); + } finally { + await close(); + } + }); +}); diff --git a/src/mcp/server.test.ts b/src/mcp/server.test.ts index 4e8e74275..32f7da2fd 100644 --- a/src/mcp/server.test.ts +++ b/src/mcp/server.test.ts @@ -1,5 +1,3 @@ -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { CallToolResultSchema, ListToolsResultSchema, @@ -7,16 +5,10 @@ import { ProgressNotificationSchema, } from "@modelcontextprotocol/sdk/types.js"; import { describe, expect, it } from "vitest"; - -import { SimlockError, type StatusGetOutput } from "../client/index.js"; -import { FakeClock } from "../ports/index.js"; +import { SimlockError } from "../client/index.js"; import { FakeSimlockClient, sampleGrant, startingStatus } from "./test-support.js"; -import { McpSession } from "./session.js"; import { leaseSimulatorInputSchema } from "./contracts.js"; -import { createMcpServer } from "./server.js"; - -type StatusDevice = NonNullable[number]; -type WorkerView = NonNullable[number]; +import { call, connectedServer, text, waitFor } from "./test-server.js"; describe("MCP server (smoke)", () => { it("advertises exactly the lease, catalog, status, run_on_device and rotate_device tools, and walks lease -> list -> status -> release", async () => { @@ -568,629 +560,3 @@ describe("MCP server (smoke)", () => { } }); }); - -async function connectedServer(client: FakeSimlockClient) { - const clock = new FakeClock(); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const server = createMcpServer(session, { clock }); - const mcpClient = new Client({ name: "mcp-test-client", version: "1.0.0" }); - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); - await Promise.all([mcpClient.connect(clientTransport), server.connect(serverTransport)]); - return { - clock, - close: async () => { - await Promise.all([mcpClient.close(), server.close(), session.close()]); - }, - mcpClient, - }; -} - -describe("MCP run_on_device", () => { - it("returns the command's stdout, stderr and exit code, run on the session's lease", async () => { - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); - const { mcpClient, close } = await connectedServer(client); - try { - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const cases = [ - { - input: { args: ["listapps", "booted"], tool: "simctl" }, - chunks: [{ chunk: "com.x\n", stream: "stdout" as const }], - exitCode: 0, - expected: { exitCode: 0, stderr: "", stdout: "com.x\n", truncated: false }, - }, - { - input: { args: ["install", "/b/App.app"], tool: "simctl" }, - chunks: [{ chunk: "no such file", stream: "stderr" as const }], - exitCode: 3, - expected: { exitCode: 3, stderr: "no such file", stdout: "", truncated: false }, - }, - { - input: { args: ["shell", "cat"], stdin: "hello", tool: "adb" }, - chunks: [], - exitCode: 0, - expected: { exitCode: 0, stderr: "", stdout: "", truncated: false }, - }, - ]; - for (const entry of cases) { - client.execImpl = (_input, options) => { - for (const chunk of entry.chunks) options.onOutput?.(chunk); - return Promise.resolve({ exitCode: entry.exitCode }); - }; - const result = await call(mcpClient, "run_on_device", entry.input); - expect(result.isError).not.toBe(true); - expect(result.structuredContent).toEqual(entry.expected); - expect(client.calls.filter((c) => c.method === "exec").at(-1)?.input).toEqual({ - ...entry.input, - leaseId: "lease-1", - }); - } - } finally { - await close(); - } - }); -}); - -describe("MCP run_on_device output cap", () => { - it("keeps the last 32768 characters of each stream and sets truncated when either lost any", async () => { - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); - const { mcpClient, close } = await connectedServer(client); - try { - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const cap = 32_768; - const cases = [ - { - chunks: [ - { chunk: "a".repeat(cap), stream: "stdout" as const }, - { chunk: "tail", stream: "stdout" as const }, - ], - expected: { - stderr: "", - stdout: "a".repeat(cap - 4) + "tail", - truncated: true, - }, - }, - { - chunks: [ - { chunk: "out", stream: "stdout" as const }, - { chunk: "b".repeat(cap + 1), stream: "stderr" as const }, - ], - expected: { stderr: "b".repeat(cap), stdout: "out", truncated: true }, - }, - { - chunks: [{ chunk: "c".repeat(cap), stream: "stdout" as const }], - expected: { stderr: "", stdout: "c".repeat(cap), truncated: false }, - }, - ]; - for (const entry of cases) { - client.execImpl = (_input, options) => { - for (const chunk of entry.chunks) options.onOutput?.(chunk); - return Promise.resolve({ exitCode: 0 }); - }; - const result = await call(mcpClient, "run_on_device", { args: [], tool: "adb" }); - expect(result.structuredContent).toEqual({ exitCode: 0, ...entry.expected }); - } - } finally { - await close(); - } - }); -}); - -describe("MCP run_on_device without a lease", () => { - it("answers NO_LEASE with no daemon call when the session holds no lease", async () => { - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - client.execImpl = () => Promise.resolve({ exitCode: 0 }); - const { mcpClient, close } = await connectedServer(client); - const noLease = { - code: "NO_LEASE", - message: "This MCP session holds no lease. Call lease_simulator first.", - }; - const run = () => call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); - try { - const never = await run(); - expect(never.isError).toBe(true); - expect(JSON.parse(text(never))).toEqual(noLease); - - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - expect((await run()).isError).not.toBe(true); - - await call(mcpClient, "release_simulator", { leaseId: "lease-1" }); - expect(JSON.parse(text(await run()))).toEqual(noLease); - - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - expect((await run()).isError).not.toBe(true); - client.emitLeaseLost({ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }); - expect(JSON.parse(text(await run()))).toEqual(noLease); - - expect(client.calls.filter((entry) => entry.method === "exec")).toHaveLength(2); - } finally { - await close(); - } - }); -}); - -describe("MCP run_on_device daemon errors", () => { - it("comes back as isError with the daemon's code and message", async () => { - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); - const { mcpClient, close } = await connectedServer(client); - try { - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const refusal = new SimlockError( - "PASSTHROUGH_REFUSED", - "domain", - "Refusing `simctl boot booted`: it names device booted.", - { tool: "simctl" }, - ); - const cases = [ - refusal, - new SimlockError("UNKNOWN_PASSTHROUGH_TOOL", "domain", "No driver wraps rm.", { - tool: "rm", - }), - new SimlockError("EXEC_TIMEOUT", "domain", "The command ran past its time limit.", {}), - ]; - for (const error of cases) { - client.execImpl = () => Promise.reject(error); - const result = await call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); - expect(result.isError).toBe(true); - expect(JSON.parse(text(result))).toEqual({ code: error.code, message: error.message }); - } - } finally { - await close(); - } - }); -}); - -describe("MCP run_on_device progress", () => { - it("sends progress at most once per second while output arrives and every 10 s while none does, only with a progress token", async () => { - const client = new FakeSimlockClient(); - const grant = sampleGrant(); - client.requestLeaseImpl = () => - Promise.resolve({ - ...grant, - lease: { ...grant.lease, ttlDeadline: 3_600_000, ttlMs: 3_600_000 }, - }); - const { clock, mcpClient, close } = await connectedServer(client); - try { - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const install = ["install", "/b/App.app"]; - const cases = [ - { - args: install, - message: "Running simctl install", - script: [ - { at: 0, chunk: true }, - { at: 400, chunk: true }, - { at: 900, chunk: true }, - ], - token: true, - notifications: 1, - }, - { - args: install, - message: "Running simctl install", - script: [ - { at: 0, chunk: true }, - { at: 1500, chunk: true }, - ], - token: true, - notifications: 2, - }, - { - args: install, - message: "Running simctl install", - script: [{ at: 25_000, chunk: false }], - token: true, - notifications: 2, - }, - { - args: install, - message: "Running simctl install", - script: [ - { at: 0, chunk: true }, - { at: 1000, chunk: true }, - ], - token: true, - notifications: 2, - }, - { - args: install, - message: "Running simctl install", - script: [{ at: 9_999, chunk: false }], - token: true, - notifications: 0, - }, - { - args: install, - message: "Running simctl install", - script: [{ at: 10_000, chunk: false }], - token: true, - notifications: 1, - }, - { - args: install, - message: "Running simctl install", - script: [ - { at: 0, chunk: true }, - { at: 5000, chunk: true }, - { at: 12_000, chunk: false }, - ], - token: true, - notifications: 2, - }, - { - args: [], - message: "Running simctl", - script: [{ at: 0, chunk: true }], - token: true, - notifications: 1, - }, - { - args: install, - message: "", - script: [ - { at: 0, chunk: true }, - { at: 1500, chunk: true }, - { at: 25_000, chunk: false }, - ], - token: false, - notifications: 0, - }, - ]; - for (const entry of cases) { - let emit: ((chunk: { stream: "stdout"; chunk: string }) => void) | undefined; - let finish: (() => void) | undefined; - client.execImpl = (_input, options) => - new Promise((resolve) => { - emit = (chunk) => options.onOutput?.(chunk); - finish = () => resolve({ exitCode: 0 }); - }); - const events: Array<{ - message?: string | undefined; - progress: number; - total?: number | undefined; - }> = []; - const unexpected: Error[] = []; - mcpClient.onerror = (error) => unexpected.push(error); - const pending = mcpClient.request( - { - method: "tools/call", - params: { arguments: { args: entry.args, tool: "simctl" }, name: "run_on_device" }, - }, - CallToolResultSchema, - entry.token ? { onprogress: (progress) => events.push(progress) } : undefined, - ); - await waitFor(() => emit !== undefined); - let elapsed = 0; - for (const step of entry.script) { - clock.advance(step.at - elapsed); - elapsed = step.at; - if (step.chunk) emit?.({ chunk: "x", stream: "stdout" }); - } - finish?.(); - await pending; - // why: a notification the server sends without a token, or after the call ended, reaches - // the client as an error for an unknown token. - clock.advance(30_000); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(unexpected).toEqual([]); - expect(events.map((event) => event.progress)).toEqual( - Array.from({ length: entry.notifications }, (_, index) => index + 1), - ); - for (const event of events) { - expect(event.message).toBe(entry.message); - expect(event.total).toBeUndefined(); - } - } - } finally { - await close(); - } - }); -}); - -describe("MCP run_on_device alongside other tools", () => { - it("lets lease_status answer while the command is still running", async () => { - const client = new FakeSimlockClient(); - const grant = sampleGrant(); - client.requestLeaseImpl = () => Promise.resolve(grant); - client.listLeasesImpl = () => - Promise.resolve({ - leases: [{ ...grant.lease, deviceId: grant.device.id }], - }); - let finish: (() => void) | undefined; - client.execImpl = () => - new Promise((resolve) => { - finish = () => resolve({ exitCode: 0 }); - }); - const { mcpClient, close } = await connectedServer(client); - try { - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const running = call(mcpClient, "run_on_device", { args: [], tool: "simctl" }); - await waitFor(() => finish !== undefined); - const status = await call(mcpClient, "lease_status", {}); - expect(status.structuredContent).toMatchObject({ held: true, id: "lease-1" }); - finish?.(); - expect((await running).structuredContent).toMatchObject({ exitCode: 0 }); - } finally { - await close(); - } - }); -}); - -describe("MCP list_devices physicalDevices", () => { - const phone = { - physical: true, - platform: "ios", - model: "iPhone 15", - osVersion: "26.4", - class: "phone", - } as const; - const listed = { - platform: "ios", - model: "iPhone 15", - osVersion: "26.4", - class: "phone", - } as const; - const cases: ReadonlyArray<{ - readonly name: string; - readonly devices: StatusDevice[] | undefined; - readonly workers?: WorkerView[]; - readonly expected: unknown[]; - }> = [ - { - name: "a ready phone", - devices: [statusDevice({ ...phone, id: "p1", state: "ready" })], - expected: [{ ...listed, state: "ready" }], - }, - { - name: "a phone through a gateway, with its worker", - devices: [statusDevice({ ...phone, id: "p1", state: "ready", workerId: "mac-mini-2" })], - workers: [workerView("mac-mini-2", {})], - expected: [{ ...listed, workerId: "mac-mini-2", state: "ready" }], - }, - { - name: "phones on a lost, an incompatible and a drained worker are left out", - devices: [ - statusDevice({ ...phone, id: "p1", state: "ready", workerId: "lost" }), - statusDevice({ ...phone, id: "p2", state: "ready", workerId: "old" }), - statusDevice({ ...phone, id: "p3", state: "ready", workerId: "drained" }), - statusDevice({ ...phone, id: "p4", model: "iPhone 16", state: "ready", workerId: "ok" }), - ], - workers: [ - workerView("lost", { connection: "disconnected" }), - workerView("old", { connection: "incompatible" }), - workerView("drained", { drained: true }), - workerView("ok", {}), - ], - expected: [{ ...listed, model: "iPhone 16", workerId: "ok", state: "ready" }], - }, - { - name: "simulators and emulators are left out", - devices: [ - statusDevice({ - id: "s1", - physical: false, - platform: "ios", - model: "iPhone 17", - osVersion: "26.5", - state: "ready", - }), - statusDevice({ - id: "e1", - physical: false, - platform: "android", - model: "Pixel 8", - osVersion: "35", - state: "leased", - }), - ], - expected: [], - }, - { name: "no phone enrolled", devices: [], expected: [] }, - { name: "a daemon still starting, with no device list", devices: undefined, expected: [] }, - { - name: "a removed phone is left out", - devices: [ - statusDevice({ ...phone, id: "p1", state: "deleted" }), - statusDevice({ ...phone, id: "p2", model: "iPhone 16", state: "ready" }), - ], - expected: [{ ...listed, model: "iPhone 16", state: "ready" }], - }, - { - name: "a leased phone, with no holder field", - devices: [statusDevice({ ...phone, id: "p1", state: "leased" })], - expected: [{ ...listed, state: "leased" }], - }, - ...(["locked", "untrusted", "not-present"] as const).map((absentReason) => ({ - name: `an absent phone, reason ${absentReason}`, - devices: [statusDevice({ ...phone, id: "p1", state: "absent", absentReason })], - expected: [{ ...listed, state: "absent", absentReason }], - })), - ]; - - it("lists the catalog plus each enrolled phone's worker, platform, model, OS version, class, state and absent reason, and no holder", async () => { - for (const { name, devices, workers, expected } of cases) { - const client = new FakeSimlockClient(); - client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - client.getStatusImpl = () => Promise.resolve(statusOutput(devices, workers)); - const { mcpClient, close } = await connectedServer(client); - try { - const result = await call(mcpClient, "list_devices", {}); - expect(result.isError, name).not.toBe(true); - expect(result.structuredContent, name).toEqual({ - platforms: [], - physicalDevices: expected, - }); - } finally { - await close(); - } - } - - const failing = new FakeSimlockClient(); - failing.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - failing.getStatusImpl = () => - Promise.reject(new SimlockError("FORBIDDEN", "protocol", "not allowed", {})); - const { mcpClient, close } = await connectedServer(failing); - try { - const result = await call(mcpClient, "list_devices", {}); - expect(result.isError).toBe(true); - expect(JSON.parse(text(result))).toMatchObject({ code: "FORBIDDEN" }); - } finally { - await close(); - } - }); -}); - -describe("MCP rotate_device", () => { - it("rotates the session's held lease and returns its id and the canonical orientation", async () => { - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant()); - client.rotateImpl = (leaseId, orientation) => Promise.resolve({ leaseId, orientation }); - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - const { mcpClient, close } = await connectedServer(client); - try { - const never = await call(mcpClient, "rotate_device", { orientation: "left" }); - expect(never.isError).toBe(true); - expect(JSON.parse(text(never)).code).toBe("NO_LEASE"); - await call(mcpClient, "lease_simulator", { model: "iPhone 17 Pro", platform: "ios" }); - const result = await call(mcpClient, "rotate_device", { orientation: "landscape-left" }); - expect(result.isError).not.toBe(true); - expect(result.structuredContent).toEqual({ - leaseId: "lease-1", - orientation: "landscape_left", - }); - expect(client.calls.filter((entry) => entry.method === "rotate")).toEqual([ - { input: { leaseId: "lease-1", orientation: "landscape_left" }, method: "rotate" }, - ]); - - for (const [spelling, canonical] of [ - ["LEFT", "landscape_left"], - [" upside-down ", "portrait_upside_down"], - ]) { - const spelled = await call(mcpClient, "rotate_device", { orientation: spelling }); - expect(spelled.isError).not.toBe(true); - expect(spelled.structuredContent).toEqual({ leaseId: "lease-1", orientation: canonical }); - } - for (const [orientation, bound] of [ - ["", "expected string to have >=1 characters"], - ["x".repeat(65), "expected string to have <=64 characters"], - ] as const) { - const refused = await call(mcpClient, "rotate_device", { orientation }); - expect(refused.isError).toBe(true); - expect(text(refused)).toContain(`Invalid arguments for tool rotate_device: Too `); - expect(text(refused)).toContain(bound); - } - const longest = await call(mcpClient, "rotate_device", { orientation: "x".repeat(64) }); - expect(JSON.parse(text(longest)).code).toBe("BAD_REQUEST"); - expect(client.calls.filter((entry) => entry.method === "rotate")).toHaveLength(3); - const unknown = await call(mcpClient, "rotate_device", { orientation: "sideways" }); - expect(unknown.isError).toBe(true); - expect(JSON.parse(text(unknown))).toEqual({ - code: "BAD_REQUEST", - message: - "Unknown orientation sideways; use portrait, portrait-upside-down, landscape-left or landscape-right", - }); - const unsupported = new SimlockError( - "ROTATION_UNSUPPORTED", - "domain", - "Lease lease-1 is on an Android emulator; only an iOS simulator can be rotated", - { class: "emulator", leaseId: "lease-1", physical: false, platform: "android" }, - ); - client.rotateImpl = () => Promise.reject(unsupported); - const refusal = await call(mcpClient, "rotate_device", { orientation: "left" }); - expect(refusal.isError).toBe(true); - expect(JSON.parse(text(refusal))).toEqual({ - code: "ROTATION_UNSUPPORTED", - message: unsupported.message, - }); - client.rotateImpl = (leaseId, orientation) => Promise.resolve({ leaseId, orientation }); - - await call(mcpClient, "release_simulator", { leaseId: "lease-1" }); - const released = await call(mcpClient, "rotate_device", { orientation: "left" }); - expect(released.isError).toBe(true); - expect(JSON.parse(text(released))).toEqual({ - code: "NO_LEASE", - message: "This MCP session holds no lease. Call lease_simulator first.", - }); - } finally { - await close(); - } - }); -}); - -interface StatusDeviceFixture { - readonly id: string; - readonly physical: boolean; - readonly platform: "ios" | "android"; - readonly model: string; - readonly osVersion: string; - readonly class?: "phone" | "tablet"; - readonly state: StatusDevice["state"]; - readonly absentReason?: StatusDevice["absentReason"]; - readonly workerId?: string; -} - -function statusDevice(fixture: StatusDeviceFixture): StatusDevice { - return { - id: fixture.id, - spec: { - physical: fixture.physical, - platform: fixture.platform, - model: fixture.model, - osVersion: fixture.osVersion, - }, - state: fixture.state, - physical: fixture.physical, - servesDefaultMode: false, - ...(fixture.class === undefined ? {} : { class: fixture.class }), - ...(fixture.absentReason === undefined ? {} : { absentReason: fixture.absentReason }), - ...(fixture.workerId === undefined ? {} : { workerId: fixture.workerId }), - }; -} - -function statusOutput( - devices: StatusDevice[] | undefined, - workers?: WorkerView[], -): StatusGetOutput { - return { - ...startingStatus(), - ...(devices === undefined ? {} : { devices }), - ...(workers === undefined ? {} : { workers }), - }; -} - -function workerView( - id: string, - state: { readonly connection?: WorkerView["connection"]; readonly drained?: boolean }, -): WorkerView { - return { - id, - connection: state.connection ?? "connected", - drained: state.drained ?? false, - lastSeenAt: 0, - }; -} - -function call(client: Client, name: string, arguments_: Record) { - return client.request( - { method: "tools/call", params: { arguments: arguments_, name } }, - CallToolResultSchema, - ); -} - -function text(result: { content: Array<{ type: string; text?: string }> }): string { - const first = result.content[0]; - if (first?.type !== "text" || first.text === undefined) throw new Error("Expected text result"); - return first.text; -} - -async function waitFor(predicate: () => boolean): Promise { - while (!predicate()) await new Promise((resolve) => setTimeout(resolve, 0)); -} diff --git a/src/mcp/test-server.ts b/src/mcp/test-server.ts new file mode 100644 index 000000000..b994ad413 --- /dev/null +++ b/src/mcp/test-server.ts @@ -0,0 +1,44 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { CallToolResultSchema } from "@modelcontextprotocol/sdk/types.js"; +import { FakeClock } from "../ports/index.js"; +import { FakeSimlockClient } from "./test-support.js"; +import { McpSession } from "./session.js"; +import { createMcpServer } from "./server.js"; + +export async function connectedServer(client: FakeSimlockClient) { + const clock = new FakeClock(); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const server = createMcpServer(session, { clock }); + const mcpClient = new Client({ name: "mcp-test-client", version: "1.0.0" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await Promise.all([mcpClient.connect(clientTransport), server.connect(serverTransport)]); + return { + clock, + close: async () => { + await Promise.all([mcpClient.close(), server.close(), session.close()]); + }, + mcpClient, + }; +} + +export function call(client: Client, name: string, arguments_: Record) { + return client.request( + { method: "tools/call", params: { arguments: arguments_, name } }, + CallToolResultSchema, + ); +} + +export function text(result: { content: Array<{ type: string; text?: string }> }): string { + const first = result.content[0]; + if (first?.type !== "text" || first.text === undefined) throw new Error("Expected text result"); + return first.text; +} + +export async function waitFor(predicate: () => boolean): Promise { + while (!predicate()) await new Promise((resolve) => setTimeout(resolve, 0)); +} From 7cf04a8a1003b1bfded33fa6d9d50bb01252306c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:35:59 +0100 Subject: [PATCH 71/83] test(mcp): split session.test.ts by topic --- src/mcp/session-lease-endings.test.ts | 190 ++++++ src/mcp/session-reconnect.test.ts | 184 ++++++ src/mcp/session-release.test.ts | 364 +++++++++++ src/mcp/session-renewal.test.ts | 139 +++++ src/mcp/session.test.ts | 852 -------------------------- src/mcp/test-session.ts | 4 + 6 files changed, 881 insertions(+), 852 deletions(-) create mode 100644 src/mcp/session-lease-endings.test.ts create mode 100644 src/mcp/session-reconnect.test.ts create mode 100644 src/mcp/session-release.test.ts create mode 100644 src/mcp/session-renewal.test.ts create mode 100644 src/mcp/test-session.ts diff --git a/src/mcp/session-lease-endings.test.ts b/src/mcp/session-lease-endings.test.ts new file mode 100644 index 000000000..20bd3077b --- /dev/null +++ b/src/mcp/session-lease-endings.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../client/index.js"; +import { FakeClock } from "../ports/index.js"; +import { FakeSimlockClient, sampleGrant, startingStatus } from "./test-support.js"; +import { McpSession } from "./session.js"; +import { flushMicrotasks } from "./test-session.js"; + +describe("McpSession", () => { + it("reports a lease the daemon says is gone as lease-lost, and releases nothing on close", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.renewLeaseImpl = () => + Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), + ); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + // why: A lease that expired while this session was idle may never produce a push -- the + // renewal's own answer is what tells the agent. + clock.advance(4_115); + await flushMicrotasks(); + expect(notices).toEqual([ + { deviceId: "device-1", leaseId: "lease-1", reason: "renew-rejected" }, + ]); + expect(clock.pendingTimerCount).toBe(0); + + await session.close(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([]); + expect(client.calls.filter((call) => call.method === "renewLease")).toHaveLength(1); + }); + + it("reports a lease it could not keep alive as lost, and releases nothing for it on close", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + // why: Never terminal, never successful: the ladder retries until `sampleGrant`'s own 12_345 + // deadline passes and renewal gives up. + client.renewLeaseImpl = () => + Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist", {})); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + clock.advance(4_115); + await flushMicrotasks(); + clock.advance(20_000); // the retry lands past the deadline + await flushMicrotasks(); + + expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "renew-failed" }]); + expect(clock.pendingTimerCount).toBe(0); + + await session.close(); + expect( + client.calls.filter((call) => call.method === "releaseLease"), + "a lease that expired is not the session's to release", + ).toEqual([]); + }); + + it("announces one ending once, even when the renewal and the daemon both report it", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.renewLeaseImpl = () => + Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), + ); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + clock.advance(4_115); + await flushMicrotasks(); + // why: The daemon's own push for the same lease arrives a moment later; an agent must not read + // that as a second device lost. + client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); + + expect(notices).toEqual([ + { deviceId: "device-1", leaseId: "lease-1", reason: "renew-rejected" }, + ]); + + await session.close(); + }); + + it("stops renewing a lease that ended elsewhere", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); + + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); + + await session.close(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([]); + }); + + it("remembers the endings it announced up to a cap, dropping the oldest first", async () => { + // why: The dedupe memory of a long-lived session cannot grow without bound. It is a fixed + // window over the most recent endings, which is where duplicates actually live: a + // renewal's answer and the daemon's push about one lease arrive moments apart, never + // dozens of leases later. + const client = new FakeSimlockClient(); + client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + client.getStatusImpl = () => Promise.resolve(startingStatus()); + const session = new McpSession({ + clock: new FakeClock(), + connect: async () => client, + connectForRenew: async () => client, + }); + await session.listDevices({}); // wires the push relays up + const notices: Array<{ leaseId: string }> = []; + session.onLeaseLost((notice) => notices.push(notice)); + + // why: One past the cap, so the very first id has been dropped by the end of it. + const announced = 65; + for (let index = 0; index < announced; index += 1) { + client.emitLeaseLost({ + deviceId: "device-1", + leaseId: `lease-${String(index)}`, + reason: "expired", + }); + } + expect(notices).toHaveLength(announced); + + client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-64", reason: "expired" }); + expect(notices).toHaveLength(announced); + + // why: The oldest has aged out of the window, so its duplicate -- if one ever came, long after + // everything else -- would be announced again. That is the trade the cap makes. + client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-0", reason: "expired" }); + expect(notices).toHaveLength(announced + 1); + expect(notices.at(-1)?.leaseId).toBe("lease-0"); + + await session.close(); + }); + + it("relays the client's lease-lost, device-unhealthy, and device-recovered pushes to session listeners", async () => { + const client = new FakeSimlockClient(); + client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + client.getStatusImpl = () => Promise.resolve(startingStatus()); + const session = new McpSession({ + clock: new FakeClock(), + connect: async () => client, + connectForRenew: async () => client, + }); + await session.listDevices({}); // forces the client to connect and wire up push relays + + const leaseLost: unknown[] = []; + const deviceHealth: unknown[] = []; + session.onLeaseLost((notice) => leaseLost.push(notice)); + session.onDeviceHealth((notice) => deviceHealth.push(notice)); + + client.emitLeaseLost({ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }); + client.emitDeviceUnhealthy({ deviceId: "SIM-1", leaseId: "lease-1" }); + client.emitDeviceRecovered({ attempts: 2, deviceId: "SIM-1", leaseId: "lease-1" }); + + expect(leaseLost).toEqual([{ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }]); + expect(deviceHealth).toEqual([ + { deviceId: "SIM-1", kind: "unhealthy", leaseId: "lease-1", reason: "crashed" }, + { attempts: 2, deviceId: "SIM-1", kind: "recovered", leaseId: "lease-1" }, + ]); + }); +}); diff --git a/src/mcp/session-reconnect.test.ts b/src/mcp/session-reconnect.test.ts new file mode 100644 index 000000000..20148aa54 --- /dev/null +++ b/src/mcp/session-reconnect.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../client/index.js"; +import { FakeClock } from "../ports/index.js"; +import { FakeSimlockClient, sampleGrant, startingStatus } from "./test-support.js"; +import { McpSession } from "./session.js"; +import { flushMicrotasks } from "./test-session.js"; + +describe("McpSession", () => { + it("renews over connectForRenew after a dead connection, and never over connect", async () => { + // why: ADR 0004 §2's named safety property: the renew timer reconnects so an idle session does + // not lose its lease waiting for a tool call, but it reaches only a daemon that is already + // listening. `connect` is the trigger that may auto-launch one, so an operator's `simlock + // daemon stop` staying done depends on the timer never reaching for it. + const clock = new FakeClock(0); + const first = new FakeSimlockClient(); + first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const second = new FakeSimlockClient(); + second.renewLeaseImpl = (input) => + Promise.resolve({ + deviceId: "device-1", + grantedAt: 0, + id: input.leaseId, + lastRenewedAt: clock.now(), + idChosenByRequester: false, + ownerId: "agent-1", + requesterId: "agent-1", + ttlMs: 1_000, + ttlDeadline: clock.now() + 1_000, + }); + + let toolCallConnects = 0; + let renewConnects = 0; + const session = new McpSession({ + clock, + connect: async () => { + toolCallConnects += 1; + // why: Reached a second time only by a timer taking the launching path -- which would be + // the bug, so it fails loudly here rather than quietly handing back a client. + if (toolCallConnects > 1) throw new Error("the renew timer reached the launching connect"); + return first; + }, + connectForRenew: async () => { + renewConnects += 1; + return second; + }, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + expect(toolCallConnects).toBe(1); + expect(renewConnects).toBe(0); + + first.emitConnectionLost(); + + clock.advance(4_115); + await flushMicrotasks(); + expect(renewConnects).toBe(1); + expect(toolCallConnects).toBe(1); + expect(second.calls.map((call) => call.method)).toEqual(["renewLease"]); + + // why: And the session listens to the connection it just built, not only to the one it started + // with: a lease-lost over the new wire is the agent's only warning that the device is + // gone, and a session that forgot to re-wire would sit silent through it. + second.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); + expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }]); + + await session.close(); + }); + + it("retries alone with the launching connect when a tool call joined a renew's attempt", async () => { + // why: The CLI reference and the client guide promise a tool call may start a daemon that is not + // running. Sharing `#connecting` with the renew timer is free in the other direction, but + // here it would hand the tool call the timer's weaker attempt -- and "nothing is + // listening" for something it was allowed to fix. + const clock = new FakeClock(0); + const first = new FakeSimlockClient(); + first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const second = new FakeSimlockClient(); + second.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + second.getStatusImpl = () => Promise.resolve(startingStatus()); + let toolCallConnects = 0; + let failRenewConnect!: (error: unknown) => void; + const session = new McpSession({ + clock, + connect: async () => { + toolCallConnects += 1; + return toolCallConnects === 1 ? first : second; + }, + connectForRenew: () => + new Promise((_resolve, reject) => { + failRenewConnect = reject; + }), + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + first.emitConnectionLost(); + + clock.advance(4_115); + await flushMicrotasks(); + const listing = session.listDevices({}); + await flushMicrotasks(); + expect(toolCallConnects).toBe(1); + + // why: Nothing is listening, so that attempt fails -- for the timer, which will try again on + // its own cadence, and for the tool call, which instead tries once more with the connect + // that may start a daemon. + failRenewConnect( + new SimlockError("DAEMON_CONNECTION_LOST", "transport", "nothing is listening", {}), + ); + + await expect(listing).resolves.toEqual({ platforms: [], physicalDevices: [] }); + expect(toolCallConnects, "the tool call's own attempt, made alone").toBe(2); + + await session.close(); + }); + + it("lets a renew tick share a tool call's connect, which is the direction that costs nothing", async () => { + // why: The timer wanted a connection and got one. That the tool call was going to launch a + // daemon anyway is the tool call's business, not the timer's doing -- so this direction + // stays shared, and the session ends up with one connection rather than two. + const clock = new FakeClock(0); + const first = new FakeSimlockClient(); + first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const second = new FakeSimlockClient(); + second.getCatalogImpl = () => Promise.resolve({ platforms: [] }); + second.getStatusImpl = () => Promise.resolve(startingStatus()); + second.renewLeaseImpl = (input) => + Promise.resolve({ + deviceId: "device-1", + grantedAt: 0, + id: input.leaseId, + lastRenewedAt: clock.now(), + idChosenByRequester: false, + ownerId: "mcp-test", + requesterId: "mcp-test", + ttlMs: 60_000, + ttlDeadline: clock.now() + 60_000, + }); + let toolCallConnects = 0; + let renewConnects = 0; + let answerToolCallConnect!: (client: FakeSimlockClient) => void; + const session = new McpSession({ + clock, + connect: () => { + toolCallConnects += 1; + if (toolCallConnects === 1) return Promise.resolve(first); + return new Promise((resolve) => { + answerToolCallConnect = resolve; + }); + }, + connectForRenew: async () => { + renewConnects += 1; + return second; + }, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + first.emitConnectionLost(); + + const listing = session.listDevices({}); + await flushMicrotasks(); + expect(toolCallConnects).toBe(2); + clock.advance(4_115); + await flushMicrotasks(); + expect(renewConnects, "it joined the attempt already in flight").toBe(0); + + answerToolCallConnect(second); + await expect(listing).resolves.toEqual({ platforms: [], physicalDevices: [] }); + await flushMicrotasks(); + + expect(renewConnects).toBe(0); + expect(toolCallConnects).toBe(2); + expect(second.calls.filter((call) => call.method === "renewLease")).toHaveLength(1); + + // why: Both callers came back holding this client, and exactly one of them may wire it up: a + // second set of push relays would strand the first (only the latest can be unsubscribed), + // and every push would reach the agent twice for one thing that happened. + const deviceHealth: unknown[] = []; + session.onDeviceHealth((notice) => deviceHealth.push(notice)); + second.emitDeviceUnhealthy({ deviceId: "device-1", leaseId: "lease-1" }); + expect(deviceHealth).toHaveLength(1); + + await session.close(); + }); +}); diff --git a/src/mcp/session-release.test.ts b/src/mcp/session-release.test.ts new file mode 100644 index 000000000..56c58dec5 --- /dev/null +++ b/src/mcp/session-release.test.ts @@ -0,0 +1,364 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../client/index.js"; +import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; +import { FakeClock } from "../ports/index.js"; +import { FakeSimlockClient, sampleGrant } from "./test-support.js"; +import { McpSession } from "./session.js"; +import { flushMicrotasks } from "./test-session.js"; + +describe("McpSession", () => { + it("releases the session's lease explicitly on close, before closing the connection", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + await session.close(); + + expect(client.calls.map((call) => call.method)).toEqual(["requestLease", "releaseLease"]); + expect(client.calls.at(-1)?.input).toEqual({ leaseId: "lease-1" }); + expect(client.closeCalls).toBe(1); + expect(clock.pendingTimerCount, "the renew timer must not outlive the session").toBe(0); + }); + + it("closes cleanly even when the farewell release fails", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.releaseLeaseImpl = () => + Promise.reject(new SimlockError("UNKNOWN_LEASE", "domain", "gone", { leaseId: "lease-1" })); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + await expect(session.close()).resolves.toBeUndefined(); + expect(client.closeCalls).toBe(1); + }); + + it("does not release on close when the lease was already released by a tool call", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + await session.release({ leaseId: "lease-1" }); + expect(clock.pendingTimerCount).toBe(0); + + await session.close(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(1); + }); + + it("still sends its own release when a tool call's release is already in flight", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const pending: Array<(result: { leaseId: string }) => void> = []; + client.releaseLeaseImpl = () => + new Promise<{ leaseId: string }>((resolve) => { + pending.push(resolve); + }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + // why: `close()` does not queue behind the tool-call serializer, so it can land while a + // `release_simulator` call is still waiting on the daemon. It waits for that call, then + // sends its own release anyway: deferring to the in-flight one would mean no release at + // all if the wire died first, and a duplicate only costs an `UNKNOWN_LEASE` it swallows. + const releasing = session.release({ leaseId: "lease-1" }); + await flushMicrotasks(); + const closing = session.close(); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(1); + + pending[0]?.({ leaseId: "lease-1" }); // the tool call's own release is answered + await expect(releasing).resolves.toMatchObject({ leaseId: "lease-1", released: true }); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(2); + + pending[1]?.({ leaseId: "lease-1" }); + await closing; + expect(client.closeCalls).toBe(1); + expect(clock.pendingTimerCount, "no bound outlives the close").toBe(0); + }); + + it("spends one shutdown budget in total, not one per step", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.listLeasesImpl = () => new Promise(() => {}); + client.releaseLeaseImpl = () => new Promise(() => {}); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + void session.status().catch(() => undefined); // in flight, never answered + await flushMicrotasks(); + + let closed = false; + const closing = session.close().then(() => { + closed = true; + }); + await flushMicrotasks(); + + // why: A budget per step -- one for the in-flight call, then one more for the lease -- would + // leave `simlock mcp` sitting here for a multiple of this after stdin EOF. + clock.advance(RELEASE_TIMEOUT_MS - 1); + await flushMicrotasks(); + expect(closed).toBe(false); + + clock.advance(1); + await closing; + expect(closed).toBe(true); + expect(client.closeCalls, "the wire closes when the budget is spent").toBe(1); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("stops releasing when the shutdown budget runs out, rather than calling a closed client", async () => { + // why: `awaitWithin` stops *waiting*; it cannot stop the loop it was waiting on. Without a + // check the abandoned loop would carry on down the remaining leases against the client the + // `finally` has already closed -- one rejection per lease, after the session was over. + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + let granted = 0; + client.requestLeaseImpl = () => { + granted += 1; + return Promise.resolve(sampleGrant({ leaseId: `lease-${String(granted)}` })); + }; + let answerFirstRelease!: (result: { leaseId: string }) => void; + client.releaseLeaseImpl = (input) => + input.leaseId === "lease-1" + ? new Promise((resolve) => { + answerFirstRelease = resolve; + }) + : Promise.resolve({ leaseId: input.leaseId }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + const closing = session.close(); + await flushMicrotasks(); + clock.advance(RELEASE_TIMEOUT_MS); + await closing; + expect(client.closeCalls).toBe(1); + + answerFirstRelease({ leaseId: "lease-1" }); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([ + { input: { leaseId: "lease-1" }, method: "releaseLease" }, + ]); + }); + + it("releases a grant that lands after the session closed, and arms no timer for it", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + let answerGrant!: (grant: ReturnType) => void; + client.requestLeaseImpl = () => + new Promise((resolve) => { + answerGrant = resolve; + }); + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + + const leasing = session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + await flushMicrotasks(); + const closing = session.close(); + await flushMicrotasks(); + + answerGrant(sampleGrant({ leaseId: "lease-1" })); + await expect(leasing).rejects.toMatchObject({ code: "SESSION_CLOSED" }); + await closing; + + expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([ + { input: { leaseId: "lease-1" }, method: "releaseLease" }, + ]); + expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); + expect(clock.pendingTimerCount, "a closed session must leave no timer behind").toBe(0); + }); + + it("stops renewing before it asks to release, so a renew cannot overtake the answer", async () => { + // why: ADR 0003 §8: `onLeaseLost` never fires for a release this same client asked for. A timer + // left armed across an in-flight release breaks that -- the renew lands after + // `Registry#beginRelease` has dropped the record, comes back UNKNOWN_LEASE, and renewal + // reports the agent's own release to it as a lost device. Ordering hides it on a unix + // socket; a gateway hop (ADR 0003 §12) would not. + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + let answerRelease!: (result: { leaseId: string }) => void; + client.releaseLeaseImpl = () => + new Promise((resolve) => { + answerRelease = resolve; + }); + client.renewLeaseImpl = () => + Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), + ); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + const releasing = session.release({ leaseId: "lease-1" }); + await flushMicrotasks(); + + // why: The daemon has the release and has not answered yet. This is exactly the window a renew + // would land in -- and there is no timer left to send one. + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); + + answerRelease({ leaseId: "lease-1" }); + await expect(releasing).resolves.toEqual({ leaseId: "lease-1", released: true }); + expect(notices, "the agent asked for this; it is not news of a lost device").toEqual([]); + + await session.close(); + }); + + it("does not put the timer back for a lease that was lost while the release was in flight", async () => { + // why: A failed release usually means the session still holds the device, so the timer goes + // back. Not here: the daemon ended this lease (an expiry, an operator) while the release + // was on the wire, so there is nothing left to renew and the retry would run a ladder + // against a record that no longer exists. + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + let failRelease!: (error: unknown) => void; + client.releaseLeaseImpl = () => + new Promise((_resolve, reject) => { + failRelease = reject; + }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + const releasing = session.release({ leaseId: "lease-1" }); + await flushMicrotasks(); + client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); + + // why: Transient on its face -- the daemon may serve the next one -- but the lease it was about + // is gone regardless. + failRelease(new SimlockError("INTERNAL", "domain", "could not release", {})); + await expect(releasing).rejects.toMatchObject({ code: "INTERNAL" }); + + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); + expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }]); + + await session.close(); + }); + + it("leaves the timer stopped when the release fails because the lease is already gone", async () => { + // why: The other side of restarting a stopped timer: an UNKNOWN_LEASE means there is nothing + // left to renew, so putting the timer back would only produce the same answer on a ladder + // until the deadline -- and announce it as a lease lost. + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.releaseLeaseImpl = () => + Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), + ); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + const notices: unknown[] = []; + session.onLeaseLost((notice) => notices.push(notice)); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + await expect(session.release({ leaseId: "lease-1" })).rejects.toMatchObject({ + code: "UNKNOWN_LEASE", + }); + + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + await flushMicrotasks(); + expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); + expect(notices).toEqual([]); + + await session.close(); + }); + + it("keeps renewing when a release fails, because the session still holds the device", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + client.releaseLeaseImpl = () => + Promise.reject(new SimlockError("INTERNAL", "domain", "could not release", {})); + let renewals = 0; + client.renewLeaseImpl = (input) => { + renewals += 1; + return Promise.resolve({ + deviceId: "device-1", + grantedAt: 0, + id: input.leaseId, + ownerId: "mcp-test", + requesterId: "mcp-test", + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: clock.now() + 12_000, + }); + }; + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + await expect(session.release({ leaseId: "lease-1" })).rejects.toMatchObject({ + code: "INTERNAL", + }); + + // why: The daemon still has the lease, so the timer that keeps it must still be running -- + // otherwise the device is reclaimed at the deadline under a session that was told its + // release failed. + clock.advance(4_115); + await flushMicrotasks(); + expect(renewals).toBe(1); + await session.close(); + }); +}); diff --git a/src/mcp/session-renewal.test.ts b/src/mcp/session-renewal.test.ts new file mode 100644 index 000000000..e9c66d031 --- /dev/null +++ b/src/mcp/session-renewal.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { FakeSimlockClient, sampleGrant } from "./test-support.js"; +import { McpSession } from "./session.js"; +import { flushMicrotasks } from "./test-session.js"; + +describe("McpSession", () => { + it("renews its lease at a third of the TTL, re-deriving the cadence from each renewal", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); + const renewals: Array<{ leaseId: string; at: number }> = []; + client.renewLeaseImpl = (input) => { + renewals.push({ at: clock.now(), leaseId: input.leaseId }); + return Promise.resolve({ + deviceId: "device-1", + grantedAt: 0, + id: input.leaseId, + ownerId: "mcp-test", + requesterId: "mcp-test", + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: clock.now() + 3_000, + }); + }; + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + + // why: `sampleGrant`'s deadline is 12_345, so the first renewal is due at 4_115. + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + clock.advance(4_114); + await flushMicrotasks(); + expect(renewals).toEqual([]); + clock.advance(1); + await flushMicrotasks(); + expect(renewals).toEqual([{ at: 4_115, leaseId: "lease-1" }]); + + // why: 3_000ms came back, so the next one is 1_000ms later -- off the renewal's own deadline. + clock.advance(1_000); + await flushMicrotasks(); + expect(renewals).toHaveLength(2); + expect(renewals[1]).toEqual({ at: 5_115, leaseId: "lease-1" }); + + await session.close(); + }); + + it("renews on the lease's own width, not on the time left to its deadline", async () => { + // why: The session passes `ttlMs` as well as `ttlDeadline` (see `startLeaseRenewal`): the width + // sets the cadence, the deadline only bounds it. A lease granted narrow against a distant + // deadline is where the two visibly differ -- 10_000 apart, not 100_000. + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + const grant = sampleGrant({ leaseId: "lease-1" }); + client.requestLeaseImpl = () => + Promise.resolve({ + ...grant, + lease: { ...grant.lease, ttlDeadline: 300_000, ttlMs: 30_000 }, + }); + const renewedAt: number[] = []; + client.renewLeaseImpl = (input) => { + renewedAt.push(clock.now()); + return Promise.resolve({ + ...grant.lease, + id: input.leaseId, + lastRenewedAt: clock.now(), + idChosenByRequester: false, + ttlDeadline: clock.now() + 300_000, + ttlMs: 30_000, + }); + }; + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + clock.advance(10_000); + await flushMicrotasks(); + expect(renewedAt).toEqual([10_000]); + + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + await session.close(); + }); + + it("renews and releases every lease it obtained, not only the latest", async () => { + const clock = new FakeClock(0); + const client = new FakeSimlockClient(); + let granted = 0; + client.requestLeaseImpl = () => { + granted += 1; + return Promise.resolve(sampleGrant({ leaseId: `lease-${granted}` })); + }; + const renewed: string[] = []; + client.renewLeaseImpl = (input) => { + renewed.push(input.leaseId); + return Promise.resolve({ + deviceId: "device-1", + grantedAt: 0, + id: input.leaseId, + lastRenewedAt: clock.now(), + idChosenByRequester: false, + ownerId: "mcp-test", + requesterId: "mcp-test", + ttlMs: 12_345, + ttlDeadline: clock.now() + 12_345, + }); + }; + client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); + const session = new McpSession({ + clock, + connect: async () => client, + connectForRenew: async () => client, + }); + + // why: Nothing here limits a session to one lease -- the daemon's one-lease-per-requester rule + // is the authority (ADR 0003 §11). Whatever it grants, this session has to keep alive and + // hand back; the second grant must not silently strand the first. + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); + + clock.advance(4_115); + await flushMicrotasks(); + expect(renewed.slice().sort()).toEqual(["lease-1", "lease-2"]); + + await session.close(); + expect( + client.calls + .filter((call) => call.method === "releaseLease") + .map((call) => (call.input as { leaseId: string }).leaseId) + .sort(), + ).toEqual(["lease-1", "lease-2"]); + expect(clock.pendingTimerCount).toBe(0); + }); +}); diff --git a/src/mcp/session.test.ts b/src/mcp/session.test.ts index 4b57e98f9..e5feb3bcd 100644 --- a/src/mcp/session.test.ts +++ b/src/mcp/session.test.ts @@ -1,16 +1,9 @@ import { describe, expect, it } from "vitest"; - import { SimlockError } from "../client/index.js"; -import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; import { FakeClock } from "../ports/index.js"; import { FakeSimlockClient, sampleGrant, startingStatus } from "./test-support.js"; import { McpSession } from "./session.js"; -/** why: Lets an awaited renewal settle; the fake clock never moves on its own. */ -async function flushMicrotasks(times = 10): Promise { - for (let index = 0; index < times; index += 1) await Promise.resolve(); -} - describe("McpSession", () => { it("requests a held lease with the tool input as-is, and returns the grant unchanged", async () => { const client = new FakeSimlockClient(); @@ -284,851 +277,6 @@ describe("McpSession", () => { await expect(session.status()).rejects.toMatchObject({ code: "SESSION_CLOSED" }); expect(client.calls).toEqual([]); }); - - it("renews its lease at a third of the TTL, re-deriving the cadence from each renewal", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const renewals: Array<{ leaseId: string; at: number }> = []; - client.renewLeaseImpl = (input) => { - renewals.push({ at: clock.now(), leaseId: input.leaseId }); - return Promise.resolve({ - deviceId: "device-1", - grantedAt: 0, - id: input.leaseId, - ownerId: "mcp-test", - requesterId: "mcp-test", - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: clock.now() + 3_000, - }); - }; - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - - // why: `sampleGrant`'s deadline is 12_345, so the first renewal is due at 4_115. - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - clock.advance(4_114); - await flushMicrotasks(); - expect(renewals).toEqual([]); - clock.advance(1); - await flushMicrotasks(); - expect(renewals).toEqual([{ at: 4_115, leaseId: "lease-1" }]); - - // why: 3_000ms came back, so the next one is 1_000ms later -- off the renewal's own deadline. - clock.advance(1_000); - await flushMicrotasks(); - expect(renewals).toHaveLength(2); - expect(renewals[1]).toEqual({ at: 5_115, leaseId: "lease-1" }); - - await session.close(); - }); - - it("renews on the lease's own width, not on the time left to its deadline", async () => { - // why: The session passes `ttlMs` as well as `ttlDeadline` (see `startLeaseRenewal`): the width - // sets the cadence, the deadline only bounds it. A lease granted narrow against a distant - // deadline is where the two visibly differ -- 10_000 apart, not 100_000. - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - const grant = sampleGrant({ leaseId: "lease-1" }); - client.requestLeaseImpl = () => - Promise.resolve({ - ...grant, - lease: { ...grant.lease, ttlDeadline: 300_000, ttlMs: 30_000 }, - }); - const renewedAt: number[] = []; - client.renewLeaseImpl = (input) => { - renewedAt.push(clock.now()); - return Promise.resolve({ - ...grant.lease, - id: input.leaseId, - lastRenewedAt: clock.now(), - idChosenByRequester: false, - ttlDeadline: clock.now() + 300_000, - ttlMs: 30_000, - }); - }; - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - clock.advance(10_000); - await flushMicrotasks(); - expect(renewedAt).toEqual([10_000]); - - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - await session.close(); - }); - - it("releases the session's lease explicitly on close, before closing the connection", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - await session.close(); - - expect(client.calls.map((call) => call.method)).toEqual(["requestLease", "releaseLease"]); - expect(client.calls.at(-1)?.input).toEqual({ leaseId: "lease-1" }); - expect(client.closeCalls).toBe(1); - expect(clock.pendingTimerCount, "the renew timer must not outlive the session").toBe(0); - }); - - it("closes cleanly even when the farewell release fails", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.releaseLeaseImpl = () => - Promise.reject(new SimlockError("UNKNOWN_LEASE", "domain", "gone", { leaseId: "lease-1" })); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - await expect(session.close()).resolves.toBeUndefined(); - expect(client.closeCalls).toBe(1); - }); - - it("does not release on close when the lease was already released by a tool call", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - await session.release({ leaseId: "lease-1" }); - expect(clock.pendingTimerCount).toBe(0); - - await session.close(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(1); - }); - - it("renews and releases every lease it obtained, not only the latest", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - let granted = 0; - client.requestLeaseImpl = () => { - granted += 1; - return Promise.resolve(sampleGrant({ leaseId: `lease-${granted}` })); - }; - const renewed: string[] = []; - client.renewLeaseImpl = (input) => { - renewed.push(input.leaseId); - return Promise.resolve({ - deviceId: "device-1", - grantedAt: 0, - id: input.leaseId, - lastRenewedAt: clock.now(), - idChosenByRequester: false, - ownerId: "mcp-test", - requesterId: "mcp-test", - ttlMs: 12_345, - ttlDeadline: clock.now() + 12_345, - }); - }; - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - - // why: Nothing here limits a session to one lease -- the daemon's one-lease-per-requester rule - // is the authority (ADR 0003 §11). Whatever it grants, this session has to keep alive and - // hand back; the second grant must not silently strand the first. - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - clock.advance(4_115); - await flushMicrotasks(); - expect(renewed.slice().sort()).toEqual(["lease-1", "lease-2"]); - - await session.close(); - expect( - client.calls - .filter((call) => call.method === "releaseLease") - .map((call) => (call.input as { leaseId: string }).leaseId) - .sort(), - ).toEqual(["lease-1", "lease-2"]); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("still sends its own release when a tool call's release is already in flight", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const pending: Array<(result: { leaseId: string }) => void> = []; - client.releaseLeaseImpl = () => - new Promise<{ leaseId: string }>((resolve) => { - pending.push(resolve); - }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - // why: `close()` does not queue behind the tool-call serializer, so it can land while a - // `release_simulator` call is still waiting on the daemon. It waits for that call, then - // sends its own release anyway: deferring to the in-flight one would mean no release at - // all if the wire died first, and a duplicate only costs an `UNKNOWN_LEASE` it swallows. - const releasing = session.release({ leaseId: "lease-1" }); - await flushMicrotasks(); - const closing = session.close(); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(1); - - pending[0]?.({ leaseId: "lease-1" }); // the tool call's own release is answered - await expect(releasing).resolves.toMatchObject({ leaseId: "lease-1", released: true }); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toHaveLength(2); - - pending[1]?.({ leaseId: "lease-1" }); - await closing; - expect(client.closeCalls).toBe(1); - expect(clock.pendingTimerCount, "no bound outlives the close").toBe(0); - }); - - it("spends one shutdown budget in total, not one per step", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.listLeasesImpl = () => new Promise(() => {}); - client.releaseLeaseImpl = () => new Promise(() => {}); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - void session.status().catch(() => undefined); // in flight, never answered - await flushMicrotasks(); - - let closed = false; - const closing = session.close().then(() => { - closed = true; - }); - await flushMicrotasks(); - - // why: A budget per step -- one for the in-flight call, then one more for the lease -- would - // leave `simlock mcp` sitting here for a multiple of this after stdin EOF. - clock.advance(RELEASE_TIMEOUT_MS - 1); - await flushMicrotasks(); - expect(closed).toBe(false); - - clock.advance(1); - await closing; - expect(closed).toBe(true); - expect(client.closeCalls, "the wire closes when the budget is spent").toBe(1); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("stops releasing when the shutdown budget runs out, rather than calling a closed client", async () => { - // why: `awaitWithin` stops *waiting*; it cannot stop the loop it was waiting on. Without a - // check the abandoned loop would carry on down the remaining leases against the client the - // `finally` has already closed -- one rejection per lease, after the session was over. - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - let granted = 0; - client.requestLeaseImpl = () => { - granted += 1; - return Promise.resolve(sampleGrant({ leaseId: `lease-${String(granted)}` })); - }; - let answerFirstRelease!: (result: { leaseId: string }) => void; - client.releaseLeaseImpl = (input) => - input.leaseId === "lease-1" - ? new Promise((resolve) => { - answerFirstRelease = resolve; - }) - : Promise.resolve({ leaseId: input.leaseId }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - const closing = session.close(); - await flushMicrotasks(); - clock.advance(RELEASE_TIMEOUT_MS); - await closing; - expect(client.closeCalls).toBe(1); - - answerFirstRelease({ leaseId: "lease-1" }); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([ - { input: { leaseId: "lease-1" }, method: "releaseLease" }, - ]); - }); - - it("releases a grant that lands after the session closed, and arms no timer for it", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - let answerGrant!: (grant: ReturnType) => void; - client.requestLeaseImpl = () => - new Promise((resolve) => { - answerGrant = resolve; - }); - client.releaseLeaseImpl = (input) => Promise.resolve({ leaseId: input.leaseId }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - - const leasing = session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - await flushMicrotasks(); - const closing = session.close(); - await flushMicrotasks(); - - answerGrant(sampleGrant({ leaseId: "lease-1" })); - await expect(leasing).rejects.toMatchObject({ code: "SESSION_CLOSED" }); - await closing; - - expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([ - { input: { leaseId: "lease-1" }, method: "releaseLease" }, - ]); - expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); - expect(clock.pendingTimerCount, "a closed session must leave no timer behind").toBe(0); - }); - - it("stops renewing before it asks to release, so a renew cannot overtake the answer", async () => { - // why: ADR 0003 §8: `onLeaseLost` never fires for a release this same client asked for. A timer - // left armed across an in-flight release breaks that -- the renew lands after - // `Registry#beginRelease` has dropped the record, comes back UNKNOWN_LEASE, and renewal - // reports the agent's own release to it as a lost device. Ordering hides it on a unix - // socket; a gateway hop (ADR 0003 §12) would not. - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - let answerRelease!: (result: { leaseId: string }) => void; - client.releaseLeaseImpl = () => - new Promise((resolve) => { - answerRelease = resolve; - }); - client.renewLeaseImpl = () => - Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), - ); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - const releasing = session.release({ leaseId: "lease-1" }); - await flushMicrotasks(); - - // why: The daemon has the release and has not answered yet. This is exactly the window a renew - // would land in -- and there is no timer left to send one. - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); - - answerRelease({ leaseId: "lease-1" }); - await expect(releasing).resolves.toEqual({ leaseId: "lease-1", released: true }); - expect(notices, "the agent asked for this; it is not news of a lost device").toEqual([]); - - await session.close(); - }); - - it("does not put the timer back for a lease that was lost while the release was in flight", async () => { - // why: A failed release usually means the session still holds the device, so the timer goes - // back. Not here: the daemon ended this lease (an expiry, an operator) while the release - // was on the wire, so there is nothing left to renew and the retry would run a ladder - // against a record that no longer exists. - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - let failRelease!: (error: unknown) => void; - client.releaseLeaseImpl = () => - new Promise((_resolve, reject) => { - failRelease = reject; - }); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - const releasing = session.release({ leaseId: "lease-1" }); - await flushMicrotasks(); - client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); - - // why: Transient on its face -- the daemon may serve the next one -- but the lease it was about - // is gone regardless. - failRelease(new SimlockError("INTERNAL", "domain", "could not release", {})); - await expect(releasing).rejects.toMatchObject({ code: "INTERNAL" }); - - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); - expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }]); - - await session.close(); - }); - - it("leaves the timer stopped when the release fails because the lease is already gone", async () => { - // why: The other side of restarting a stopped timer: an UNKNOWN_LEASE means there is nothing - // left to renew, so putting the timer back would only produce the same answer on a ladder - // until the deadline -- and announce it as a lease lost. - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.releaseLeaseImpl = () => - Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), - ); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - await expect(session.release({ leaseId: "lease-1" })).rejects.toMatchObject({ - code: "UNKNOWN_LEASE", - }); - - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); - expect(notices).toEqual([]); - - await session.close(); - }); - - it("keeps renewing when a release fails, because the session still holds the device", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.releaseLeaseImpl = () => - Promise.reject(new SimlockError("INTERNAL", "domain", "could not release", {})); - let renewals = 0; - client.renewLeaseImpl = (input) => { - renewals += 1; - return Promise.resolve({ - deviceId: "device-1", - grantedAt: 0, - id: input.leaseId, - ownerId: "mcp-test", - requesterId: "mcp-test", - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: clock.now() + 12_000, - }); - }; - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - await expect(session.release({ leaseId: "lease-1" })).rejects.toMatchObject({ - code: "INTERNAL", - }); - - // why: The daemon still has the lease, so the timer that keeps it must still be running -- - // otherwise the device is reclaimed at the deadline under a session that was told its - // release failed. - clock.advance(4_115); - await flushMicrotasks(); - expect(renewals).toBe(1); - await session.close(); - }); - - it("reports a lease the daemon says is gone as lease-lost, and releases nothing on close", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.renewLeaseImpl = () => - Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), - ); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - // why: A lease that expired while this session was idle may never produce a push -- the - // renewal's own answer is what tells the agent. - clock.advance(4_115); - await flushMicrotasks(); - expect(notices).toEqual([ - { deviceId: "device-1", leaseId: "lease-1", reason: "renew-rejected" }, - ]); - expect(clock.pendingTimerCount).toBe(0); - - await session.close(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([]); - expect(client.calls.filter((call) => call.method === "renewLease")).toHaveLength(1); - }); - - it("reports a lease it could not keep alive as lost, and releases nothing for it on close", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - // why: Never terminal, never successful: the ladder retries until `sampleGrant`'s own 12_345 - // deadline passes and renewal gives up. - client.renewLeaseImpl = () => - Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist", {})); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - clock.advance(4_115); - await flushMicrotasks(); - clock.advance(20_000); // the retry lands past the deadline - await flushMicrotasks(); - - expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "renew-failed" }]); - expect(clock.pendingTimerCount).toBe(0); - - await session.close(); - expect( - client.calls.filter((call) => call.method === "releaseLease"), - "a lease that expired is not the session's to release", - ).toEqual([]); - }); - - it("announces one ending once, even when the renewal and the daemon both report it", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - client.renewLeaseImpl = () => - Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lease-1" }), - ); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - clock.advance(4_115); - await flushMicrotasks(); - // why: The daemon's own push for the same lease arrives a moment later; an agent must not read - // that as a second device lost. - client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); - - expect(notices).toEqual([ - { deviceId: "device-1", leaseId: "lease-1", reason: "renew-rejected" }, - ]); - - await session.close(); - }); - - it("renews over connectForRenew after a dead connection, and never over connect", async () => { - // why: ADR 0004 §2's named safety property: the renew timer reconnects so an idle session does - // not lose its lease waiting for a tool call, but it reaches only a daemon that is already - // listening. `connect` is the trigger that may auto-launch one, so an operator's `simlock - // daemon stop` staying done depends on the timer never reaching for it. - const clock = new FakeClock(0); - const first = new FakeSimlockClient(); - first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const second = new FakeSimlockClient(); - second.renewLeaseImpl = (input) => - Promise.resolve({ - deviceId: "device-1", - grantedAt: 0, - id: input.leaseId, - lastRenewedAt: clock.now(), - idChosenByRequester: false, - ownerId: "agent-1", - requesterId: "agent-1", - ttlMs: 1_000, - ttlDeadline: clock.now() + 1_000, - }); - - let toolCallConnects = 0; - let renewConnects = 0; - const session = new McpSession({ - clock, - connect: async () => { - toolCallConnects += 1; - // why: Reached a second time only by a timer taking the launching path -- which would be - // the bug, so it fails loudly here rather than quietly handing back a client. - if (toolCallConnects > 1) throw new Error("the renew timer reached the launching connect"); - return first; - }, - connectForRenew: async () => { - renewConnects += 1; - return second; - }, - }); - const notices: unknown[] = []; - session.onLeaseLost((notice) => notices.push(notice)); - - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - expect(toolCallConnects).toBe(1); - expect(renewConnects).toBe(0); - - first.emitConnectionLost(); - - clock.advance(4_115); - await flushMicrotasks(); - expect(renewConnects).toBe(1); - expect(toolCallConnects).toBe(1); - expect(second.calls.map((call) => call.method)).toEqual(["renewLease"]); - - // why: And the session listens to the connection it just built, not only to the one it started - // with: a lease-lost over the new wire is the agent's only warning that the device is - // gone, and a session that forgot to re-wire would sit silent through it. - second.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); - expect(notices).toEqual([{ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }]); - - await session.close(); - }); - - it("retries alone with the launching connect when a tool call joined a renew's attempt", async () => { - // why: The CLI reference and the client guide promise a tool call may start a daemon that is not - // running. Sharing `#connecting` with the renew timer is free in the other direction, but - // here it would hand the tool call the timer's weaker attempt -- and "nothing is - // listening" for something it was allowed to fix. - const clock = new FakeClock(0); - const first = new FakeSimlockClient(); - first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const second = new FakeSimlockClient(); - second.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - second.getStatusImpl = () => Promise.resolve(startingStatus()); - let toolCallConnects = 0; - let failRenewConnect!: (error: unknown) => void; - const session = new McpSession({ - clock, - connect: async () => { - toolCallConnects += 1; - return toolCallConnects === 1 ? first : second; - }, - connectForRenew: () => - new Promise((_resolve, reject) => { - failRenewConnect = reject; - }), - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - first.emitConnectionLost(); - - clock.advance(4_115); - await flushMicrotasks(); - const listing = session.listDevices({}); - await flushMicrotasks(); - expect(toolCallConnects).toBe(1); - - // why: Nothing is listening, so that attempt fails -- for the timer, which will try again on - // its own cadence, and for the tool call, which instead tries once more with the connect - // that may start a daemon. - failRenewConnect( - new SimlockError("DAEMON_CONNECTION_LOST", "transport", "nothing is listening", {}), - ); - - await expect(listing).resolves.toEqual({ platforms: [], physicalDevices: [] }); - expect(toolCallConnects, "the tool call's own attempt, made alone").toBe(2); - - await session.close(); - }); - - it("lets a renew tick share a tool call's connect, which is the direction that costs nothing", async () => { - // why: The timer wanted a connection and got one. That the tool call was going to launch a - // daemon anyway is the tool call's business, not the timer's doing -- so this direction - // stays shared, and the session ends up with one connection rather than two. - const clock = new FakeClock(0); - const first = new FakeSimlockClient(); - first.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const second = new FakeSimlockClient(); - second.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - second.getStatusImpl = () => Promise.resolve(startingStatus()); - second.renewLeaseImpl = (input) => - Promise.resolve({ - deviceId: "device-1", - grantedAt: 0, - id: input.leaseId, - lastRenewedAt: clock.now(), - idChosenByRequester: false, - ownerId: "mcp-test", - requesterId: "mcp-test", - ttlMs: 60_000, - ttlDeadline: clock.now() + 60_000, - }); - let toolCallConnects = 0; - let renewConnects = 0; - let answerToolCallConnect!: (client: FakeSimlockClient) => void; - const session = new McpSession({ - clock, - connect: () => { - toolCallConnects += 1; - if (toolCallConnects === 1) return Promise.resolve(first); - return new Promise((resolve) => { - answerToolCallConnect = resolve; - }); - }, - connectForRenew: async () => { - renewConnects += 1; - return second; - }, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - first.emitConnectionLost(); - - const listing = session.listDevices({}); - await flushMicrotasks(); - expect(toolCallConnects).toBe(2); - clock.advance(4_115); - await flushMicrotasks(); - expect(renewConnects, "it joined the attempt already in flight").toBe(0); - - answerToolCallConnect(second); - await expect(listing).resolves.toEqual({ platforms: [], physicalDevices: [] }); - await flushMicrotasks(); - - expect(renewConnects).toBe(0); - expect(toolCallConnects).toBe(2); - expect(second.calls.filter((call) => call.method === "renewLease")).toHaveLength(1); - - // why: Both callers came back holding this client, and exactly one of them may wire it up: a - // second set of push relays would strand the first (only the latest can be unsubscribed), - // and every push would reach the agent twice for one thing that happened. - const deviceHealth: unknown[] = []; - session.onDeviceHealth((notice) => deviceHealth.push(notice)); - second.emitDeviceUnhealthy({ deviceId: "device-1", leaseId: "lease-1" }); - expect(deviceHealth).toHaveLength(1); - - await session.close(); - }); - - it("stops renewing a lease that ended elsewhere", async () => { - const clock = new FakeClock(0); - const client = new FakeSimlockClient(); - client.requestLeaseImpl = () => Promise.resolve(sampleGrant({ leaseId: "lease-1" })); - const session = new McpSession({ - clock, - connect: async () => client, - connectForRenew: async () => client, - }); - await session.lease({ model: "iPhone 17 Pro", platform: "ios" }); - - client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-1", reason: "expired" }); - - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - await flushMicrotasks(); - expect(client.calls.filter((call) => call.method === "renewLease")).toEqual([]); - - await session.close(); - expect(client.calls.filter((call) => call.method === "releaseLease")).toEqual([]); - }); - - it("remembers the endings it announced up to a cap, dropping the oldest first", async () => { - // why: The dedupe memory of a long-lived session cannot grow without bound. It is a fixed - // window over the most recent endings, which is where duplicates actually live: a - // renewal's answer and the daemon's push about one lease arrive moments apart, never - // dozens of leases later. - const client = new FakeSimlockClient(); - client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - client.getStatusImpl = () => Promise.resolve(startingStatus()); - const session = new McpSession({ - clock: new FakeClock(), - connect: async () => client, - connectForRenew: async () => client, - }); - await session.listDevices({}); // wires the push relays up - const notices: Array<{ leaseId: string }> = []; - session.onLeaseLost((notice) => notices.push(notice)); - - // why: One past the cap, so the very first id has been dropped by the end of it. - const announced = 65; - for (let index = 0; index < announced; index += 1) { - client.emitLeaseLost({ - deviceId: "device-1", - leaseId: `lease-${String(index)}`, - reason: "expired", - }); - } - expect(notices).toHaveLength(announced); - - client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-64", reason: "expired" }); - expect(notices).toHaveLength(announced); - - // why: The oldest has aged out of the window, so its duplicate -- if one ever came, long after - // everything else -- would be announced again. That is the trade the cap makes. - client.emitLeaseLost({ deviceId: "device-1", leaseId: "lease-0", reason: "expired" }); - expect(notices).toHaveLength(announced + 1); - expect(notices.at(-1)?.leaseId).toBe("lease-0"); - - await session.close(); - }); - - it("relays the client's lease-lost, device-unhealthy, and device-recovered pushes to session listeners", async () => { - const client = new FakeSimlockClient(); - client.getCatalogImpl = () => Promise.resolve({ platforms: [] }); - client.getStatusImpl = () => Promise.resolve(startingStatus()); - const session = new McpSession({ - clock: new FakeClock(), - connect: async () => client, - connectForRenew: async () => client, - }); - await session.listDevices({}); // forces the client to connect and wire up push relays - - const leaseLost: unknown[] = []; - const deviceHealth: unknown[] = []; - session.onLeaseLost((notice) => leaseLost.push(notice)); - session.onDeviceHealth((notice) => deviceHealth.push(notice)); - - client.emitLeaseLost({ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }); - client.emitDeviceUnhealthy({ deviceId: "SIM-1", leaseId: "lease-1" }); - client.emitDeviceRecovered({ attempts: 2, deviceId: "SIM-1", leaseId: "lease-1" }); - - expect(leaseLost).toEqual([{ deviceId: "SIM-1", leaseId: "lease-1", reason: "expired" }]); - expect(deviceHealth).toEqual([ - { deviceId: "SIM-1", kind: "unhealthy", leaseId: "lease-1", reason: "crashed" }, - { attempts: 2, deviceId: "SIM-1", kind: "recovered", leaseId: "lease-1" }, - ]); - }); }); /** diff --git a/src/mcp/test-session.ts b/src/mcp/test-session.ts new file mode 100644 index 000000000..972f9677a --- /dev/null +++ b/src/mcp/test-session.ts @@ -0,0 +1,4 @@ +/** why: Lets an awaited renewal settle; the fake clock never moves on its own. */ +export async function flushMicrotasks(times = 10): Promise { + for (let index = 0; index < times; index += 1) await Promise.resolve(); +} From afbcb3dc71a2c5ad5058450420bf28f3888113c9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:36:50 +0100 Subject: [PATCH 72/83] test(tunnel): split worker.test.ts by topic --- src/tunnel/test-worker.ts | 87 ++++++ src/tunnel/worker-emulator-tunnel.test.ts | 158 ++++++++++ src/tunnel/worker-forward-tunnels.test.ts | 113 +++++++ src/tunnel/worker.test.ts | 352 +--------------------- 4 files changed, 360 insertions(+), 350 deletions(-) create mode 100644 src/tunnel/test-worker.ts create mode 100644 src/tunnel/worker-emulator-tunnel.test.ts create mode 100644 src/tunnel/worker-forward-tunnels.test.ts diff --git a/src/tunnel/test-worker.ts b/src/tunnel/test-worker.ts new file mode 100644 index 000000000..6575540c4 --- /dev/null +++ b/src/tunnel/test-worker.ts @@ -0,0 +1,87 @@ +import { expect } from "vitest"; +import type { DeviceRecord, LeaseRecord } from "../core/index.js"; +import { FakeDeviceTcp } from "../core/testing.js"; +import type { WorkerModule } from "../modules/index.js"; +import { fakeWorkerContext } from "../modules/testing.js"; +import { type FakeClock, MemoryLeaseSocket } from "../ports/index.js"; +import { tunnelModule } from "./index.js"; + +export const IDENTITY = { principal: "tok_holder", role: "agent" } as const; + +export function frame(tag: number, conn: number, body = ""): Uint8Array { + const bytes = new TextEncoder().encode(body); + const out = new Uint8Array(5 + bytes.length); + out[0] = tag; + new DataView(out.buffer).setUint32(1, conn); + out.set(bytes, 5); + return out; +} + +export function frames(socket: MemoryLeaseSocket): { tag: number; conn: number; body: string }[] { + return socket.sent.map((sent) => ({ + body: new TextDecoder().decode(sent.subarray(5)), + conn: new DataView(sent.buffer, sent.byteOffset).getUint32(1), + tag: sent[0] ?? -1, + })); +} + +export function ending(reason: "device-lost" | "expired" | "explicit" | "killed") { + return { + deviceId: "dev_1", + grantedAt: 1_000, + leaseId: "L-1", + ownerId: "tok_holder", + platform: "android", + reason, + } as const; +} + +export function hello(...tunnels: { kind: string; devicePort: number }[]): Uint8Array { + return frame(0x01, 0, JSON.stringify({ tunnels })); +} + +export async function setup( + platform = "android", + physical = false, +): Promise<{ + readonly worker: WorkerModule; + readonly deviceTcp: FakeDeviceTcp; + readonly clock: FakeClock; +}> { + const { clock, ctx } = await fakeWorkerContext("tunnel", {}); + const deviceTcp = new FakeDeviceTcp(); + const registry = { + snapshot: { + devices: [{ id: "dev_1", physical, spec: { platform } } as unknown as DeviceRecord], + leases: [ + { deviceId: "dev_1", id: "L-1" } as unknown as LeaseRecord, + { deviceId: "dev_1", id: "L-2" } as unknown as LeaseRecord, + ], + }, + }; + const loaded = await tunnelModule.loadWorker(); + const worker = loaded.createWorker({ + ...ctx, + deviceTcp: { + connect: (_platform, device, port, signal) => deviceTcp.connect(device, port, signal), + supports: (named) => named === "android", + supportsEmulatorPort: (named, device) => named === "android" && !device.physical, + connectEmulatorPort: (_platform, device, name, signal) => + deviceTcp.connectEmulatorPort(device, name, signal), + reverse: (_platform, device, port, signal) => deviceTcp.reverse(device, port, signal), + }, + registry, + }); + return { clock, deviceTcp, worker }; +} + +export async function attach(worker: WorkerModule, leaseId = "L-1"): Promise { + const opened = await worker.socket?.open(new Request("http://simlock/tunnel"), IDENTITY, { + deviceId: "dev_1", + id: leaseId, + }); + expect(opened).toBeTypeOf("function"); + const socket = new MemoryLeaseSocket(); + if (typeof opened === "function") opened(socket); + return socket; +} diff --git a/src/tunnel/worker-emulator-tunnel.test.ts b/src/tunnel/worker-emulator-tunnel.test.ts new file mode 100644 index 000000000..a105d6fb0 --- /dev/null +++ b/src/tunnel/worker-emulator-tunnel.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it, vi } from "vitest"; +import type { FakeTcpPeer } from "../ports/index.js"; +import { attach, frame, frames, hello, setup } from "./test-worker.js"; + +describe("a worker's tunnel socket with an emulator tunnel", () => { + const emulatorHello = (): Uint8Array => + frame(0x01, 0, JSON.stringify({ tunnels: [{ kind: "emulator", emulatorPort: "adb" }] })); + const open = (conn: number, body: unknown = { kind: "emulator", emulatorPort: "adb" }) => + frame(0x02, conn, JSON.stringify(body)); + + async function ready(withEmulator = true) { + const { deviceTcp, worker } = await setup(); + deviceTcp.emulatorPortsListening.add("adb"); + const socket = await attach(worker); + socket.receive(withEmulator ? emulatorHello() : hello({ devicePort: 9229, kind: "forward" })); + await vi.waitFor(() => expect(frames(socket)).toHaveLength(1)); + return { deviceTcp, socket }; + } + + it("carries a caller's emulator connection to the emulator's adb port, bytes both ways, resets one the port refuses, and closes the socket 4003 on an OPEN when HELLO named none", async () => { + const accepted = await ready(); + expect(frames(accepted.socket)).toEqual([ + { body: '{"tunnels":[{"kind":"emulator","emulatorPort":"adb"}]}', conn: 0, tag: 0x81 }, + ]); + accepted.socket.receive(open(2)); + await vi.waitFor(() => expect(accepted.deviceTcp.emulatorConnections).toHaveLength(1)); + const { name, peer } = accepted.deviceTcp.emulatorConnections[0] as { + name: string; + peer: FakeTcpPeer; + }; + expect(name).toBe("adb"); + accepted.socket.receive(frame(0x03, 2, "from the caller")); + await vi.waitFor(() => expect(peer.writtenText()).toBe("from the caller")); + peer.send("from adb"); + await vi.waitFor(() => + expect(frames(accepted.socket).at(-1)).toEqual({ body: "from adb", conn: 2, tag: 0x03 }), + ); + expect(accepted.deviceTcp.connections).toHaveLength(0); + + const refusing = await ready(); + refusing.deviceTcp.emulatorPortsListening.delete("adb"); + refusing.socket.receive(open(2)); + await vi.waitFor(() => + expect(frames(refusing.socket).at(-1)).toEqual({ + body: '{"reason":"closed"}', + conn: 2, + tag: 0x06, + }), + ); + expect(refusing.socket.closed).toBeUndefined(); + + const unnamed = await ready(false); + unnamed.socket.receive(open(2)); + await vi.waitFor(() => + expect(unnamed.socket.closed).toEqual({ + code: 4003, + reason: "OPEN must name a forward tunnel of this socket", + }), + ); + expect(unnamed.deviceTcp.emulatorConnections).toHaveLength(0); + + const other = await ready(); + other.socket.receive(open(2, { kind: "emulator", emulatorPort: "console" })); + await vi.waitFor(() => + expect(other.socket.closed).toEqual({ + code: 4003, + reason: "OPEN must name a forward tunnel of this socket", + }), + ); + expect(other.deviceTcp.emulatorConnections).toHaveLength(0); + }); +}); + +describe("a worker's HELLO with an emulator tunnel", () => { + const emulator = { kind: "emulator", emulatorPort: "adb" }; + const send = (...tunnels: unknown[]): Uint8Array => frame(0x01, 0, JSON.stringify({ tunnels })); + + it("checks a holder's emulator entry and answers it: READY echoing it, counted toward 8, BAD_REQUEST for another name, TUNNEL_UNSUPPORTED off an emulator", async () => { + const forward = (devicePort: number) => ({ kind: "forward", devicePort }); + const { worker } = await setup(); + const first = await attach(worker); + first.receive(send(emulator, forward(9229), forward(9230))); + await vi.waitFor(() => + expect(frames(first)).toEqual([ + { + body: JSON.stringify({ tunnels: [emulator, forward(9229), forward(9230)] }), + conn: 0, + tag: 0x81, + }, + ]), + ); + const second = await attach(worker); + second.receive(send(...[9231, 9232, 9233, 9234, 9235].map(forward))); + await vi.waitFor(() => expect(frames(second).map((sent) => sent.tag)).toEqual([0x81])); + const third = await attach(worker); + third.receive(send(emulator)); + await vi.waitFor(() => + expect(frames(third)).toEqual([ + { + body: JSON.stringify({ + code: "TUNNEL_LIMIT", + message: "the lease already has 8 tunnels; at most 8", + }), + conn: 0, + tag: 0x82, + }, + ]), + ); + + const badName = "HELLO emulator tunnels must name the port adb"; + const refusals = [ + { + code: "BAD_REQUEST", + message: badName, + tunnel: { kind: "emulator", emulatorPort: "console" }, + }, + { code: "BAD_REQUEST", message: badName, tunnel: { kind: "emulator" } }, + { code: "BAD_REQUEST", message: badName, tunnel: { kind: "emulator", emulatorPort: 5555 } }, + { + code: "TUNNEL_UNSUPPORTED", + message: + "emulator adb: the lease is on an Android phone; emulator ports need an Android emulator", + physical: true, + platform: "android", + tunnel: emulator, + }, + { + code: "TUNNEL_UNSUPPORTED", + message: + "emulator adb: the lease is on an iOS simulator; emulator ports need an Android emulator", + platform: "ios", + tunnel: emulator, + }, + { + code: "TUNNEL_UNSUPPORTED", + message: "emulator adb: the lease is on an iPhone; emulator ports need an Android emulator", + physical: true, + platform: "ios", + tunnel: emulator, + }, + ]; + for (const refused of refusals) { + const { worker: other } = await setup(refused.platform, refused.physical); + const socket = await attach(other); + socket.receive(send(refused.tunnel)); + await vi.waitFor(() => + expect(socket.closed).toEqual({ code: 4003, reason: refused.message }), + ); + expect(frames(socket)).toEqual([ + { + body: JSON.stringify({ code: refused.code, message: refused.message }), + conn: 0, + tag: 0x82, + }, + ]); + } + }); +}); diff --git a/src/tunnel/worker-forward-tunnels.test.ts b/src/tunnel/worker-forward-tunnels.test.ts new file mode 100644 index 000000000..1918d1fdb --- /dev/null +++ b/src/tunnel/worker-forward-tunnels.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from "vitest"; +import { DeviceTcpError } from "../core/index.js"; +import { attach, ending, frame, frames, hello, setup } from "./test-worker.js"; + +describe("a worker's tunnel socket with forward tunnels", () => { + const tunnels = [ + { devicePort: 9229, kind: "forward" }, + { devicePort: 5173, kind: "reverse" }, + ]; + const open = (conn: number, devicePort: number, kind = "forward"): Uint8Array => + frame(0x02, conn, JSON.stringify({ devicePort, kind })); + + async function ready() { + const { deviceTcp, worker } = await setup(); + deviceTcp.listening.add(9229); + const socket = await attach(worker); + socket.receive(hello(...tunnels)); + await vi.waitFor(() => expect(frames(socket)).toHaveLength(1)); + expect(deviceTcp.reversed).toEqual([5173]); + return { deviceTcp, socket, worker }; + } + + it("refuses a caller OPEN that is not an even rising id for a forward tunnel of the socket with BAD_REQUEST and close 4003, and counts caller and device connections together toward 256", async () => { + const accepted = await ready(); + accepted.socket.receive(open(2, 9229)); + await vi.waitFor(() => expect(accepted.deviceTcp.connections).toHaveLength(1)); + expect(accepted.socket.closed).toBeUndefined(); + + const cases = [ + { frames: [open(2, 9230)], message: "OPEN must name a forward tunnel of this socket" }, + { frames: [open(2, 5173)], message: "OPEN must name a forward tunnel of this socket" }, + { + frames: [open(2, 9229, "reverse")], + message: "OPEN must name a forward tunnel of this socket", + }, + { frames: [open(1, 9229)], message: "OPEN needs an even connection id above the last" }, + { + frames: [open(2, 9229), open(2, 9229)], + message: "OPEN needs an even connection id above the last", + }, + { + frames: [open(4, 9229), open(2, 9229)], + message: "OPEN needs an even connection id above the last", + }, + { + frames: [frame(0x02, 2, "{nope")], + message: "OPEN must name a forward tunnel of this socket", + }, + ]; + for (const bad of cases) { + const { socket } = await ready(); + for (const sent of bad.frames) socket.receive(sent); + await vi.waitFor(() => expect(socket.closed).toEqual({ code: 4003, reason: bad.message })); + expect(frames(socket).at(-1)).toEqual({ + body: JSON.stringify({ code: "BAD_REQUEST", message: bad.message }), + conn: 0, + tag: 0x82, + }); + } + + const { deviceTcp, socket } = await ready(); + for (let conn = 2; conn <= 512; conn += 2) { + if (conn <= 256) deviceTcp.openFromDevice(5173); + else socket.receive(open(conn, 9229)); + } + await vi.waitFor(() => expect(deviceTcp.connections).toHaveLength(128)); + socket.receive(open(514, 9229)); + await vi.waitFor(() => + expect(frames(socket).at(-1)).toEqual({ + body: '{"reason":"too many connections"}', + conn: 514, + tag: 0x06, + }), + ); + expect(deviceTcp.connections).toHaveLength(128); + const past = deviceTcp.openFromDevice(5173); + await vi.waitFor(() => expect(past.closed).toBe(true)); + expect(socket.closed).toBeUndefined(); + + const failing = await ready(); + failing.deviceTcp.listening.delete(9229); + failing.deviceTcp.connectFailures.set(9231, new DeviceTcpError("TUNNEL_FAILED", "timeout")); + failing.socket.receive(open(2, 9229)); + await vi.waitFor(() => + expect(frames(failing.socket).at(-1)).toEqual({ + body: '{"reason":"closed"}', + conn: 2, + tag: 0x06, + }), + ); + expect(failing.socket.closed).toBeUndefined(); + + const silent = await ready(); + silent.deviceTcp.connectFailures.set(9229, new DeviceTcpError("TUNNEL_FAILED", "timeout")); + silent.socket.receive(open(2, 9229)); + await vi.waitFor(() => + expect(frames(silent.socket).at(-1)).toEqual({ + body: '{"reason":"timeout"}', + conn: 2, + tag: 0x06, + }), + ); + + const ended = await ready(); + await ended.worker.hooks?.beforeDeviceReclaim?.( + ending("expired"), + new AbortController().signal, + ); + ended.socket.receive(open(2, 9229)); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(ended.deviceTcp.connections).toHaveLength(0); + }); +}); diff --git a/src/tunnel/worker.test.ts b/src/tunnel/worker.test.ts index e46bfdc1d..26c7189a3 100644 --- a/src/tunnel/worker.test.ts +++ b/src/tunnel/worker.test.ts @@ -1,31 +1,9 @@ import { describe, expect, it, vi } from "vitest"; - -import type { DeviceRecord, LeaseRecord } from "../core/index.js"; import { DeviceTcpError } from "../core/index.js"; import { FakeDeviceTcp } from "../core/testing.js"; import type { WorkerModule } from "../modules/index.js"; -import { fakeWorkerContext } from "../modules/testing.js"; -import { type FakeClock, type FakeTcpPeer, MemoryLeaseSocket } from "../ports/index.js"; -import { tunnelModule } from "./index.js"; - -const IDENTITY = { principal: "tok_holder", role: "agent" } as const; - -function frame(tag: number, conn: number, body = ""): Uint8Array { - const bytes = new TextEncoder().encode(body); - const out = new Uint8Array(5 + bytes.length); - out[0] = tag; - new DataView(out.buffer).setUint32(1, conn); - out.set(bytes, 5); - return out; -} - -function frames(socket: MemoryLeaseSocket): { tag: number; conn: number; body: string }[] { - return socket.sent.map((sent) => ({ - body: new TextDecoder().decode(sent.subarray(5)), - conn: new DataView(sent.buffer, sent.byteOffset).getUint32(1), - tag: sent[0] ?? -1, - })); -} +import { type FakeClock, MemoryLeaseSocket } from "../ports/index.js"; +import { IDENTITY, attach, ending, frame, frames, hello, setup } from "./test-worker.js"; function sentData(socket: MemoryLeaseSocket, conn: number): number { return socket.sent @@ -50,67 +28,6 @@ function oversizeData(conn: number): Uint8Array { return bytes; } -function ending(reason: "device-lost" | "expired" | "explicit" | "killed") { - return { - deviceId: "dev_1", - grantedAt: 1_000, - leaseId: "L-1", - ownerId: "tok_holder", - platform: "android", - reason, - } as const; -} - -function hello(...tunnels: { kind: string; devicePort: number }[]): Uint8Array { - return frame(0x01, 0, JSON.stringify({ tunnels })); -} - -async function setup( - platform = "android", - physical = false, -): Promise<{ - readonly worker: WorkerModule; - readonly deviceTcp: FakeDeviceTcp; - readonly clock: FakeClock; -}> { - const { clock, ctx } = await fakeWorkerContext("tunnel", {}); - const deviceTcp = new FakeDeviceTcp(); - const registry = { - snapshot: { - devices: [{ id: "dev_1", physical, spec: { platform } } as unknown as DeviceRecord], - leases: [ - { deviceId: "dev_1", id: "L-1" } as unknown as LeaseRecord, - { deviceId: "dev_1", id: "L-2" } as unknown as LeaseRecord, - ], - }, - }; - const loaded = await tunnelModule.loadWorker(); - const worker = loaded.createWorker({ - ...ctx, - deviceTcp: { - connect: (_platform, device, port, signal) => deviceTcp.connect(device, port, signal), - supports: (named) => named === "android", - supportsEmulatorPort: (named, device) => named === "android" && !device.physical, - connectEmulatorPort: (_platform, device, name, signal) => - deviceTcp.connectEmulatorPort(device, name, signal), - reverse: (_platform, device, port, signal) => deviceTcp.reverse(device, port, signal), - }, - registry, - }); - return { clock, deviceTcp, worker }; -} - -async function attach(worker: WorkerModule, leaseId = "L-1"): Promise { - const opened = await worker.socket?.open(new Request("http://simlock/tunnel"), IDENTITY, { - deviceId: "dev_1", - id: leaseId, - }); - expect(opened).toBeTypeOf("function"); - const socket = new MemoryLeaseSocket(); - if (typeof opened === "function") opened(socket); - return socket; -} - async function carriesOneTunnel(): Promise { const { deviceTcp, worker } = await setup(); const socket = await attach(worker); @@ -844,268 +761,3 @@ describe("a worker's tunnel socket", () => { expect(clock.pendingTimerCount).toBe(idleTimers); }); }); - -describe("a worker's tunnel socket with forward tunnels", () => { - const tunnels = [ - { devicePort: 9229, kind: "forward" }, - { devicePort: 5173, kind: "reverse" }, - ]; - const open = (conn: number, devicePort: number, kind = "forward"): Uint8Array => - frame(0x02, conn, JSON.stringify({ devicePort, kind })); - - async function ready() { - const { deviceTcp, worker } = await setup(); - deviceTcp.listening.add(9229); - const socket = await attach(worker); - socket.receive(hello(...tunnels)); - await vi.waitFor(() => expect(frames(socket)).toHaveLength(1)); - expect(deviceTcp.reversed).toEqual([5173]); - return { deviceTcp, socket, worker }; - } - - it("refuses a caller OPEN that is not an even rising id for a forward tunnel of the socket with BAD_REQUEST and close 4003, and counts caller and device connections together toward 256", async () => { - const accepted = await ready(); - accepted.socket.receive(open(2, 9229)); - await vi.waitFor(() => expect(accepted.deviceTcp.connections).toHaveLength(1)); - expect(accepted.socket.closed).toBeUndefined(); - - const cases = [ - { frames: [open(2, 9230)], message: "OPEN must name a forward tunnel of this socket" }, - { frames: [open(2, 5173)], message: "OPEN must name a forward tunnel of this socket" }, - { - frames: [open(2, 9229, "reverse")], - message: "OPEN must name a forward tunnel of this socket", - }, - { frames: [open(1, 9229)], message: "OPEN needs an even connection id above the last" }, - { - frames: [open(2, 9229), open(2, 9229)], - message: "OPEN needs an even connection id above the last", - }, - { - frames: [open(4, 9229), open(2, 9229)], - message: "OPEN needs an even connection id above the last", - }, - { - frames: [frame(0x02, 2, "{nope")], - message: "OPEN must name a forward tunnel of this socket", - }, - ]; - for (const bad of cases) { - const { socket } = await ready(); - for (const sent of bad.frames) socket.receive(sent); - await vi.waitFor(() => expect(socket.closed).toEqual({ code: 4003, reason: bad.message })); - expect(frames(socket).at(-1)).toEqual({ - body: JSON.stringify({ code: "BAD_REQUEST", message: bad.message }), - conn: 0, - tag: 0x82, - }); - } - - const { deviceTcp, socket } = await ready(); - for (let conn = 2; conn <= 512; conn += 2) { - if (conn <= 256) deviceTcp.openFromDevice(5173); - else socket.receive(open(conn, 9229)); - } - await vi.waitFor(() => expect(deviceTcp.connections).toHaveLength(128)); - socket.receive(open(514, 9229)); - await vi.waitFor(() => - expect(frames(socket).at(-1)).toEqual({ - body: '{"reason":"too many connections"}', - conn: 514, - tag: 0x06, - }), - ); - expect(deviceTcp.connections).toHaveLength(128); - const past = deviceTcp.openFromDevice(5173); - await vi.waitFor(() => expect(past.closed).toBe(true)); - expect(socket.closed).toBeUndefined(); - - const failing = await ready(); - failing.deviceTcp.listening.delete(9229); - failing.deviceTcp.connectFailures.set(9231, new DeviceTcpError("TUNNEL_FAILED", "timeout")); - failing.socket.receive(open(2, 9229)); - await vi.waitFor(() => - expect(frames(failing.socket).at(-1)).toEqual({ - body: '{"reason":"closed"}', - conn: 2, - tag: 0x06, - }), - ); - expect(failing.socket.closed).toBeUndefined(); - - const silent = await ready(); - silent.deviceTcp.connectFailures.set(9229, new DeviceTcpError("TUNNEL_FAILED", "timeout")); - silent.socket.receive(open(2, 9229)); - await vi.waitFor(() => - expect(frames(silent.socket).at(-1)).toEqual({ - body: '{"reason":"timeout"}', - conn: 2, - tag: 0x06, - }), - ); - - const ended = await ready(); - await ended.worker.hooks?.beforeDeviceReclaim?.( - ending("expired"), - new AbortController().signal, - ); - ended.socket.receive(open(2, 9229)); - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(ended.deviceTcp.connections).toHaveLength(0); - }); -}); - -describe("a worker's tunnel socket with an emulator tunnel", () => { - const emulatorHello = (): Uint8Array => - frame(0x01, 0, JSON.stringify({ tunnels: [{ kind: "emulator", emulatorPort: "adb" }] })); - const open = (conn: number, body: unknown = { kind: "emulator", emulatorPort: "adb" }) => - frame(0x02, conn, JSON.stringify(body)); - - async function ready(withEmulator = true) { - const { deviceTcp, worker } = await setup(); - deviceTcp.emulatorPortsListening.add("adb"); - const socket = await attach(worker); - socket.receive(withEmulator ? emulatorHello() : hello({ devicePort: 9229, kind: "forward" })); - await vi.waitFor(() => expect(frames(socket)).toHaveLength(1)); - return { deviceTcp, socket }; - } - - it("carries a caller's emulator connection to the emulator's adb port, bytes both ways, resets one the port refuses, and closes the socket 4003 on an OPEN when HELLO named none", async () => { - const accepted = await ready(); - expect(frames(accepted.socket)).toEqual([ - { body: '{"tunnels":[{"kind":"emulator","emulatorPort":"adb"}]}', conn: 0, tag: 0x81 }, - ]); - accepted.socket.receive(open(2)); - await vi.waitFor(() => expect(accepted.deviceTcp.emulatorConnections).toHaveLength(1)); - const { name, peer } = accepted.deviceTcp.emulatorConnections[0] as { - name: string; - peer: FakeTcpPeer; - }; - expect(name).toBe("adb"); - accepted.socket.receive(frame(0x03, 2, "from the caller")); - await vi.waitFor(() => expect(peer.writtenText()).toBe("from the caller")); - peer.send("from adb"); - await vi.waitFor(() => - expect(frames(accepted.socket).at(-1)).toEqual({ body: "from adb", conn: 2, tag: 0x03 }), - ); - expect(accepted.deviceTcp.connections).toHaveLength(0); - - const refusing = await ready(); - refusing.deviceTcp.emulatorPortsListening.delete("adb"); - refusing.socket.receive(open(2)); - await vi.waitFor(() => - expect(frames(refusing.socket).at(-1)).toEqual({ - body: '{"reason":"closed"}', - conn: 2, - tag: 0x06, - }), - ); - expect(refusing.socket.closed).toBeUndefined(); - - const unnamed = await ready(false); - unnamed.socket.receive(open(2)); - await vi.waitFor(() => - expect(unnamed.socket.closed).toEqual({ - code: 4003, - reason: "OPEN must name a forward tunnel of this socket", - }), - ); - expect(unnamed.deviceTcp.emulatorConnections).toHaveLength(0); - - const other = await ready(); - other.socket.receive(open(2, { kind: "emulator", emulatorPort: "console" })); - await vi.waitFor(() => - expect(other.socket.closed).toEqual({ - code: 4003, - reason: "OPEN must name a forward tunnel of this socket", - }), - ); - expect(other.deviceTcp.emulatorConnections).toHaveLength(0); - }); -}); - -describe("a worker's HELLO with an emulator tunnel", () => { - const emulator = { kind: "emulator", emulatorPort: "adb" }; - const send = (...tunnels: unknown[]): Uint8Array => frame(0x01, 0, JSON.stringify({ tunnels })); - - it("checks a holder's emulator entry and answers it: READY echoing it, counted toward 8, BAD_REQUEST for another name, TUNNEL_UNSUPPORTED off an emulator", async () => { - const forward = (devicePort: number) => ({ kind: "forward", devicePort }); - const { worker } = await setup(); - const first = await attach(worker); - first.receive(send(emulator, forward(9229), forward(9230))); - await vi.waitFor(() => - expect(frames(first)).toEqual([ - { - body: JSON.stringify({ tunnels: [emulator, forward(9229), forward(9230)] }), - conn: 0, - tag: 0x81, - }, - ]), - ); - const second = await attach(worker); - second.receive(send(...[9231, 9232, 9233, 9234, 9235].map(forward))); - await vi.waitFor(() => expect(frames(second).map((sent) => sent.tag)).toEqual([0x81])); - const third = await attach(worker); - third.receive(send(emulator)); - await vi.waitFor(() => - expect(frames(third)).toEqual([ - { - body: JSON.stringify({ - code: "TUNNEL_LIMIT", - message: "the lease already has 8 tunnels; at most 8", - }), - conn: 0, - tag: 0x82, - }, - ]), - ); - - const badName = "HELLO emulator tunnels must name the port adb"; - const refusals = [ - { - code: "BAD_REQUEST", - message: badName, - tunnel: { kind: "emulator", emulatorPort: "console" }, - }, - { code: "BAD_REQUEST", message: badName, tunnel: { kind: "emulator" } }, - { code: "BAD_REQUEST", message: badName, tunnel: { kind: "emulator", emulatorPort: 5555 } }, - { - code: "TUNNEL_UNSUPPORTED", - message: - "emulator adb: the lease is on an Android phone; emulator ports need an Android emulator", - physical: true, - platform: "android", - tunnel: emulator, - }, - { - code: "TUNNEL_UNSUPPORTED", - message: - "emulator adb: the lease is on an iOS simulator; emulator ports need an Android emulator", - platform: "ios", - tunnel: emulator, - }, - { - code: "TUNNEL_UNSUPPORTED", - message: "emulator adb: the lease is on an iPhone; emulator ports need an Android emulator", - physical: true, - platform: "ios", - tunnel: emulator, - }, - ]; - for (const refused of refusals) { - const { worker: other } = await setup(refused.platform, refused.physical); - const socket = await attach(other); - socket.receive(send(refused.tunnel)); - await vi.waitFor(() => - expect(socket.closed).toEqual({ code: 4003, reason: refused.message }), - ); - expect(frames(socket)).toEqual([ - { - body: JSON.stringify({ code: refused.code, message: refused.message }), - conn: 0, - tag: 0x82, - }, - ]); - } - }); -}); From dc3b581dc893b139fbd3ea79059e7d9d32fee69d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:35:39 +0100 Subject: [PATCH 73/83] test(e2e): split physical-enrollment.test.ts by topic --- e2e/physical-enrollment-auto-enroll.test.ts | 446 ++++++++++++ e2e/physical-enrollment-enrolled.test.ts | 63 ++ e2e/physical-enrollment-transports.test.ts | 125 ++++ e2e/physical-enrollment.test.ts | 758 +------------------- e2e/test-physical-enrollment.ts | 158 ++++ 5 files changed, 809 insertions(+), 741 deletions(-) create mode 100644 e2e/physical-enrollment-auto-enroll.test.ts create mode 100644 e2e/physical-enrollment-enrolled.test.ts create mode 100644 e2e/physical-enrollment-transports.test.ts create mode 100644 e2e/test-physical-enrollment.ts diff --git a/e2e/physical-enrollment-auto-enroll.test.ts b/e2e/physical-enrollment-auto-enroll.test.ts new file mode 100644 index 000000000..c139013e3 --- /dev/null +++ b/e2e/physical-enrollment-auto-enroll.test.ts @@ -0,0 +1,446 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import type { FakeDriverRefusalReason, ScriptedPhysicalDevice } from "./fake-driver/types.js"; +import { type TestEnv, waitFor, withDaemon } from "./helpers/index.js"; +import { observeFor } from "./helpers/physical-state.js"; +import { readEnrolledApps } from "./helpers/state-db.js"; +import { + Added, + IPHONE, + PIXEL, + UDID, + add, + daemonOptions, + inspectCalls, + named, + refused, + rows, +} from "./test-physical-enrollment.js"; + +const AUTO_ENROLL = { devices: { autoEnroll: true }, health: { probeIntervalMs: 200 } }; + +const IPHONE_TWO: ScriptedPhysicalDevice = { + apps: ["com.example.second"], + class: "phone", + ids: ["00008110-00FFEEDDCCBB"], + model: "iPhone 15 Pro", + osVersion: "18.4", +}; + +async function autoDaemon( + options: { + readonly ios?: Record; + readonly android?: Record; + readonly config?: Record; + readonly platforms?: readonly ("ios" | "android")[]; + } = {}, +): Promise { + const base = daemonOptions(options); + return withDaemon({ + ...base, + configOverrides: { ...AUTO_ENROLL, ...options.config }, + }); +} + +async function plug( + env: TestEnv, + platform: "ios" | "android", + devices: readonly ScriptedPhysicalDevice[], +): Promise { + await env.driverScript.merge({ [platform]: { physicalDevices: devices } }); +} + +async function autoEnrollLog(env: TestEnv): Promise { + const text = await readFile(env.logPath, "utf8").catch(() => ""); + return text + .split("\n") + .filter((line) => line.startsWith("{")) + .map((line) => (JSON.parse(line) as { message: string }).message) + .filter((message) => message.startsWith("auto-enroll:")); +} + +describe("devices.autoEnroll", () => { + it("enrols a phone plugged in, an iPhone, an Android phone and a phone attached before the daemon starts, each with one device.enrolled initiator auto and its installed apps as kept apps", async () => { + const env = await autoDaemon({ ios: { physicalDevices: [IPHONE] } }); + + await plug(env, "ios", [IPHONE, IPHONE_TWO]); + await plug(env, "android", [PIXEL]); + + await waitFor(async () => (await named(env, "device.enrolled")).length === 3, { + label: "the three phones are enrolled", + }); + const enrolled = (await named(env, "device.enrolled")).map( + (entry) => entry.payload as Added & { readonly initiator: string }, + ); + expect( + enrolled.map((payload) => [payload.driverDeviceId, payload.initiator, payload.appCount]), + ).toEqual( + expect.arrayContaining([ + [UDID, "auto", 12], + ["00008110-00FFEEDDCCBB", "auto", 1], + ["R58M123ABC", "auto", 1], + ]), + ); + expect(enrolled.every((payload) => !payload.reenrolled)).toBe(true); + expect( + readEnrolledApps(env) + .map((apps) => apps.length) + .sort(), + ).toEqual([1, 1, 12]); + expect((await rows(env)).map((row) => row.state)).toEqual(["ready", "ready", "ready"]); + }); + + it("retries a phone that cannot be enrolled and logs one auto-enroll line each time its reason changes, word for word, then enrols it once unlocked", async () => { + const env = await autoDaemon(); + const [a, b, c] = [IPHONE, { ...IPHONE, ids: ["PHONE-B"] }, IPHONE_TWO]; + const line = (device: ScriptedPhysicalDevice, reason: FakeDriverRefusalReason) => + `auto-enroll: ios ${device.ids[0]} not enrolled: scripted ${reason} for ${device.ids[0]}`; + + await plug(env, "ios", [ + refused(a, "locked"), + refused(b, "untrusted"), + refused(c, "untrusted"), + ]); + await waitFor(async () => (await autoEnrollLog(env)).length === 3, { label: "three lines" }); + await observeFor(700, async () => { + expect(await autoEnrollLog(env), "three runs at one reason write one line").toHaveLength(3); + }); + await plug(env, "ios", [refused(a, "locked"), refused(b, "untrusted"), refused(c, "locked")]); + await waitFor(async () => (await autoEnrollLog(env)).length === 4, { label: "locked line" }); + await plug(env, "ios", [ + refused(a, "locked"), + refused(b, "untrusted"), + refused(c, "untrusted"), + ]); + await waitFor(async () => (await autoEnrollLog(env)).length === 5, { + label: "untrusted again", + }); + await observeFor(700, async () => { + expect(await autoEnrollLog(env)).toHaveLength(5); + }); + await plug(env, "ios", [a, refused(b, "untrusted"), refused(c, "untrusted")]); + await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { + label: "the unlocked phone is enrolled", + }); + + const lines = await autoEnrollLog(env); + expect(lines).toEqual( + expect.arrayContaining([line(a, "locked"), line(b, "untrusted"), line(c, "untrusted")]), + ); + expect(lines.filter((entry) => entry.includes(String(c.ids[0])))).toEqual([ + line(c, "untrusted"), + line(c, "locked"), + line(c, "untrusted"), + ]); + expect(lines.filter((entry) => entry.includes(UDID))).toEqual([line(a, "locked")]); + expect((await named(env, "device.enrolled")).map((entry) => entry.payload)).toEqual([ + expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), + ]); + }); + + it("keeps a phone out that was removed while plugged in until it has been missing two runs, while an operator's device add enrols it again", async () => { + const phone = (id: string): ScriptedPhysicalDevice => ({ ...IPHONE, ids: [id] }); + const removed = phone("PHONE-1"); + const brief = phone("PHONE-2"); + const long = phone("PHONE-3"); + const added = phone("PHONE-4"); + const iphones = [removed, brief, long, added]; + const pixel: ScriptedPhysicalDevice = { ...PIXEL, present: true }; + const env = await autoDaemon({ + android: { + failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, + physicalDevices: [pixel], + }, + config: { + health: { probeIntervalMs: 500 }, + warmPool: { quarantine: { maxRetries: 1, retryBackoffMs: 300, retryBackoffMultiplier: 1 } }, + }, + ios: { physicalDevices: iphones }, + }); + const enrolledOf = async (id: string) => + (await named(env, "device.enrolled")) + .map((entry) => entry.payload as Added & { readonly initiator: string }) + .filter((payload) => payload.driverDeviceId === id); + const runs = async (count: number) => { + const lists = async () => + (await env.driverLog.calls()).filter( + (call) => call.operation === "listAttachedPhysical" && call.platform === "ios", + ).length; + const before = await lists(); + await waitFor(async () => (await lists()) >= before + count, { interval: 20 }); + }; + await waitFor(async () => (await named(env, "device.enrolled")).length === 5, { + label: "the five phones are enrolled", + }); + const remove = async (id: string) => { + const result = await env.cli(["device", "remove", id]); + expect(result.code, result.stderr).toBe(0); + }; + + await remove("PHONE-1"); + const grant = await env.cli([ + "lease", + "--platform", + "android", + "--physical", + "--agent-id", + "a", + "--detach", + ]); + expect(grant.code, grant.stderr).toBe(0); + const released = await env.cli(["release", (grant.json as { lease: { id: string } }).lease.id]); + expect(released.code, released.stderr).toBe(0); + await waitFor(async () => (await named(env, "device.quarantine-abandoned")).length === 1, { + label: "the quarantine gives up and unenrols the Android phone", + }); + await remove("PHONE-2"); + await remove("PHONE-3"); + await plug(env, "ios", [ + removed, + { ...brief, present: false }, + { ...long, present: false }, + added, + ]); + await runs(1); + await plug(env, "ios", [removed, brief, { ...long, present: false }, added]); + await runs(1); + await plug(env, "ios", iphones); + await waitFor(async () => (await enrolledOf("PHONE-3")).length === 2, { + label: "the phone missing two runs is enrolled again", + }); + await remove("PHONE-4"); + const again = await env.cli(["device", "add", "--platform", "ios", "PHONE-4"]); + expect(again.code, again.stderr).toBe(0); + await runs(3); + + expect( + (await enrolledOf("PHONE-1")).map((payload) => payload.initiator), + "removed, plugged in", + ).toEqual(["auto"]); + expect( + (await enrolledOf(pixel.ids[0] ?? "")).map((payload) => payload.initiator), + "unenrolled after quarantine", + ).toEqual(["auto"]); + expect( + (await enrolledOf("PHONE-2")).map((payload) => payload.initiator), + "missing one run", + ).toEqual(["auto"]); + expect( + (await enrolledOf("PHONE-3")).map((payload) => payload.initiator), + "missing two runs", + ).toEqual(["auto", "auto"]); + expect( + (await enrolledOf("PHONE-4")).map((payload) => payload.initiator), + "device add while held out", + ).toEqual(["auto", "operator"]); + + const slow = await autoDaemon({ + config: { health: { probeIntervalMs: 5000 } }, + ios: { physicalDevices: [IPHONE, refused({ ...IPHONE, ids: ["PHONE-Y"] }, "locked")] }, + }); + const slowEnrolled = async (id: string) => + (await named(slow, "device.enrolled")) + .map((entry) => entry.payload as Added & { readonly initiator: string }) + .filter((payload) => payload.driverDeviceId === id); + await waitFor(async () => (await slowEnrolled(UDID)).length === 1, { + label: "the first run enrols the unlocked phone", + timeout: 30_000, + }); + const gone = await slow.cli(["device", "remove", UDID]); + expect(gone.code, gone.stderr).toBe(0); + await plug(slow, "ios", [IPHONE, { ...IPHONE, ids: ["PHONE-Y"] }]); + await add(slow, "ios", "PHONE-Y"); + const goneToo = await slow.cli(["device", "remove", "PHONE-Y"]); + expect(goneToo.code, goneToo.stderr).toBe(0); + const listed = async () => + (await slow.driverLog.calls()).filter((call) => call.operation === "listAttachedPhysical") + .length; + const seen = await listed(); + await waitFor(async () => (await listed()) >= seen + 4, { interval: 50, timeout: 30_000 }); + + expect( + (await slowEnrolled(UDID)).map((payload) => payload.initiator), + "removed before the next run", + ).toEqual(["auto"]); + expect( + (await slowEnrolled("PHONE-Y")).map((payload) => payload.initiator), + "added and removed between two runs", + ).toEqual(["operator"]); + }); + + it("forgets which phones are held out when the daemon restarts, so a removed phone still plugged in is enrolled on the first run", async () => { + const env = await autoDaemon({ ios: { physicalDevices: [IPHONE] } }); + await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { + label: "the phone is enrolled", + }); + const removed = await env.cli(["device", "remove", UDID]); + expect(removed.code, removed.stderr).toBe(0); + + await env.restartDaemon(); + + await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { + label: "the restarted daemon enrols the removed phone again", + }); + expect((await named(env, "device.enrolled")).map((entry) => entry.payload)).toEqual([ + expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), + expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), + ]); + }); + + it("skips a platform that cannot be listed for that run, still enrols the other platform, logs one line per error message, and logs nothing for a platform with no driver", async () => { + const down = { listAttachedPhysical: { message: "adb is down", type: "generic" } } as const; + const env = await autoDaemon({ + android: { + failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, + physicalDevices: [PIXEL], + }, + config: { warmPool: { quarantine: { retryBackoffMs: 600_000 } } }, + ios: { physicalDevices: [IPHONE] }, + }); + await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { + label: "both phones are enrolled", + }); + const removed = await env.cli(["device", "remove", "R58M123ABC"]); + expect(removed.code, removed.stderr).toBe(0); + const androidLists = async () => + (await env.driverLog.calls()).filter( + (call) => call.operation === "listAttachedPhysical" && call.platform === "android", + ).length; + const runs = async (count: number) => { + const before = await androidLists(); + await waitFor(async () => (await androidLists()) >= before + count, { interval: 20 }); + }; + const couldNotList = async () => + (await autoEnrollLog(env)).filter((line) => line.includes("could not list")); + + await env.driverScript.merge({ android: { failures: down } }); + await runs(4); + await plug(env, "ios", [IPHONE, IPHONE_TWO]); + await waitFor(async () => (await named(env, "device.enrolled")).length === 3, { + label: "the iPhone plugged in while Android cannot be listed is enrolled", + }); + + expect(await couldNotList()).toEqual([ + "auto-enroll: could not list android phones: adb is down", + ]); + + await env.driverScript.merge({ android: { failures: {} } }); + await runs(4); + expect( + (await named(env, "device.enrolled")).filter( + (e) => (e.payload as Added).platform === "android", + ), + "held out across the failing runs", + ).toHaveLength(1); + await env.driverScript.merge({ android: { failures: down } }); + await runs(2); + expect(await couldNotList(), "a list that succeeds resets the message").toHaveLength(2); + + const iosOnly = await autoDaemon({ ios: { physicalDevices: [IPHONE] }, platforms: ["ios"] }); + await waitFor(async () => (await named(iosOnly, "device.enrolled")).length === 1, { + label: "the iPhone is enrolled on a worker with no Android driver", + }); + expect(await autoEnrollLog(iosOnly)).toEqual([]); + }); + + it("grants a waiting physical request the auto-enrolled phone within one run, with no second request, and still fails a model with no record at once", async () => { + const second: ScriptedPhysicalDevice = { ...PIXEL, ids: ["R58M456DEF"] }; + const env = await autoDaemon({ android: { physicalDevices: [PIXEL] } }); + await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { + label: "the first Pixel is enrolled", + }); + const ask = (agent: string, model: string) => [ + "lease", + "--platform", + "android", + "--physical", + "--device", + model, + "--agent-id", + agent, + ]; + const first = await env.cli([...ask("agent-a", "Pixel 9"), "--detach"], { timeout: 30_000 }); + expect(first.code, first.stderr).toBe(0); + const waiter = env.cliBackground(ask("agent-b", "Pixel 9")); + await waitFor(async () => (await named(env, "lease.queued")).length === 1, { + label: "the second request is queued", + }); + + await plug(env, "android", [PIXEL, second]); + + const grant = JSON.parse(await waiter.firstStdoutLine(30_000)) as { + device: { driverDeviceId: string }; + }; + expect(grant.device.driverDeviceId).toBe("R58M456DEF"); + const unknown = await env.cli(ask("agent-c", "Galaxy S99"), { timeout: 30_000 }); + expect(unknown.error?.code).toBe("UNKNOWN_MODEL"); + }); + + it("does nothing without the setting, unset or false: no device.enrolled and no inspectPhysical call after three runs", async () => { + for (const devices of [undefined, { autoEnroll: false }]) { + const env = await withDaemon({ + ...daemonOptions({ ios: { physicalDevices: [IPHONE] } }), + configOverrides: { + health: { probeIntervalMs: 200 }, + ...(devices === undefined ? {} : { devices }), + }, + }); + await observeFor(700, async () => { + expect(await named(env, "device.enrolled")).toEqual([]); + }); + expect(await inspectCalls(env)).toEqual([]); + } + }); + + it("leaves an enrolled phone alone in the states ready, absent, leased and quarantined: no device.enrolled and the kept apps unchanged", async () => { + const env = await autoDaemon({ + android: { + failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, + physicalDevices: [PIXEL], + }, + config: { warmPool: { quarantine: { retryBackoffMs: 600_000 } } }, + ios: { physicalDevices: [IPHONE] }, + }); + await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { + label: "both phones are enrolled", + }); + const keptApps = readEnrolledApps(env); + const states = async () => (await rows(env)).map((row) => row.state).sort(); + const stays = async (label: string) => { + await observeFor(700, async () => { + expect(await named(env, "device.enrolled"), label).toHaveLength(2); + }); + expect(readEnrolledApps(env), label).toEqual(keptApps); + }; + + await stays("ready"); + await plug(env, "ios", [{ ...IPHONE, present: false }]); + await waitFor(async () => (await states()).includes("absent"), { + label: "the iPhone is absent", + }); + await stays("absent"); + const leased = await env.cli([ + "lease", + "--platform", + "android", + "--physical", + "--agent-id", + "a", + "--detach", + ]); + expect(leased.code, leased.stderr).toBe(0); + expect(await states()).toContain("leased"); + await stays("leased"); + const released = await env.cli([ + "release", + (leased.json as { lease: { id: string } }).lease.id, + ]); + expect(released.code, released.stderr).toBe(0); + await waitFor(async () => (await states()).includes("quarantined"), { + label: "the Android phone is quarantined", + }); + await stays("quarantined"); + }); +}); diff --git a/e2e/physical-enrollment-enrolled.test.ts b/e2e/physical-enrollment-enrolled.test.ts new file mode 100644 index 000000000..33e160a43 --- /dev/null +++ b/e2e/physical-enrollment-enrolled.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; + +import { readEnrolledApps } from "./helpers/state-db.js"; +import { + IPHONE, + PIXEL, + Row, + UDID, + add, + leaseGrant, + rows, + startDaemon, +} from "./test-physical-enrollment.js"; + +describe("an enrolled physical device", () => { + it("is not granted to a lease --platform ios: the lease gets a simulator", async () => { + const env = await startDaemon({ ios: { physicalDevices: [IPHONE] } }); + const enrolled = await add(env, "ios", UDID); + + const grant = await env.cli(["lease", "--platform", "ios", "--agent-id", "sim", "--detach"]); + + expect(grant.code, grant.stderr).toBe(0); + const device = (grant.json as { device: { id: string; physical: boolean } }).device; + expect(device.physical).toBe(false); + expect(device.id).not.toBe(enrolled.id); + }); + + it("shows its canonical ID as driverDeviceId in status --json from an operator and from an agent session and in list --devices, and a simulator's status entry has none", async () => { + const env = await startDaemon({ ios: { physicalDevices: [IPHONE] } }); + await add(env, "ios", UDID); + const simulator = await leaseGrant(env, "ios", "sim-agent"); + + const operator = await env.cli(["status", "--json"]); + const agent = await env.cli(["--token", "not-an-admin-credential", "status", "--json"]); + + for (const status of [operator, agent]) { + expect(status.code, status.stderr).toBe(0); + const devices = (status.json as { devices: Row[] }).devices; + const physical = devices.find((device) => device.physical); + const virtual = devices.find((device) => !device.physical); + expect(physical?.driverDeviceId).toBe(UDID); + expect(virtual).toBeDefined(); + expect(virtual?.driverDeviceId).toBeUndefined(); + } + const listed = (await rows(env)).find((row) => row.physical); + expect(listed?.driverDeviceId).toBe(UDID); + expect(simulator.device.driverDeviceId).not.toBe(UDID); + }); +}); + +describe("android enrolment", () => { + it("records the apps the driver lists and shows the device with its Android OS version", async () => { + const env = await startDaemon({ android: { physicalDevices: [PIXEL] } }); + + const result = await env.cli(["device", "add", "--platform", "android", "R58M123ABC"]); + + expect(result.code, result.stderr).toBe(0); + expect(result.stdout).toBe( + "Enrolled Pixel 9 (Android 15) as R58M123ABC, 1 user app recorded.\n", + ); + expect(readEnrolledApps(env)).toEqual([["com.example.mail"]]); + }); +}); diff --git a/e2e/physical-enrollment-transports.test.ts b/e2e/physical-enrollment-transports.test.ts new file mode 100644 index 000000000..d4ef28ed8 --- /dev/null +++ b/e2e/physical-enrollment-transports.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; + +import { type TestEnv, waitFor, withDaemon, withHttpDaemon } from "./helpers/index.js"; +import { + IOS_BASE, + IPHONE, + Row, + UDID, + add, + bearer, + httpDaemon, + inspectCalls, + rows, +} from "./test-physical-enrollment.js"; + +describe("device add over HTTP", () => { + it("POST /v1/devices with an operator token enrols the device; with an agent token it is 403 FORBIDDEN and no inspectPhysical call is made; a body with a worker field or any other unknown field is 400 BAD_REQUEST", async () => { + const { base, env } = await httpDaemon({ ios: { physicalDevices: [IPHONE] } }); + const agent = await bearer(env, "agent"); + const operator = await bearer(env, "operator"); + const post = (headers: Record, body: unknown) => + fetch(`${base}/v1/devices`, { body: JSON.stringify(body), headers, method: "POST" }); + + const refused = await post(agent, { id: UDID, platform: "ios" }); + expect(refused.status).toBe(403); + expect(((await refused.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + expect(await inspectCalls(env)).toEqual([]); + + for (const extra of [{ worker: "w1" }, { nickname: "mine" }]) { + const bad = await post(operator, { id: UDID, platform: "ios", ...extra }); + expect(bad.status).toBe(400); + expect(((await bad.json()) as { error: { code: string } }).error.code).toBe("BAD_REQUEST"); + } + expect(await inspectCalls(env)).toEqual([]); + + const accepted = await post(operator, { id: UDID, platform: "ios" }); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toMatchObject({ + appCount: 12, + driverDeviceId: UDID, + reenrolled: false, + }); + expect((await rows(env)).filter((row) => row.state === "ready")).toHaveLength(1); + }); + + it("DELETE /v1/devices/ with an operator token unenrols the device; with an agent token it is 403 FORBIDDEN and the record stays ready", async () => { + const { base, env } = await httpDaemon({ ios: { physicalDevices: [IPHONE] } }); + const enrolled = await add(env, "ios", UDID); + const agent = await bearer(env, "agent"); + const operator = await bearer(env, "operator"); + const remove = (headers: Record) => + fetch(`${base}/v1/devices/${encodeURIComponent(UDID)}`, { headers, method: "DELETE" }); + + const refused = await remove(agent); + expect(refused.status).toBe(403); + expect(((await refused.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); + expect((await rows(env)).find((row) => row.id === enrolled.id)?.state).toBe("ready"); + + const accepted = await remove(operator); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toEqual({ driverDeviceId: UDID, id: enrolled.id }); + expect((await rows(env)).find((row) => row.id === enrolled.id)?.state).toBe("deleted"); + }); +}); + +describe("device add through a gateway", () => { + async function fleet(): Promise<{ gateway: TestEnv; worker: TestEnv; workerId: string }> { + const { env: gateway, port } = await withHttpDaemon((port) => ({ + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + role: "gateway", + driver: "none", + })); + const minted = await gateway.cli(["token", "create", "--role", "worker"]); + expect(minted.code, minted.stderr).toBe(0); + const { secret } = minted.json as { secret: string }; + const worker = await withDaemon({ + configOverrides: { gateway: { label: "w1", token: secret, url: `ws://127.0.0.1:${port}` } }, + driverScript: { ios: { ...IOS_BASE, physicalDevices: [IPHONE] } }, + }); + let workerId = ""; + await waitFor( + async () => { + const listed = await gateway.cli(["worker", "list", "--json"]); + const workers = (listed.json as { workers?: { id: string; connection: string }[] } | null) + ?.workers; + const connected = workers?.find((entry) => entry.connection === "connected"); + workerId = connected?.id ?? ""; + return connected !== undefined; + }, + { label: "the worker is connected to the gateway" }, + ); + return { gateway, worker, workerId }; + } + + it("shows a device enrolled on worker w1's own daemon in the gateway's list --devices on w1 with its driverDeviceId", async () => { + const { gateway, worker, workerId } = await fleet(); + + const added = await worker.cli(["device", "add", "--platform", "ios", UDID]); + expect(added.code, added.stderr).toBe(0); + + await waitFor( + async () => { + const listed = await gateway.cli(["list", "--devices"]); + const device = (listed.json as Row[] | null)?.find((row) => row.physical); + return device?.workerId === workerId && device.driverDeviceId === UDID; + }, + { label: "the gateway lists the enrolled device on w1" }, + ); + }); + + it("answers device add and device remove with UNSUPPORTED_IN_GATEWAY_MODE and exit 2, saying to run the command on the worker the device is plugged into, and no worker's driver log has an inspectPhysical call", async () => { + const { gateway, worker } = await fleet(); + await worker.driverLog.clear(); + + const added = await gateway.cli(["device", "add", "--platform", "ios", UDID]); + const removed = await gateway.cli(["device", "remove", UDID]); + + for (const result of [added, removed]) { + expect(result.code, result.stderr).toBe(2); + expect(result.error?.code).toBe("UNSUPPORTED_IN_GATEWAY_MODE"); + expect(result.error?.message).toContain("worker the device is plugged into"); + } + expect(await inspectCalls(worker)).toEqual([]); + }); +}); diff --git a/e2e/physical-enrollment.test.ts b/e2e/physical-enrollment.test.ts index d2b753b7d..b38677700 100644 --- a/e2e/physical-enrollment.test.ts +++ b/e2e/physical-enrollment.test.ts @@ -1,48 +1,25 @@ -import { readFile } from "node:fs/promises"; - import { describe, expect, it } from "vitest"; import { PROTOCOL_VERSION_RANGE } from "../src/contract/index.js"; import { NodeIpcTransport } from "../src/ports/index.js"; import { SimlockWire, WireCallError } from "../src/simlock-client/wire.js"; -import type { FakeDriverRefusalReason, ScriptedPhysicalDevice } from "./fake-driver/types.js"; -import { type TestEnv, waitFor, withDaemon, withHttpDaemon } from "./helpers/index.js"; -import { observeFor } from "./helpers/physical-state.js"; -import { readEnrolledApps } from "./helpers/state-db.js"; - -const UDID = "00008110-001A2B3C4D5E"; -const ALIAS = "7A1C0E52-6B7D-4F1E-9C1A-2D3E4F5A6B7C"; -const APPS = Array.from({ length: 12 }, (_, index) => `com.example.app${String(index)}`); - -const IPHONE: ScriptedPhysicalDevice = { - apps: APPS, - class: "phone", - ids: [UDID, ALIAS], - model: "iPhone 15 Pro", - osVersion: "18.4", -}; - -const PIXEL: ScriptedPhysicalDevice = { - apps: ["com.example.mail"], - class: "phone", - ids: ["R58M123ABC"], - model: "Pixel 9", - osVersion: "15", -}; - -const IOS_BASE = { - availableOsVersions: ["18.4"], - defaultModels: { phone: ["iPhone 16"] }, - knownModels: ["iPhone 16", "iPhone 15 Pro"], - modelClasses: { "iPhone 15 Pro": "phone", "iPhone 16": "phone" }, -} as const; - -const ANDROID_BASE = { - availableOsVersions: ["35"], - defaultModels: { phone: ["Pixel 9"] }, - knownModels: ["Pixel 9"], - modelClasses: { "Pixel 9": "phone" }, -} as const; +import type { FakeDriverRefusalReason } from "./fake-driver/types.js"; +import { + ALIAS, + HttpDaemon, + IPHONE, + PIXEL, + Row, + UDID, + add, + bearer, + httpDaemon, + inspectCalls, + leaseGrant, + named, + rows, + startDaemon, +} from "./test-physical-enrollment.js"; const REFUSALS: readonly FakeDriverRefusalReason[] = [ "not-connected", @@ -56,114 +33,6 @@ const REFUSALS: readonly FakeDriverRefusalReason[] = [ "virtual-device", ]; -interface Row { - readonly id: string; - readonly driverDeviceId?: string; - readonly state: string; - readonly physical: boolean; - readonly class?: string; - readonly workerId?: string; - readonly spec: { readonly model: string; readonly platform: string; readonly osVersion: string }; -} - -interface Added { - readonly id: string; - readonly driverDeviceId: string; - readonly platform: string; - readonly model: string; - readonly osVersion: string; - readonly class: string; - readonly appCount: number; - readonly reenrolled: boolean; -} - -async function startDaemon( - options: { - readonly ios?: Record; - readonly android?: Record; - readonly platforms?: readonly ("ios" | "android")[]; - readonly http?: number; - } = {}, -): Promise { - return withDaemon(daemonOptions(options)); -} - -function daemonOptions(options: Parameters[0] = {}) { - return { - ...(options.platforms === undefined ? {} : { fakeDriverPlatforms: options.platforms }), - ...(options.http === undefined - ? {} - : { configOverrides: { http: { enabled: true, host: "127.0.0.1", port: options.http } } }), - driverScript: { - android: { ...ANDROID_BASE, ...options.android }, - ios: { ...IOS_BASE, ...options.ios }, - }, - }; -} - -async function rows(env: TestEnv): Promise { - const result = await env.cli(["list", "--devices"]); - expect(result.code, result.stderr).toBe(0); - return result.json as Row[]; -} - -async function inspectCalls(env: TestEnv): Promise { - return (await env.driverLog.calls()).filter((call) => call.operation === "inspectPhysical"); -} - -async function named(env: TestEnv, ...names: string[]) { - return (await env.events()).filter((entry) => names.includes(entry.event)); -} - -async function add(env: TestEnv, platform: string, id: string): Promise { - const result = await env.cli(["device", "add", "--platform", platform, id, "--json"]); - expect(result.code, result.stderr).toBe(0); - return result.json as Added; -} - -async function leaseGrant( - env: TestEnv, - platform: string, - agent: string, -): Promise<{ device: { driverDeviceId: string; address: string } }> { - const result = await env.cli(["lease", "--platform", platform, "--agent-id", agent, "--detach"]); - expect(result.code, result.stderr).toBe(0); - return result.json as { device: { driverDeviceId: string; address: string } }; -} - -interface HttpDaemon { - readonly env: TestEnv; - readonly base: string; -} - -async function httpDaemon(options: { - readonly ios?: Record; - readonly android?: Record; -}): Promise { - const { env, port } = await withHttpDaemon((port) => daemonOptions({ ...options, http: port })); - const base = `http://127.0.0.1:${String(port)}`; - await waitFor( - async () => { - try { - return (await fetch(`${base}/v1/healthz`)).ok; - } catch { - return false; - } - }, - { label: "HTTP API accepting connections" }, - ); - return { base, env }; -} - -async function bearer(env: TestEnv, role: "agent" | "operator"): Promise> { - const created = await env.cli(["token", "create", "--role", role]); - expect(created.code, created.stderr).toBe(0); - return { - authorization: `Bearer ${(created.json as { secret: string }).secret}`, - "content-type": "application/json", - }; -} - interface ErrorBody { readonly error: { readonly code: string; @@ -500,596 +369,3 @@ describe("simlock device remove", () => { expect(byId.get(second.id)).toBe("ready"); }); }); - -describe("an enrolled physical device", () => { - it("is not granted to a lease --platform ios: the lease gets a simulator", async () => { - const env = await startDaemon({ ios: { physicalDevices: [IPHONE] } }); - const enrolled = await add(env, "ios", UDID); - - const grant = await env.cli(["lease", "--platform", "ios", "--agent-id", "sim", "--detach"]); - - expect(grant.code, grant.stderr).toBe(0); - const device = (grant.json as { device: { id: string; physical: boolean } }).device; - expect(device.physical).toBe(false); - expect(device.id).not.toBe(enrolled.id); - }); - - it("shows its canonical ID as driverDeviceId in status --json from an operator and from an agent session and in list --devices, and a simulator's status entry has none", async () => { - const env = await startDaemon({ ios: { physicalDevices: [IPHONE] } }); - await add(env, "ios", UDID); - const simulator = await leaseGrant(env, "ios", "sim-agent"); - - const operator = await env.cli(["status", "--json"]); - const agent = await env.cli(["--token", "not-an-admin-credential", "status", "--json"]); - - for (const status of [operator, agent]) { - expect(status.code, status.stderr).toBe(0); - const devices = (status.json as { devices: Row[] }).devices; - const physical = devices.find((device) => device.physical); - const virtual = devices.find((device) => !device.physical); - expect(physical?.driverDeviceId).toBe(UDID); - expect(virtual).toBeDefined(); - expect(virtual?.driverDeviceId).toBeUndefined(); - } - const listed = (await rows(env)).find((row) => row.physical); - expect(listed?.driverDeviceId).toBe(UDID); - expect(simulator.device.driverDeviceId).not.toBe(UDID); - }); -}); - -describe("device add over HTTP", () => { - it("POST /v1/devices with an operator token enrols the device; with an agent token it is 403 FORBIDDEN and no inspectPhysical call is made; a body with a worker field or any other unknown field is 400 BAD_REQUEST", async () => { - const { base, env } = await httpDaemon({ ios: { physicalDevices: [IPHONE] } }); - const agent = await bearer(env, "agent"); - const operator = await bearer(env, "operator"); - const post = (headers: Record, body: unknown) => - fetch(`${base}/v1/devices`, { body: JSON.stringify(body), headers, method: "POST" }); - - const refused = await post(agent, { id: UDID, platform: "ios" }); - expect(refused.status).toBe(403); - expect(((await refused.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - expect(await inspectCalls(env)).toEqual([]); - - for (const extra of [{ worker: "w1" }, { nickname: "mine" }]) { - const bad = await post(operator, { id: UDID, platform: "ios", ...extra }); - expect(bad.status).toBe(400); - expect(((await bad.json()) as { error: { code: string } }).error.code).toBe("BAD_REQUEST"); - } - expect(await inspectCalls(env)).toEqual([]); - - const accepted = await post(operator, { id: UDID, platform: "ios" }); - expect(accepted.status).toBe(200); - expect(await accepted.json()).toMatchObject({ - appCount: 12, - driverDeviceId: UDID, - reenrolled: false, - }); - expect((await rows(env)).filter((row) => row.state === "ready")).toHaveLength(1); - }); - - it("DELETE /v1/devices/ with an operator token unenrols the device; with an agent token it is 403 FORBIDDEN and the record stays ready", async () => { - const { base, env } = await httpDaemon({ ios: { physicalDevices: [IPHONE] } }); - const enrolled = await add(env, "ios", UDID); - const agent = await bearer(env, "agent"); - const operator = await bearer(env, "operator"); - const remove = (headers: Record) => - fetch(`${base}/v1/devices/${encodeURIComponent(UDID)}`, { headers, method: "DELETE" }); - - const refused = await remove(agent); - expect(refused.status).toBe(403); - expect(((await refused.json()) as { error: { code: string } }).error.code).toBe("FORBIDDEN"); - expect((await rows(env)).find((row) => row.id === enrolled.id)?.state).toBe("ready"); - - const accepted = await remove(operator); - expect(accepted.status).toBe(200); - expect(await accepted.json()).toEqual({ driverDeviceId: UDID, id: enrolled.id }); - expect((await rows(env)).find((row) => row.id === enrolled.id)?.state).toBe("deleted"); - }); -}); - -describe("device add through a gateway", () => { - async function fleet(): Promise<{ gateway: TestEnv; worker: TestEnv; workerId: string }> { - const { env: gateway, port } = await withHttpDaemon((port) => ({ - configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, - role: "gateway", - driver: "none", - })); - const minted = await gateway.cli(["token", "create", "--role", "worker"]); - expect(minted.code, minted.stderr).toBe(0); - const { secret } = minted.json as { secret: string }; - const worker = await withDaemon({ - configOverrides: { gateway: { label: "w1", token: secret, url: `ws://127.0.0.1:${port}` } }, - driverScript: { ios: { ...IOS_BASE, physicalDevices: [IPHONE] } }, - }); - let workerId = ""; - await waitFor( - async () => { - const listed = await gateway.cli(["worker", "list", "--json"]); - const workers = (listed.json as { workers?: { id: string; connection: string }[] } | null) - ?.workers; - const connected = workers?.find((entry) => entry.connection === "connected"); - workerId = connected?.id ?? ""; - return connected !== undefined; - }, - { label: "the worker is connected to the gateway" }, - ); - return { gateway, worker, workerId }; - } - - it("shows a device enrolled on worker w1's own daemon in the gateway's list --devices on w1 with its driverDeviceId", async () => { - const { gateway, worker, workerId } = await fleet(); - - const added = await worker.cli(["device", "add", "--platform", "ios", UDID]); - expect(added.code, added.stderr).toBe(0); - - await waitFor( - async () => { - const listed = await gateway.cli(["list", "--devices"]); - const device = (listed.json as Row[] | null)?.find((row) => row.physical); - return device?.workerId === workerId && device.driverDeviceId === UDID; - }, - { label: "the gateway lists the enrolled device on w1" }, - ); - }); - - it("answers device add and device remove with UNSUPPORTED_IN_GATEWAY_MODE and exit 2, saying to run the command on the worker the device is plugged into, and no worker's driver log has an inspectPhysical call", async () => { - const { gateway, worker } = await fleet(); - await worker.driverLog.clear(); - - const added = await gateway.cli(["device", "add", "--platform", "ios", UDID]); - const removed = await gateway.cli(["device", "remove", UDID]); - - for (const result of [added, removed]) { - expect(result.code, result.stderr).toBe(2); - expect(result.error?.code).toBe("UNSUPPORTED_IN_GATEWAY_MODE"); - expect(result.error?.message).toContain("worker the device is plugged into"); - } - expect(await inspectCalls(worker)).toEqual([]); - }); -}); - -describe("android enrolment", () => { - it("records the apps the driver lists and shows the device with its Android OS version", async () => { - const env = await startDaemon({ android: { physicalDevices: [PIXEL] } }); - - const result = await env.cli(["device", "add", "--platform", "android", "R58M123ABC"]); - - expect(result.code, result.stderr).toBe(0); - expect(result.stdout).toBe( - "Enrolled Pixel 9 (Android 15) as R58M123ABC, 1 user app recorded.\n", - ); - expect(readEnrolledApps(env)).toEqual([["com.example.mail"]]); - }); -}); - -const AUTO_ENROLL = { devices: { autoEnroll: true }, health: { probeIntervalMs: 200 } }; -const IPHONE_TWO: ScriptedPhysicalDevice = { - apps: ["com.example.second"], - class: "phone", - ids: ["00008110-00FFEEDDCCBB"], - model: "iPhone 15 Pro", - osVersion: "18.4", -}; - -async function autoDaemon( - options: { - readonly ios?: Record; - readonly android?: Record; - readonly config?: Record; - readonly platforms?: readonly ("ios" | "android")[]; - } = {}, -): Promise { - const base = daemonOptions(options); - return withDaemon({ - ...base, - configOverrides: { ...AUTO_ENROLL, ...options.config }, - }); -} - -async function plug( - env: TestEnv, - platform: "ios" | "android", - devices: readonly ScriptedPhysicalDevice[], -): Promise { - await env.driverScript.merge({ [platform]: { physicalDevices: devices } }); -} - -async function autoEnrollLog(env: TestEnv): Promise { - const text = await readFile(env.logPath, "utf8").catch(() => ""); - return text - .split("\n") - .filter((line) => line.startsWith("{")) - .map((line) => (JSON.parse(line) as { message: string }).message) - .filter((message) => message.startsWith("auto-enroll:")); -} - -function refused( - device: ScriptedPhysicalDevice, - reason: FakeDriverRefusalReason, -): ScriptedPhysicalDevice { - return { ...device, refusal: { message: `scripted ${reason} for ${device.ids[0]}`, reason } }; -} - -describe("devices.autoEnroll", () => { - it("enrols a phone plugged in, an iPhone, an Android phone and a phone attached before the daemon starts, each with one device.enrolled initiator auto and its installed apps as kept apps", async () => { - const env = await autoDaemon({ ios: { physicalDevices: [IPHONE] } }); - - await plug(env, "ios", [IPHONE, IPHONE_TWO]); - await plug(env, "android", [PIXEL]); - - await waitFor(async () => (await named(env, "device.enrolled")).length === 3, { - label: "the three phones are enrolled", - }); - const enrolled = (await named(env, "device.enrolled")).map( - (entry) => entry.payload as Added & { readonly initiator: string }, - ); - expect( - enrolled.map((payload) => [payload.driverDeviceId, payload.initiator, payload.appCount]), - ).toEqual( - expect.arrayContaining([ - [UDID, "auto", 12], - ["00008110-00FFEEDDCCBB", "auto", 1], - ["R58M123ABC", "auto", 1], - ]), - ); - expect(enrolled.every((payload) => !payload.reenrolled)).toBe(true); - expect( - readEnrolledApps(env) - .map((apps) => apps.length) - .sort(), - ).toEqual([1, 1, 12]); - expect((await rows(env)).map((row) => row.state)).toEqual(["ready", "ready", "ready"]); - }); - - it("retries a phone that cannot be enrolled and logs one auto-enroll line each time its reason changes, word for word, then enrols it once unlocked", async () => { - const env = await autoDaemon(); - const [a, b, c] = [IPHONE, { ...IPHONE, ids: ["PHONE-B"] }, IPHONE_TWO]; - const line = (device: ScriptedPhysicalDevice, reason: FakeDriverRefusalReason) => - `auto-enroll: ios ${device.ids[0]} not enrolled: scripted ${reason} for ${device.ids[0]}`; - - await plug(env, "ios", [ - refused(a, "locked"), - refused(b, "untrusted"), - refused(c, "untrusted"), - ]); - await waitFor(async () => (await autoEnrollLog(env)).length === 3, { label: "three lines" }); - await observeFor(700, async () => { - expect(await autoEnrollLog(env), "three runs at one reason write one line").toHaveLength(3); - }); - await plug(env, "ios", [refused(a, "locked"), refused(b, "untrusted"), refused(c, "locked")]); - await waitFor(async () => (await autoEnrollLog(env)).length === 4, { label: "locked line" }); - await plug(env, "ios", [ - refused(a, "locked"), - refused(b, "untrusted"), - refused(c, "untrusted"), - ]); - await waitFor(async () => (await autoEnrollLog(env)).length === 5, { - label: "untrusted again", - }); - await observeFor(700, async () => { - expect(await autoEnrollLog(env)).toHaveLength(5); - }); - await plug(env, "ios", [a, refused(b, "untrusted"), refused(c, "untrusted")]); - await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { - label: "the unlocked phone is enrolled", - }); - - const lines = await autoEnrollLog(env); - expect(lines).toEqual( - expect.arrayContaining([line(a, "locked"), line(b, "untrusted"), line(c, "untrusted")]), - ); - expect(lines.filter((entry) => entry.includes(String(c.ids[0])))).toEqual([ - line(c, "untrusted"), - line(c, "locked"), - line(c, "untrusted"), - ]); - expect(lines.filter((entry) => entry.includes(UDID))).toEqual([line(a, "locked")]); - expect((await named(env, "device.enrolled")).map((entry) => entry.payload)).toEqual([ - expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), - ]); - }); - - it("keeps a phone out that was removed while plugged in until it has been missing two runs, while an operator's device add enrols it again", async () => { - const phone = (id: string): ScriptedPhysicalDevice => ({ ...IPHONE, ids: [id] }); - const removed = phone("PHONE-1"); - const brief = phone("PHONE-2"); - const long = phone("PHONE-3"); - const added = phone("PHONE-4"); - const iphones = [removed, brief, long, added]; - const pixel: ScriptedPhysicalDevice = { ...PIXEL, present: true }; - const env = await autoDaemon({ - android: { - failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, - physicalDevices: [pixel], - }, - config: { - health: { probeIntervalMs: 500 }, - warmPool: { quarantine: { maxRetries: 1, retryBackoffMs: 300, retryBackoffMultiplier: 1 } }, - }, - ios: { physicalDevices: iphones }, - }); - const enrolledOf = async (id: string) => - (await named(env, "device.enrolled")) - .map((entry) => entry.payload as Added & { readonly initiator: string }) - .filter((payload) => payload.driverDeviceId === id); - const runs = async (count: number) => { - const lists = async () => - (await env.driverLog.calls()).filter( - (call) => call.operation === "listAttachedPhysical" && call.platform === "ios", - ).length; - const before = await lists(); - await waitFor(async () => (await lists()) >= before + count, { interval: 20 }); - }; - await waitFor(async () => (await named(env, "device.enrolled")).length === 5, { - label: "the five phones are enrolled", - }); - const remove = async (id: string) => { - const result = await env.cli(["device", "remove", id]); - expect(result.code, result.stderr).toBe(0); - }; - - await remove("PHONE-1"); - const grant = await env.cli([ - "lease", - "--platform", - "android", - "--physical", - "--agent-id", - "a", - "--detach", - ]); - expect(grant.code, grant.stderr).toBe(0); - const released = await env.cli(["release", (grant.json as { lease: { id: string } }).lease.id]); - expect(released.code, released.stderr).toBe(0); - await waitFor(async () => (await named(env, "device.quarantine-abandoned")).length === 1, { - label: "the quarantine gives up and unenrols the Android phone", - }); - await remove("PHONE-2"); - await remove("PHONE-3"); - await plug(env, "ios", [ - removed, - { ...brief, present: false }, - { ...long, present: false }, - added, - ]); - await runs(1); - await plug(env, "ios", [removed, brief, { ...long, present: false }, added]); - await runs(1); - await plug(env, "ios", iphones); - await waitFor(async () => (await enrolledOf("PHONE-3")).length === 2, { - label: "the phone missing two runs is enrolled again", - }); - await remove("PHONE-4"); - const again = await env.cli(["device", "add", "--platform", "ios", "PHONE-4"]); - expect(again.code, again.stderr).toBe(0); - await runs(3); - - expect( - (await enrolledOf("PHONE-1")).map((payload) => payload.initiator), - "removed, plugged in", - ).toEqual(["auto"]); - expect( - (await enrolledOf(pixel.ids[0] ?? "")).map((payload) => payload.initiator), - "unenrolled after quarantine", - ).toEqual(["auto"]); - expect( - (await enrolledOf("PHONE-2")).map((payload) => payload.initiator), - "missing one run", - ).toEqual(["auto"]); - expect( - (await enrolledOf("PHONE-3")).map((payload) => payload.initiator), - "missing two runs", - ).toEqual(["auto", "auto"]); - expect( - (await enrolledOf("PHONE-4")).map((payload) => payload.initiator), - "device add while held out", - ).toEqual(["auto", "operator"]); - - const slow = await autoDaemon({ - config: { health: { probeIntervalMs: 5000 } }, - ios: { physicalDevices: [IPHONE, refused({ ...IPHONE, ids: ["PHONE-Y"] }, "locked")] }, - }); - const slowEnrolled = async (id: string) => - (await named(slow, "device.enrolled")) - .map((entry) => entry.payload as Added & { readonly initiator: string }) - .filter((payload) => payload.driverDeviceId === id); - await waitFor(async () => (await slowEnrolled(UDID)).length === 1, { - label: "the first run enrols the unlocked phone", - timeout: 30_000, - }); - const gone = await slow.cli(["device", "remove", UDID]); - expect(gone.code, gone.stderr).toBe(0); - await plug(slow, "ios", [IPHONE, { ...IPHONE, ids: ["PHONE-Y"] }]); - await add(slow, "ios", "PHONE-Y"); - const goneToo = await slow.cli(["device", "remove", "PHONE-Y"]); - expect(goneToo.code, goneToo.stderr).toBe(0); - const listed = async () => - (await slow.driverLog.calls()).filter((call) => call.operation === "listAttachedPhysical") - .length; - const seen = await listed(); - await waitFor(async () => (await listed()) >= seen + 4, { interval: 50, timeout: 30_000 }); - - expect( - (await slowEnrolled(UDID)).map((payload) => payload.initiator), - "removed before the next run", - ).toEqual(["auto"]); - expect( - (await slowEnrolled("PHONE-Y")).map((payload) => payload.initiator), - "added and removed between two runs", - ).toEqual(["operator"]); - }); - - it("forgets which phones are held out when the daemon restarts, so a removed phone still plugged in is enrolled on the first run", async () => { - const env = await autoDaemon({ ios: { physicalDevices: [IPHONE] } }); - await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { - label: "the phone is enrolled", - }); - const removed = await env.cli(["device", "remove", UDID]); - expect(removed.code, removed.stderr).toBe(0); - - await env.restartDaemon(); - - await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { - label: "the restarted daemon enrols the removed phone again", - }); - expect((await named(env, "device.enrolled")).map((entry) => entry.payload)).toEqual([ - expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), - expect.objectContaining({ driverDeviceId: UDID, initiator: "auto" }), - ]); - }); - - it("skips a platform that cannot be listed for that run, still enrols the other platform, logs one line per error message, and logs nothing for a platform with no driver", async () => { - const down = { listAttachedPhysical: { message: "adb is down", type: "generic" } } as const; - const env = await autoDaemon({ - android: { - failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, - physicalDevices: [PIXEL], - }, - config: { warmPool: { quarantine: { retryBackoffMs: 600_000 } } }, - ios: { physicalDevices: [IPHONE] }, - }); - await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { - label: "both phones are enrolled", - }); - const removed = await env.cli(["device", "remove", "R58M123ABC"]); - expect(removed.code, removed.stderr).toBe(0); - const androidLists = async () => - (await env.driverLog.calls()).filter( - (call) => call.operation === "listAttachedPhysical" && call.platform === "android", - ).length; - const runs = async (count: number) => { - const before = await androidLists(); - await waitFor(async () => (await androidLists()) >= before + count, { interval: 20 }); - }; - const couldNotList = async () => - (await autoEnrollLog(env)).filter((line) => line.includes("could not list")); - - await env.driverScript.merge({ android: { failures: down } }); - await runs(4); - await plug(env, "ios", [IPHONE, IPHONE_TWO]); - await waitFor(async () => (await named(env, "device.enrolled")).length === 3, { - label: "the iPhone plugged in while Android cannot be listed is enrolled", - }); - - expect(await couldNotList()).toEqual([ - "auto-enroll: could not list android phones: adb is down", - ]); - - await env.driverScript.merge({ android: { failures: {} } }); - await runs(4); - expect( - (await named(env, "device.enrolled")).filter( - (e) => (e.payload as Added).platform === "android", - ), - "held out across the failing runs", - ).toHaveLength(1); - await env.driverScript.merge({ android: { failures: down } }); - await runs(2); - expect(await couldNotList(), "a list that succeeds resets the message").toHaveLength(2); - - const iosOnly = await autoDaemon({ ios: { physicalDevices: [IPHONE] }, platforms: ["ios"] }); - await waitFor(async () => (await named(iosOnly, "device.enrolled")).length === 1, { - label: "the iPhone is enrolled on a worker with no Android driver", - }); - expect(await autoEnrollLog(iosOnly)).toEqual([]); - }); - - it("grants a waiting physical request the auto-enrolled phone within one run, with no second request, and still fails a model with no record at once", async () => { - const second: ScriptedPhysicalDevice = { ...PIXEL, ids: ["R58M456DEF"] }; - const env = await autoDaemon({ android: { physicalDevices: [PIXEL] } }); - await waitFor(async () => (await named(env, "device.enrolled")).length === 1, { - label: "the first Pixel is enrolled", - }); - const ask = (agent: string, model: string) => [ - "lease", - "--platform", - "android", - "--physical", - "--device", - model, - "--agent-id", - agent, - ]; - const first = await env.cli([...ask("agent-a", "Pixel 9"), "--detach"], { timeout: 30_000 }); - expect(first.code, first.stderr).toBe(0); - const waiter = env.cliBackground(ask("agent-b", "Pixel 9")); - await waitFor(async () => (await named(env, "lease.queued")).length === 1, { - label: "the second request is queued", - }); - - await plug(env, "android", [PIXEL, second]); - - const grant = JSON.parse(await waiter.firstStdoutLine(30_000)) as { - device: { driverDeviceId: string }; - }; - expect(grant.device.driverDeviceId).toBe("R58M456DEF"); - const unknown = await env.cli(ask("agent-c", "Galaxy S99"), { timeout: 30_000 }); - expect(unknown.error?.code).toBe("UNKNOWN_MODEL"); - }); - - it("does nothing without the setting, unset or false: no device.enrolled and no inspectPhysical call after three runs", async () => { - for (const devices of [undefined, { autoEnroll: false }]) { - const env = await withDaemon({ - ...daemonOptions({ ios: { physicalDevices: [IPHONE] } }), - configOverrides: { - health: { probeIntervalMs: 200 }, - ...(devices === undefined ? {} : { devices }), - }, - }); - await observeFor(700, async () => { - expect(await named(env, "device.enrolled")).toEqual([]); - }); - expect(await inspectCalls(env)).toEqual([]); - } - }); - - it("leaves an enrolled phone alone in the states ready, absent, leased and quarantined: no device.enrolled and the kept apps unchanged", async () => { - const env = await autoDaemon({ - android: { - failures: { reclaim: { message: "scripted reclaim failure", type: "generic" } }, - physicalDevices: [PIXEL], - }, - config: { warmPool: { quarantine: { retryBackoffMs: 600_000 } } }, - ios: { physicalDevices: [IPHONE] }, - }); - await waitFor(async () => (await named(env, "device.enrolled")).length === 2, { - label: "both phones are enrolled", - }); - const keptApps = readEnrolledApps(env); - const states = async () => (await rows(env)).map((row) => row.state).sort(); - const stays = async (label: string) => { - await observeFor(700, async () => { - expect(await named(env, "device.enrolled"), label).toHaveLength(2); - }); - expect(readEnrolledApps(env), label).toEqual(keptApps); - }; - - await stays("ready"); - await plug(env, "ios", [{ ...IPHONE, present: false }]); - await waitFor(async () => (await states()).includes("absent"), { - label: "the iPhone is absent", - }); - await stays("absent"); - const leased = await env.cli([ - "lease", - "--platform", - "android", - "--physical", - "--agent-id", - "a", - "--detach", - ]); - expect(leased.code, leased.stderr).toBe(0); - expect(await states()).toContain("leased"); - await stays("leased"); - const released = await env.cli([ - "release", - (leased.json as { lease: { id: string } }).lease.id, - ]); - expect(released.code, released.stderr).toBe(0); - await waitFor(async () => (await states()).includes("quarantined"), { - label: "the Android phone is quarantined", - }); - await stays("quarantined"); - }); -}); diff --git a/e2e/test-physical-enrollment.ts b/e2e/test-physical-enrollment.ts new file mode 100644 index 000000000..b27f18555 --- /dev/null +++ b/e2e/test-physical-enrollment.ts @@ -0,0 +1,158 @@ +import { expect } from "vitest"; + +import type { FakeDriverRefusalReason, ScriptedPhysicalDevice } from "./fake-driver/types.js"; +import { type TestEnv, waitFor, withDaemon, withHttpDaemon } from "./helpers/index.js"; + +export const UDID = "00008110-001A2B3C4D5E"; + +export const ALIAS = "7A1C0E52-6B7D-4F1E-9C1A-2D3E4F5A6B7C"; + +export const APPS = Array.from({ length: 12 }, (_, index) => `com.example.app${String(index)}`); + +export const IPHONE: ScriptedPhysicalDevice = { + apps: APPS, + class: "phone", + ids: [UDID, ALIAS], + model: "iPhone 15 Pro", + osVersion: "18.4", +}; + +export const PIXEL: ScriptedPhysicalDevice = { + apps: ["com.example.mail"], + class: "phone", + ids: ["R58M123ABC"], + model: "Pixel 9", + osVersion: "15", +}; + +export const IOS_BASE = { + availableOsVersions: ["18.4"], + defaultModels: { phone: ["iPhone 16"] }, + knownModels: ["iPhone 16", "iPhone 15 Pro"], + modelClasses: { "iPhone 15 Pro": "phone", "iPhone 16": "phone" }, +} as const; + +export const ANDROID_BASE = { + availableOsVersions: ["35"], + defaultModels: { phone: ["Pixel 9"] }, + knownModels: ["Pixel 9"], + modelClasses: { "Pixel 9": "phone" }, +} as const; + +export interface Row { + readonly id: string; + readonly driverDeviceId?: string; + readonly state: string; + readonly physical: boolean; + readonly class?: string; + readonly workerId?: string; + readonly spec: { readonly model: string; readonly platform: string; readonly osVersion: string }; +} + +export interface Added { + readonly id: string; + readonly driverDeviceId: string; + readonly platform: string; + readonly model: string; + readonly osVersion: string; + readonly class: string; + readonly appCount: number; + readonly reenrolled: boolean; +} + +export async function startDaemon( + options: { + readonly ios?: Record; + readonly android?: Record; + readonly platforms?: readonly ("ios" | "android")[]; + readonly http?: number; + } = {}, +): Promise { + return withDaemon(daemonOptions(options)); +} + +export function daemonOptions(options: Parameters[0] = {}) { + return { + ...(options.platforms === undefined ? {} : { fakeDriverPlatforms: options.platforms }), + ...(options.http === undefined + ? {} + : { configOverrides: { http: { enabled: true, host: "127.0.0.1", port: options.http } } }), + driverScript: { + android: { ...ANDROID_BASE, ...options.android }, + ios: { ...IOS_BASE, ...options.ios }, + }, + }; +} + +export async function rows(env: TestEnv): Promise { + const result = await env.cli(["list", "--devices"]); + expect(result.code, result.stderr).toBe(0); + return result.json as Row[]; +} + +export async function inspectCalls(env: TestEnv): Promise { + return (await env.driverLog.calls()).filter((call) => call.operation === "inspectPhysical"); +} + +export async function named(env: TestEnv, ...names: string[]) { + return (await env.events()).filter((entry) => names.includes(entry.event)); +} + +export async function add(env: TestEnv, platform: string, id: string): Promise { + const result = await env.cli(["device", "add", "--platform", platform, id, "--json"]); + expect(result.code, result.stderr).toBe(0); + return result.json as Added; +} + +export async function leaseGrant( + env: TestEnv, + platform: string, + agent: string, +): Promise<{ device: { driverDeviceId: string; address: string } }> { + const result = await env.cli(["lease", "--platform", platform, "--agent-id", agent, "--detach"]); + expect(result.code, result.stderr).toBe(0); + return result.json as { device: { driverDeviceId: string; address: string } }; +} + +export interface HttpDaemon { + readonly env: TestEnv; + readonly base: string; +} + +export async function httpDaemon(options: { + readonly ios?: Record; + readonly android?: Record; +}): Promise { + const { env, port } = await withHttpDaemon((port) => daemonOptions({ ...options, http: port })); + const base = `http://127.0.0.1:${String(port)}`; + await waitFor( + async () => { + try { + return (await fetch(`${base}/v1/healthz`)).ok; + } catch { + return false; + } + }, + { label: "HTTP API accepting connections" }, + ); + return { base, env }; +} + +export async function bearer( + env: TestEnv, + role: "agent" | "operator", +): Promise> { + const created = await env.cli(["token", "create", "--role", role]); + expect(created.code, created.stderr).toBe(0); + return { + authorization: `Bearer ${(created.json as { secret: string }).secret}`, + "content-type": "application/json", + }; +} + +export function refused( + device: ScriptedPhysicalDevice, + reason: FakeDriverRefusalReason, +): ScriptedPhysicalDevice { + return { ...device, refusal: { message: `scripted ${reason} for ${device.ids[0]}`, reason } }; +} From b2e8e56f19344ff69601ef17d70ad2041b888ec6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 19:50:37 +0100 Subject: [PATCH 74/83] chore(fallow): the health overrides follow the setup helpers into their test-* files --- .fallowrc.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.fallowrc.json b/.fallowrc.json index 753540981..24a7c9609 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -132,12 +132,12 @@ "reason": "Glue scripts for the delivery pipeline: git and transcript walks, option parsing, a Markdown shape check. Their tests spawn them rather than import them and there is no coverage pipeline, so Fallow counts 0% coverage and CRAP is cyclomatic squared plus cyclomatic. Cyclomatic 12 and cognitive 15 are the real limits; maxCrap 156 is the CRAP of a cyclomatic-12 function at 0% coverage (12 * 12 + 12), so CRAP never fails a function the cyclomatic ceiling passes." }, { - "files": ["src/agent-device/host/agent-device-supervisor.test.ts"], + "files": ["src/agent-device/host/test-agent-device-supervisor.ts"], "maxCrap": 80, - "reason": "The supervisor's test arranges a process table, a clock, a filesystem and a scripted runner in one helper so each of its cases reads as one claim; the helper is branching setup, not logic under test." + "reason": "The supervisor's tests arrange a process table, a clock, a filesystem and a scripted runner in one helper so each of its cases reads as one claim; the helper is branching setup, not logic under test." }, { - "files": ["src/drivers/android/screen/session.test.ts"], + "files": ["src/drivers/android/screen/test-session.ts"], "maxCrap": 45, "reason": "The capture's `setup` builds both devices' worlds — two targets, a scripted runner, an adb server, a clock, a filesystem, an optional scripted screen answer — in one helper so each case reads as one claim; the branches are optional arrangement, not logic under test." }, From dbea68555055f1da4c7e9f39a6c108e9ab53b266 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 20:14:44 +0100 Subject: [PATCH 75/83] test(cli): split index.test.ts by topic --- src/cli/index-admin-auto-launch.test.ts | 270 + src/cli/index-admin-holder-renew.test.ts | 960 ++ src/cli/index-auto-launch-handshake.test.ts | 643 + src/cli/index-baseline-alerts-warm.test.ts | 531 + src/cli/index-catalog-components.test.ts | 564 + src/cli/index-daemon-service.test.ts | 763 ++ src/cli/index-daemon-status.test.ts | 560 + src/cli/index-device-config.test.ts | 802 ++ src/cli/index-environment-helpers.test.ts | 166 + src/cli/index-events-mcp-smoke.test.ts | 758 ++ src/cli/index-exit-codes-passthrough.test.ts | 685 + src/cli/index-lease-requests.test.ts | 988 ++ src/cli/index-status-daemon-lines.test.ts | 752 ++ src/cli/index-status-warm-pool.test.ts | 495 + src/cli/index-update-status-stats.test.ts | 846 ++ src/cli/index-worker.test.ts | 858 ++ src/cli/index.test.ts | 11021 +---------------- src/cli/test-index.ts | 573 + src/source-hygiene.test.ts | 4 +- 19 files changed, 11256 insertions(+), 10983 deletions(-) create mode 100644 src/cli/index-admin-auto-launch.test.ts create mode 100644 src/cli/index-admin-holder-renew.test.ts create mode 100644 src/cli/index-auto-launch-handshake.test.ts create mode 100644 src/cli/index-baseline-alerts-warm.test.ts create mode 100644 src/cli/index-catalog-components.test.ts create mode 100644 src/cli/index-daemon-service.test.ts create mode 100644 src/cli/index-daemon-status.test.ts create mode 100644 src/cli/index-device-config.test.ts create mode 100644 src/cli/index-environment-helpers.test.ts create mode 100644 src/cli/index-events-mcp-smoke.test.ts create mode 100644 src/cli/index-exit-codes-passthrough.test.ts create mode 100644 src/cli/index-lease-requests.test.ts create mode 100644 src/cli/index-status-daemon-lines.test.ts create mode 100644 src/cli/index-status-warm-pool.test.ts create mode 100644 src/cli/index-update-status-stats.test.ts create mode 100644 src/cli/index-worker.test.ts create mode 100644 src/cli/test-index.ts diff --git a/src/cli/index-admin-auto-launch.test.ts b/src/cli/index-admin-auto-launch.test.ts new file mode 100644 index 000000000..f577c263b --- /dev/null +++ b/src/cli/index-admin-auto-launch.test.ts @@ -0,0 +1,270 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + FakeClock, + FakeDaemonLauncher, + IpcError, + MemoryFilesystem, + MemoryIpcTransport, + FakeSystemStats, +} from "../ports/index.js"; +import { runCli, type CliEnvironmentPorts } from "./index.js"; +import { + gibibyte, + outputCapture, + runningDaemons, + settle, + startInMemoryDaemon, + temporaryDirectories, +} from "./test-index.js"; + +afterEach(async () => { + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("CLI: auto-launch's startup deadline (#663)", () => { + /** + * why: the launcher hands back a daemon that claims its socket only `bootMs` later, the way + * `NodeDaemonLauncher` returns while the detached process it spawned is still booting; what + * the CLI can see is how long the socket stays absent, not how long `launch` itself blocks. + */ + function slowBootPorts(bootMs: number): { + readonly ports: CliEnvironmentPorts; + readonly clock: FakeClock; + } { + const filesystem = new MemoryFilesystem(); + const clock = new FakeClock(0); + const ipcTransport = new MemoryIpcTransport(); + const ports: CliEnvironmentPorts = { + filesystem, + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: ipcTransport, + launcher: new FakeDaemonLauncher(() => { + clock.setTimer(bootMs, () => { + void startInMemoryDaemon({ + adminTokenPath: "/simlock/admin.token", + filesystem, + ipcTransport, + socketPath: "/simlock/daemon.sock", + }); + }); + }), + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }; + return { clock, ports }; + } + + /** + * why: turns the clock at the auto-launch poll's own 50 ms step, with room to run past the + * startup deadline and keep pumping the boot the CLI abandoned; the clock is fake, so the + * extra ticks cost nothing. The `settled` guard makes a hang fail on a named assertion + * instead of on a test timeout. + */ + async function runWithTicks( + command: readonly string[], + ports: CliEnvironmentPorts, + clock: FakeClock, + runOnMs: number, + ): Promise<{ readonly exitCode: number; readonly output: ReturnType }> { + const output = outputCapture(ports); + let settled = false; + const exitCode = runCli(command, output.environmentWith()).then((code) => { + settled = true; + return code; + }); + for (let tick = 0; tick < 500 && (!settled || clock.now() < runOnMs); tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect( + settled, + `simlock ${command.join(" ")} still had not settled at ${String(clock.now())}ms`, + ).toBe(true); + return { exitCode: await exitCode, output }; + } + + it("waits 20 s for a daemon it auto-launched: within the deadline it reports as running, past it the command reports a failed start", async () => { + const cases = [ + { bootMs: 19_900, label: "19.9 s" }, + { bootMs: 20_100, label: "20.1 s" }, + ]; + + // why: every case runs before anything is asserted, so the case that passes cannot hide + // what the case that fails did to the daemon it launched. + const outcomes = await Promise.all( + cases.map(async (testCase) => { + const { clock, ports } = slowBootPorts(testCase.bootMs); + const { exitCode, output } = await runWithTicks( + ["daemon", "start", "--json"], + ports, + clock, + testCase.bootMs + 1_000, + ); + const after = outputCapture(ports); + const statusExit = await runCli(["daemon", "status", "--json"], after.environmentWith()); + const status = JSON.parse(after.stdout || "{}") as { daemon?: { health?: string } }; + return { + exitCode, + label: testCase.label, + timedOutStarting: output.stderr.includes("Timed out starting simlock daemon"), + stdout: output.stdout, + statusHealth: status.daemon?.health ?? after.stdout, + statusExit, + }; + }), + ); + + expect( + outcomes.map((outcome) => ({ + exitCode: outcome.exitCode, + label: outcome.label, + // why: the failed start is asserted as the timeout's wording, not the whole line or + // its error code, so what the code reports can change without moving this test. + timedOutStarting: outcome.timedOutStarting, + stdout: outcome.stdout, + })), + ).toEqual([ + { + exitCode: 0, + label: "19.9 s", + timedOutStarting: false, + stdout: '{"status":"running"}\n', + }, + { + exitCode: 1, + label: "20.1 s", + timedOutStarting: true, + stdout: "", + }, + ]); + + // why: asserted apart from the exit code so a fix that only softens the timeout message + // fails here -- `daemon status` never launches one, so what it reaches is the daemon this + // command launched and then gave up on. + for (const outcome of outcomes) { + expect(outcome.statusExit, `${outcome.label}: status stderr`).toBe(0); + expect( + ["starting", "running"].includes(outcome.statusHealth as string), + `${outcome.label}: status says ${JSON.stringify(outcome.statusHealth)}`, + ).toBe(true); + } + }); +}); + +describe("CLI: auto-launch's failed start (#672)", () => { + it("reports a daemon it launched but never reached as DAEMON_STARTUP_FAILED, exit 1", async () => { + // why: nothing ever claims the socket, which is what a daemon still booting past the + // deadline looks like to the CLI. The run goes well past any deadline the loop could hold, + // so the code asserted here is the one the CLI chose for giving up, never one a clock reached. + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher(); + const output = outputCapture({ + filesystem: new MemoryFilesystem(), + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: new MemoryIpcTransport(), + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli(["daemon", "start", "--json"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 700 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(settled, `simlock daemon start still had not settled at ${String(clock.now())}ms`).toBe( + true, + ); + expect(await exitCode).toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: expect.stringContaining("Timed out starting simlock daemon"), + }, + }); + expect(launcher.launches).toBe(1); + }); +}); + +describe("CLI: auto-launch's daemon that lost the socket to another daemon (#760)", () => { + it("status goes on with the daemon that holds the socket when the launched daemon exits already running", async () => { + const filesystem = new MemoryFilesystem(); + const clock = new FakeClock(0); + // why: the winner is not listening yet at the first poll after the launch, so the loop reaches + // the launched daemon's exit before it finds the winner. + class LateWinnerTransport extends MemoryIpcTransport { + connects = 0; + override async connect(endpoint: string): ReturnType { + this.connects += 1; + if (this.connects === 2) { + throw new IpcError("endpoint-not-found", `No IPC endpoint at ${endpoint}`, undefined); + } + return super.connect(endpoint); + } + } + const ipcTransport = new LateWinnerTransport(); + const launcher = new FakeDaemonLauncher( + async () => { + await startInMemoryDaemon({ + adminTokenPath: "/simlock/admin.token", + filesystem, + ipcTransport, + socketPath: "/simlock/daemon.sock", + }); + }, + { + exit: { code: 1, signal: null }, + output: `${JSON.stringify({ + message: "Daemon failed to start", + fields: { message: "Simlock daemon is already running at /simlock/daemon.sock" }, + })}\n`, + }, + ); + const output = outputCapture({ + filesystem, + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: ipcTransport, + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli(["status"], output.environmentWith()).then((code) => { + settled = true; + return code; + }); + for (let tick = 0; tick < 100 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(settled, "status had not settled by 5 s").toBe(true); + expect(await exitCode).toBe(0); + expect(output.stderr).not.toContain("DAEMON_STARTUP_FAILED"); + expect(ipcTransport.connects).toBeGreaterThanOrEqual(3); + }); +}); diff --git a/src/cli/index-admin-holder-renew.test.ts b/src/cli/index-admin-holder-renew.test.ts new file mode 100644 index 000000000..09ca70c85 --- /dev/null +++ b/src/cli/index-admin-holder-renew.test.ts @@ -0,0 +1,960 @@ +import { EventEmitter } from "node:events"; +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + FakeClock, + FakeDaemonLauncher, + FakeParentWatch, + MemoryFilesystem, + MemoryIpcTransport, + FakeSystemStats, +} from "../ports/index.js"; +import { fromWireError, SimlockError, type AnySimlockError } from "../contract/index.js"; +import type { LeaseGrant } from "../simlock-client/client.js"; +import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; +import { runCli, type CliEnvironment } from "./index.js"; +import { promiseState } from "../test-support/promise-state.js"; +import { + detachedGrant, + fakeClient, + gibibyte, + outputCapture, + runningDaemons, + settle, + simlockError, + startInMemoryDaemon, + temporaryDirectories, +} from "./test-index.js"; + +afterEach(async () => { + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("CLI: admin credential resolution (ADR 0003 §5)", () => { + it.each([ + [ + "--token over env and file", + ["--token", "from-flag", "status"], + "from-env", + "from-file", + "from-flag", + true, + ], + [ + "SIMLOCK_ADMIN_TOKEN over the file when --token is absent", + ["status"], + "from-env", + "from-file", + "from-env", + false, + ], + [ + "the admin.token file when neither --token nor the env var is set", + ["status"], + undefined, + "from-file", + "from-file", + true, + ], + ] as const)( + "resolves the admin credential in priority order: %s", + async (_case, argv, envToken, fileToken, expected, stderrEmpty) => { + const output = outputCapture(); + let seen: string | undefined; + await runCli( + [...argv], + output.environmentWith({ + ...(envToken === undefined ? {} : { adminTokenFromEnv: envToken }), + readAdminTokenFile: async () => fileToken, + connectAdmin: async (resolveCredential) => { + seen = await resolveCredential(); + return fakeClient(); + }, + }), + ); + expect(seen).toBe(expected); + if (stderrEmpty) expect(output.stderr).toBe(""); + }, + ); + + it("retries a briefly-missing admin.token file before giving up", async () => { + const output = outputCapture(); + let reads = 0; + let seen: string | undefined; + await runCli( + ["status"], + output.environmentWith({ + readAdminTokenFile: async () => { + reads += 1; + return reads < 3 ? undefined : "from-file-after-retry"; + }, + connectAdmin: async (resolveCredential) => { + seen = await resolveCredential(); + return fakeClient(); + }, + }), + ); + expect(reads).toBe(3); + expect(seen).toBe("from-file-after-retry"); + expect(output.stderr).toBe(""); + }); + + it("connects with no credential and writes a stderr notice when every source is empty", async () => { + const output = outputCapture(); + let seen: string | undefined | "unset" = "unset"; + await runCli( + ["status"], + output.environmentWith({ + readAdminTokenFile: async () => undefined, + connectAdmin: async (resolveCredential) => { + seen = await resolveCredential(); + return fakeClient(); + }, + }), + ); + expect(seen).toBeUndefined(); + expect(output.stderr.trim().split("\n")).toHaveLength(1); + const notice: unknown = JSON.parse(output.stderr); + expect((notice as { notice: string }).notice).toMatch(/agent/i); + }); + + it("does not read the admin.token file at all when --token is given (B2 ordering)", async () => { + // why: The file source must never be consulted -- let alone before the connection exists -- when + // a higher-priority source already answered. Also guards against a regression back to + // resolving credentials before `connectAdmin` is even called. + const output = outputCapture(); + let fileReads = 0; + await runCli( + ["--token", "from-flag", "status"], + output.environmentWith({ + readAdminTokenFile: async () => { + fileReads += 1; + return "from-file"; + }, + connectAdmin: async (resolveCredential) => { + await resolveCredential(); + return fakeClient(); + }, + }), + ); + expect(fileReads).toBe(0); + }); + + it("B1: degrades to an agent session with a stderr notice when the daemon rejects the credential", async () => { + const output = outputCapture(); + let attempt = 0; + let seenOnRetry: string | undefined | "unset" = "unset"; + await runCli( + ["status"], + output.environmentWith({ + readAdminTokenFile: async () => "stale-secret", + connectAdmin: async (resolveCredential) => { + attempt += 1; + if (attempt === 1) { + const credential = await resolveCredential(); + expect(credential).toBe("stale-secret"); + throw simlockError("ADMIN_AUTHENTICATION_FAILED"); + } + seenOnRetry = await resolveCredential(); + return fakeClient(); + }, + }), + ); + expect(attempt).toBe(2); + expect(seenOnRetry).toBeUndefined(); + expect(output.stderr.trim().split("\n")).toHaveLength(1); + const notice = JSON.parse(output.stderr) as { notice: string }; + expect(notice.notice).toMatch(/admin\.token/i); + expect(notice.notice).toMatch(/agent/i); + }); + + it("B1: does not retry, and propagates the error, when the connection already had no credential", async () => { + const output = outputCapture(); + let attempts = 0; + const exitCode = await runCli( + ["status"], + output.environmentWith({ + readAdminTokenFile: async () => undefined, + connectAdmin: async (resolveCredential) => { + attempts += 1; + await resolveCredential(); + throw simlockError("ADMIN_AUTHENTICATION_FAILED"); + }, + }), + ); + expect(attempts).toBe(1); + expect(exitCode).toBe(1); + expect(JSON.parse(output.stderr).error.code).toBe("ADMIN_AUTHENTICATION_FAILED"); + }); + + it("B1: an explicit but wrong --token also degrades to agent, with a generic (not file-specific) notice", async () => { + const output = outputCapture(); + let attempt = 0; + await runCli( + ["--token", "wrong", "status"], + output.environmentWith({ + connectAdmin: async (resolveCredential) => { + attempt += 1; + const credential = await resolveCredential(); + if (attempt === 1) { + expect(credential).toBe("wrong"); + throw simlockError("ADMIN_AUTHENTICATION_FAILED"); + } + expect(credential).toBeUndefined(); + return fakeClient(); + }, + }), + ); + expect(attempt).toBe(2); + const notice = JSON.parse(output.stderr) as { notice: string }; + expect(notice.notice).not.toMatch(/admin\.token/i); + expect(notice.notice).toMatch(/agent/i); + }); + + it("B2: reaches admin on a cold start, on the very first invocation, no daemon running yet", async () => { + const filesystem = new MemoryFilesystem(); + const ipcTransport = new MemoryIpcTransport(); + const socketPath = "/simlock/daemon.sock"; + const adminTokenPath = "/simlock/admin.token"; + const launcher = new FakeDaemonLauncher(async () => { + await startInMemoryDaemon({ adminTokenPath, filesystem, ipcTransport, socketPath }); + }); + const output = outputCapture({ + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + ipc: ipcTransport, + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + const exitCode = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith(), + ); + expect(exitCode).toBe(0); + expect(launcher.launches).toBe(1); + const grant = JSON.parse(output.stdout) as LeaseGrant & { role: string }; + expect(grant.role).toBe("admin"); + // why: Provisioning progress pushes are expected noise on stderr; the agent-fallback notice is + // the thing that must be absent. + expect(output.stderr).not.toContain("notice"); + }); + + it("daemon start reports the config error and launches nothing when config.json is invalid", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic( + "/simlock/config.json", + JSON.stringify({ capacity: { config: { limits: { android: { maxDevices: 0 } } } } }), + ); + const launcher = new FakeDaemonLauncher(); + const output = outputCapture({ + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + ipc: new MemoryIpcTransport(), + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + const exitCode = await runCli(["daemon", "start"], output.environmentWith()); + expect(exitCode).toBe(2); + expect(launcher.launches).toBe(0); + expect(output.stderr).toContain("limits.android.maxDevices"); + expect(output.stderr).not.toContain("Timed out"); + }); + + it("B1: a stale admin.token degrades to an agent session instead of bricking the invocation", async () => { + const filesystem = new MemoryFilesystem(); + const ipcTransport = new MemoryIpcTransport(); + const socketPath = "/simlock/daemon.sock"; + const adminTokenPath = "/simlock/admin.token"; + // why: A daemon actually runs and persists its own secret first, then the file is overwritten + // with a value that hashes to nothing the running daemon recognizes -- exactly what a + // `kill -9`'d daemon's leftover `admin.token` looks like to the *next* daemon incarnation, + // which never touches a file it did not itself just persist. + await startInMemoryDaemon({ adminTokenPath, filesystem, ipcTransport, socketPath }); + await filesystem.writeFileAtomic(adminTokenPath, "stale-secret-from-a-killed-daemon\n"); + const launcher = new FakeDaemonLauncher(); + const output = outputCapture({ + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + ipc: ipcTransport, + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + const exitCode = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith(), + ); + expect(exitCode).toBe(0); + expect(launcher.launches).toBe(0); + const grant = JSON.parse(output.stdout) as LeaseGrant & { role: string }; + expect(grant.role).toBe("agent"); + expect(output.stderr).toContain("admin.token"); + }); +}); + +describe("CLI: holder renew and release (ADR 0004 §2)", () => { + it("renews at a third of the TTL the daemon returned, re-deriving the cadence from each renewal", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + const signals = new EventEmitter(); + const renewals: Array<{ leaseId: string; at: number }> = []; + let released: string | undefined; + const client = fakeClient({ + renewLease: (input) => { + renewals.push({ at: clock.now(), leaseId: input.leaseId }); + // why: Half the grant's TTL comes back, so the next renewal must be half as far out too. + return Promise.resolve({ + deviceId: "dev_1", + grantedAt: 0, + id: input.leaseId, + ownerId: "test-requester", + requesterId: "test-requester", + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: clock.now() + 30_000, + }); + }, + releaseLease: (input) => { + released = input.leaseId; + return Promise.resolve({ leaseId: input.leaseId }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + clock, + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + + // why: The grant's deadline is 60_000, so the first renewal is due at 20_000 -- and not before. + clock.advance(19_999); + await settle(); + expect(renewals).toEqual([]); + clock.advance(1); + await settle(); + expect(renewals).toEqual([{ at: 20_000, leaseId: "lse_1" }]); + + // why: 30_000ms of TTL came back at 20_000, so the next renewal is at 30_000, not at 40_000. + clock.advance(10_000); + await settle(); + expect(renewals).toEqual([ + { at: 20_000, leaseId: "lse_1" }, + { at: 30_000, leaseId: "lse_1" }, + ]); + + signals.emit("SIGINT"); + expect(await runPromise).toBe(0); + // why: The release is the CLI's own call, not a side effect of the socket closing. + expect(released).toBe("lse_1"); + // why: And the timer is gone with it: no renewal can race the release, and nothing keeps the + // process alive after the command returns. + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + expect(renewals).toHaveLength(2); + }); + + it("exits 1 on a dead connection, naming the lease that outlived it, and releases nothing", async () => { + // why: ADR 0004 §3: the daemon released nothing when the socket died, so the lease is still + // granted -- the holder says so, says until when, and does not pretend to release it. + const output = outputCapture(); + let connectionLostListener: ((error: AnySimlockError) => void) | undefined; + let released = false; + const client = fakeClient({ + onConnectionLost: (listener) => { + connectionLostListener = listener; + return () => {}; + }, + releaseLease: () => { + released = true; + return Promise.resolve({ leaseId: "lse_1" }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ connectAdmin: async () => client }), + ); + // why: Let `requestLease` resolve and `runLease` reach its wait point, so the listener exists. + await settle(); + connectionLostListener?.( + new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), + ); + + expect(await runPromise).toBe(1); + const line = JSON.parse(output.stderr.trim().split("\n").at(-1) as string) as { + error: { code: string; message: string }; + }; + expect(line.error.code).toBe("DAEMON_CONNECTION_LOST"); + expect(line.error.message).toContain("lse_1"); + expect(line.error.message).toContain("60000"); + expect(released).toBe(false); + }); + + it("names the latest deadline in that line, not the one the grant carried", async () => { + // why: The deadline moves on every renewal, and this line is the only place a holder reports + // it. Quoting the grant-time one would, after about a TTL of uptime, name a moment in the + // past on a lease that is perfectly alive. + const clock = new FakeClock(0); + const output = outputCapture(); + let connectionLostListener: ((error: AnySimlockError) => void) | undefined; + const client = fakeClient({ + onConnectionLost: (listener) => { + connectionLostListener = listener; + return () => {}; + }, + renewLease: (input) => + Promise.resolve({ + deviceId: "dev_1", + grantedAt: 0, + id: input.leaseId, + ownerId: "test-requester", + requesterId: "test-requester", + lastRenewedAt: clock.now(), + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: clock.now() + 60_000, + }), + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + + // why: Two renewals: the grant's deadline was 60_000, so they land at 20_000 and 40_000, each + // answering with a deadline a full TTL further out. + clock.advance(20_000); + await settle(); + clock.advance(20_000); + await settle(); + + connectionLostListener?.( + new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), + ); + expect(await runPromise).toBe(1); + + const line = JSON.parse(output.stderr.trim().split("\n").at(-1) as string) as { + error: { message: string }; + }; + expect(line.error.message).toContain("100000"); + expect(line.error.message).not.toContain("60000"); + }); + + it("writes one line for a lost connection, even with a renewal in flight when it dies", async () => { + // why: The same socket carries both: the connection-lost listener writes the line that names + // the lease and its deadline, and the `lease.renew` that was in flight rejects with + // `DAEMON_CONNECTION_LOST`. That rejection is a retryable failure as far as + // `startLeaseRenewal` knows -- not a `lease-lost` -- so the holder must not answer it with + // a second line saying the daemon is gone, this one without the lease id a reader needs. + // Two things stop that, and this pins the outcome rather than either mechanism: the + // `finally` stops renewal (nothing is reported after `stop()`), and `onError` checks + // `connectionLost` so the answer does not depend on which of the two microtask chains + // wins. + const clock = new FakeClock(0); + const output = outputCapture(); + let connectionLostListener: ((error: AnySimlockError) => void) | undefined; + let rejectRenew: ((error: unknown) => void) | undefined; + const client = fakeClient({ + onConnectionLost: (listener) => { + connectionLostListener = listener; + return () => {}; + }, + renewLease: () => + new Promise((_resolve, reject) => { + rejectRenew = reject; + }), + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + + // why: The grant's deadline is 60_000: the first renewal starts at 20_000 and never answers. + clock.advance(20_000); + await settle(); + expect(rejectRenew).toBeDefined(); + + // why: The socket dies. In that order, deliberately: `SimlockClient.onConnectionLost` promises + // that "every in-flight call has already rejected `DAEMON_CONNECTION_LOST` by the time + // this listener runs", so the renewal's rejection is queued first and its handler runs + // after the listener has had its say. + rejectRenew?.(new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {})); + connectionLostListener?.( + new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), + ); + + expect(await runPromise).toBe(1); + const errorLines = output.stderr + .trim() + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as { error?: { code: string; message: string } }) + .filter((line) => line.error !== undefined); + expect(errorLines).toHaveLength(1); + expect(errorLines[0]?.error?.code).toBe("DAEMON_CONNECTION_LOST"); + expect(errorLines[0]?.error?.message).toContain("lse_1"); + }); + + it("sends --ttl as the request's own ttlMs, and nothing when it is not given", async () => { + const output = outputCapture(); + const signals = new EventEmitter(); + const requested: Array = []; + const environment = output.environmentWith({ + clock: new FakeClock(0), + connectAdmin: async () => + fakeClient({ + requestLease: (input) => { + requested.push(input.ttlMs); + return Promise.resolve(detachedGrant); + }, + }), + signals: signals as unknown as CliEnvironment["signals"], + }); + await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach", "--ttl", "30m"], + environment, + ); + await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + environment, + ); + + // why: ADR 0004 §4: `--ttl` replaces `lease.defaultTtlMs` for this lease; omitting it sends no + // TTL at all, so the daemon's own default applies rather than a number the CLI invented. + expect(requested).toEqual([30 * 60_000, undefined]); + }); + + it("the CLI --lease-id flag sends leaseId and exits 13 on LEASE_ID_TAKEN", async () => { + const output = outputCapture(); + const requested: unknown[] = []; + let answer: "grant" | "taken" = "grant"; + const environment = output.environmentWith({ + clock: new FakeClock(0), + connectAdmin: async () => + fakeClient({ + requestLease: (input) => { + requested.push("leaseId" in input ? input.leaseId : "absent"); + return answer === "grant" + ? Promise.resolve(detachedGrant) + : Promise.reject( + fromWireError("LEASE_ID_TAKEN", "lease ID ad-7f3a is already in use", { + leaseId: "ad-7f3a", + }), + ); + }, + }), + }); + const lease = ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"]; + + await runCli([...lease, "--lease-id", "ad-7f3a"], environment); + await runCli(lease, environment); + answer = "taken"; + const exitCode = await runCli([...lease, "--lease-id", "ad-7f3a"], environment); + + // why: The flag's value goes out as typed and an omitted flag sends none: the contract decides + // what an ID may look like. + expect(requested).toEqual(["ad-7f3a", "absent", "ad-7f3a"]); + expect(exitCode).toBe(13); + expect(output.stderr).toContain('"code":"LEASE_ID_TAKEN"'); + }); + + it.each(["SIGINT", "SIGTERM"] as const)( + "releases explicitly on %s (ADR 0004 §2's catchable signals)", + async (signal) => { + const output = outputCapture(); + const signals = new EventEmitter(); + let released: string | undefined; + const client = fakeClient({ + releaseLease: (input) => { + released = input.leaseId; + return Promise.resolve({ leaseId: input.leaseId }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + clock: new FakeClock(0), + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + signals.emit(signal); + + expect(await runPromise).toBe(0); + expect(released).toBe("lse_1"); + }, + ); + + it.each([ + { trigger: "SIGINT", act: (signals: EventEmitter) => signals.emit("SIGINT") }, + { trigger: "SIGTERM", act: (signals: EventEmitter) => signals.emit("SIGTERM") }, + { + trigger: "the exit of the --bind-pid process", + act: (_signals: EventEmitter, parentWatch: FakeParentWatch) => parentWatch.exit(9876), + }, + ])("cancels its waiting request and exits 0 on $trigger before a grant", async ({ act }) => { + const output = outputCapture(); + const signals = new EventEmitter(); + const parentWatch = new FakeParentWatch(); + let requestSignal: AbortSignal | undefined; + let requested = false; + const client = fakeClient({ + // why: The request waits in the queue and never resolves on its own; like the client's + // documented abort, it rejects CANCELLED once its signal aborts. + requestLease: (_input, options) => { + requested = true; + requestSignal = options?.signal; + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener("abort", () => { + reject(new SimlockError("CANCELLED", "domain", "Lease request was cancelled", {})); + }); + }); + }, + }); + const run = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "9876"], + output.environmentWith({ + connectAdmin: async () => client, + parentPid: 4321, + parentWatch, + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + const runState = promiseState(run); + await vi.waitFor(() => expect(requested).toBe(true)); + + act(signals, parentWatch); + await settle(); + + expect(requestSignal?.aborted).toBe(true); + expect(runState.state).toBe("fulfilled"); + expect(await run).toBe(0); + expect(output.stdout).toBe(""); + }); + + it("reports a failed renewal on stderr and keeps holding the lease", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + const signals = new EventEmitter(); + let released: string | undefined; + const client = fakeClient({ + renewLease: () => + Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist the lease", {})), + releaseLease: (input) => { + released = input.leaseId; + return Promise.resolve({ leaseId: input.leaseId }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + clock, + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + clock.advance(20_000); + await settle(); + + const errorLines = output.stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record) + .filter((line) => line.error !== undefined); + expect(errorLines).toContainEqual({ + error: { code: "INTERNAL", message: "could not persist the lease" }, + }); + + // why: A failed renewal is not an exit condition: the holder still holds, and still releases. + signals.emit("SIGINT"); + expect(await runPromise).toBe(0); + expect(released).toBe("lse_1"); + }); + + it("still closes the connection when stderr cannot take the failed release's error line", async () => { + const output = outputCapture(); + const signals = new EventEmitter(); + let closed = false; + const client = fakeClient({ + close: () => { + closed = true; + return Promise.resolve(); + }, + releaseLease: () => Promise.reject(new Error("release failed")), + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + clock: new FakeClock(0), + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], + stderr: { + write: (value: string) => { + if (value.includes("release failed")) throw new Error("EPIPE"); + return undefined; + }, + }, + }), + ); + await vi.waitFor(() => expect(output.stdout).not.toBe("")); + + signals.emit("SIGINT"); + + // why: a closed stderr pipe must not skip the dispose and the close that follow the release. + expect(await runPromise).toBe(0); + expect(closed).toBe(true); + }); + + it("exits 14 without a farewell release when a renewal is answered UNKNOWN_LEASE", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let released = false; + let renewals = 0; + const client = fakeClient({ + renewLease: () => { + renewals += 1; + return Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lse_1" }), + ); + }, + releaseLease: () => { + released = true; + return Promise.resolve({ leaseId: "lse_1" }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + clock.advance(20_000); + + // why: The daemon's answer is final: this ends the same way the lease-lost push does, rather + // than repeating the rejection until the deadline and then sitting alive. + expect(await runPromise).toBe(14); + expect(released, "asking again would only raise UNKNOWN_LEASE").toBe(false); + expect(renewals).toBe(1); + expect(clock.pendingTimerCount).toBe(0); + + // why: The same stderr line the push path writes, so a script tailing for `push: "lease-lost"` + // sees this way of losing a lease too, with the error line after it as the detail. + const lines = output.stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(lines).toContainEqual({ + push: "lease-lost", + deviceId: "dev_1", + leaseId: "lse_1", + reason: "renew-rejected", + }); + expect(lines.at(-1)).toEqual({ + error: { code: "UNKNOWN_LEASE", message: "no such lease" }, + }); + }); + + it("exits 14 when renewal gives up, rather than holding a lease it cannot keep", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let released = false; + const client = fakeClient({ + // why: Never terminal, never successful: the ladder retries until the lease's own deadline + // passes, and then there is nothing left to hold. + renewLease: () => + Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist", {})), + releaseLease: () => { + released = true; + return Promise.resolve({ leaseId: "lse_1" }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + clock.advance(20_000); // first attempt, fails, retries + await settle(); + clock.advance(60_000); // the retry lands past the lease's 60_000 deadline + + expect(await runPromise).toBe(14); + expect(released, "a lease that expired is not this process's to release").toBe(false); + const lines = output.stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(lines).toContainEqual({ + push: "lease-lost", + deviceId: "dev_1", + leaseId: "lse_1", + reason: "renew-failed", + }); + }); + + it("writes one lease-lost line when a rejected renewal and the daemon's push both arrive", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let leaseLostListener: + | ((push: { leaseId: string; deviceId: string; reason: string }) => void) + | undefined; + const client = fakeClient({ + onLeaseLost: (listener) => { + leaseLostListener = listener; + return () => {}; + }, + renewLease: () => + Promise.reject( + new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lse_1" }), + ), + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + clock.advance(20_000); + await settle(); + // why: The daemon's own push for the same lease follows the renewal's answer. One ending, one + // line: a script tailing stderr must not read this as two devices lost. + leaseLostListener?.({ deviceId: "dev_1", leaseId: "lse_1", reason: "expired" }); + + expect(await runPromise).toBe(14); + const leaseLostLines = output.stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record) + .filter((line) => line.push === "lease-lost"); + expect(leaseLostLines).toEqual([ + { push: "lease-lost", deviceId: "dev_1", leaseId: "lse_1", reason: "renew-rejected" }, + ]); + }); + + it("gives up on a farewell release the daemon never answers, instead of hanging", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + const signals = new EventEmitter(); + const client = fakeClient({ releaseLease: () => new Promise(() => {}) }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + clock, + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + signals.emit("SIGINT"); + await settle(); + + clock.advance(RELEASE_TIMEOUT_MS); + + expect(await runPromise).toBe(0); + expect(output.stderr).toContain("Timed out releasing lease lse_1"); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("still releases the lease when printing the grant throws (--export-env with a bad key)", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let released: string | undefined; + const client = fakeClient({ + requestLease: () => + Promise.resolve({ + device: { + physical: false, + id: "dev_1", + driverDeviceId: "dev_1", + spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, + mode: "full", + }, + // why: A driver-supplied key that is not a shell identifier fails the command rather than + // being silently dropped -- and that throw must not cost the device. + environment: { "NOT A KEY": "value" }, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }), + releaseLease: (input) => { + released = input.leaseId; + return Promise.resolve({ leaseId: input.leaseId }); + }, + }); + const exitCode = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--export-env"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + + expect(exitCode).toBe(1); + expect(released, "an exit through a throw is still an exit, and still owes a release").toBe( + "lse_1", + ); + expect(clock.pendingTimerCount).toBe(0); + }); + + it("arms no renew timer for --detach: it prints and exits, exactly as before", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let renewed = 0; + const client = fakeClient({ + renewLease: () => { + renewed += 1; + return Promise.reject(new Error("--detach must not renew")); + }, + }); + const exitCode = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + + expect(exitCode).toBe(0); + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + expect(renewed).toBe(0); + }); +}); diff --git a/src/cli/index-auto-launch-handshake.test.ts b/src/cli/index-auto-launch-handshake.test.ts new file mode 100644 index 000000000..256a9f507 --- /dev/null +++ b/src/cli/index-auto-launch-handshake.test.ts @@ -0,0 +1,643 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createServer, type Socket } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { STARTUP_READ_LIMIT_MS } from "../core/index.js"; +import { + FakeClock, + FakeDaemonLauncher, + IpcError, + MemoryFilesystem, + MemoryIpcTransport, + NodeIpcTransport, + type IpcConnection, + type IpcConnector, + FakeSystemStats, +} from "../ports/index.js"; +import { PROTOCOL_VERSION_RANGE } from "../contract/index.js"; +import { runCli } from "./index.js"; +import { + EMPTY_STATUS, + detachedGrant, + gibibyte, + outputCapture, + realCliEnvironmentPorts, + runningDaemons, + settle, + startInMemoryDaemon, + temporaryDirectories, +} from "./test-index.js"; + +afterEach(async () => { + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("CLI: auto-launch's handshake cut short by the launched daemon (#684)", () => { + it.each([ + { closeAfter: "hello", rejectedCredential: false }, + { closeAfter: "request", rejectedCredential: false }, + { closeAfter: "hello", rejectedCredential: true }, + { closeAfter: "request", rejectedCredential: true }, + ] as const)( + "reports a daemon it launched that answers hello then dies during that start as a failed start, not a lost connection (after $closeAfter, credential rejected: $rejectedCredential)", + async ({ closeAfter, rejectedCredential }) => { + // why: this stages the daemon that loses its HTTP port race exactly as the socket shows it + // to the CLI: absent before the launch, then listening, answering `hello`, and closing on + // its own `http-start-failed` stop before the command's first real request is answered. + const dyingDaemon = new HelloThenDeathIpc(closeAfter, rejectedCredential); + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher(() => { + dyingDaemon.startListening(); + }); + const output = outputCapture({ + filesystem: new MemoryFilesystem(), + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: dyingDaemon, + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli( + ["daemon", "start", "--json", "--token", "probe"], + output.environmentWith(), + ).then((code) => { + settled = true; + return code; + }); + // why: the tick bound is generous because #671 made every module a command loads lazy, so + // one `daemon start` now passes through several more import turns than when this repro was + // first staged; the clock is fake and the loop stops the moment the command settles, so the + // extra headroom pins nothing -- it only keeps a hung command failing on this named + // assertion instead of on a test timeout. + for (let tick = 0; tick < 100 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect( + settled, + `simlock daemon start still had not settled at ${String(clock.now())}ms`, + ).toBe(true); + + // why: the staging itself, proven from the fake's side -- the socket was absent until this + // command launched the daemon and answered `hello` before it closed the connection. Without + // these the assertion below could be + // reached by a connection that never saw a daemon at all. + expect(launcher.launches).toBe(1); + expect(dyingDaemon.helloReplies).toBe(1); + expect(dyingDaemon.deaths).toBe(1); + expect(dyingDaemon.rejectedCredentials).toBe(rejectedCredential ? 1 : 0); + + expect(await exitCode).toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toMatchObject({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: expect.stringContaining("daemon"), + }, + }); + }, + ); + + it.each([false, true])( + "keeps a pre-existing daemon's hello-then-close as DAEMON_CONNECTION_LOST (credential rejected: %s)", + async (rejectedCredential) => { + const dyingDaemon = new HelloThenDeathIpc("hello", rejectedCredential); + dyingDaemon.startListening(); + const launcher = new FakeDaemonLauncher(); + const output = outputCapture({ + ...realCliEnvironmentPorts(), + ipc: dyingDaemon, + launcher, + }); + + expect( + await runCli(["daemon", "start", "--json", "--token", "probe"], output.environmentWith()), + ).toBe(1); + expect(launcher.launches).toBe(0); + expect(dyingDaemon.helloReplies).toBe(1); + expect(dyingDaemon.deaths).toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toEqual({ + error: { code: "DAEMON_CONNECTION_LOST", message: "Daemon connection closed" }, + }); + }, + ); + + it("reports a daemon that answers the startup status probe with DAEMON_STOPPING as its own start error and closes its client", async () => { + const daemon = new HelloThenDeathIpc("stopping"); + const launcher = new FakeDaemonLauncher(() => daemon.startListening(), { + exit: { code: 1, signal: null }, + output: `${JSON.stringify({ message: "Daemon failed to start", fields: { message: "listen EADDRINUSE: address already in use :::7777" } })}\n`, + }); + const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, launcher }); + expect(await runCli(["daemon", "start", "--token", "probe"], output.environmentWith())).toBe(1); + expect(launcher.launches).toBe(1); + expect(daemon.helloReplies).toBe(1); + expect(daemon.clientCloses).toBe(1); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: "The daemon could not start: listen EADDRINUSE: address already in use :::7777", + }, + }); + }); + + it("daemon start's wait for health to leave starting ends past the daemon's startup-read cap with DAEMON_STARTUP_FAILED, leaving the daemon running", async () => { + const daemon = new HelloThenDeathIpc("starting"); + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher(() => daemon.startListening()); + const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, clock, launcher }); + let settled = false; + const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 3_000 && !settled; tick += 1) { + clock.advance(50); + await settle(); + } + expect(settled, `daemon start still waiting at ${String(clock.now())}ms`).toBe(true); + expect( + clock.now(), + "the wait outlasts the daemon's own startup read by 30 s", + ).toBeGreaterThanOrEqual(STARTUP_READ_LIMIT_MS + 30_000); + expect(await run).toBe(1); + expect(daemon.frameTypes, "the CLI sends the daemon no stop").not.toContain("daemon.stop"); + expect(daemon.deaths, "the daemon is not stopped by the CLI").toBe(0); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: expect.stringContaining("still starting"), + }, + }); + }); + + it("bounds the startup status probe by the original launch deadline and closes its client", async () => { + const daemon = new HelloThenDeathIpc("silent"); + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher(() => { + clock.setTimer(19_000, () => daemon.startListening()); + }); + const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, clock, launcher }); + let settled = false; + const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 500 && !settled; tick += 1) { + clock.advance(50); + await settle(); + } + expect(settled, "silent startup probe must settle within the launch budget").toBe(true); + expect( + clock.now(), + "probe must use the budget left after the socket appeared", + ).toBeLessThanOrEqual(20_050); + expect(clock.now(), "probe must wait out the launch deadline").toBeGreaterThanOrEqual(20_000); + expect(await run).toBe(1); + expect(launcher.launches).toBe(1); + expect(daemon.helloReplies).toBe(1); + expect(daemon.clientCloses).toBe(1); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: expect.stringContaining("Timed out starting"), + }, + }); + }); + + it("fails a start with DAEMON_STARTUP_FAILED when the daemon stops reading after hello, without waiting for its side of the close", async () => { + const directory = await mkdtemp(join(tmpdir(), "simlock-wedged-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + const sockets: Socket[] = []; + let hellos = 0; + const server = createServer((socket) => { + sockets.push(socket); + let buffered = ""; + socket.on("data", (chunk) => { + buffered += chunk.toString(); + for (const line of buffered.split("\n").slice(0, -1)) { + const frame = JSON.parse(line) as { id: number; type: string }; + if (frame.type !== "hello") continue; + hellos += 1; + socket.write( + `${JSON.stringify({ + id: frame.id, + ok: true, + payload: { + protocolVersion: PROTOCOL_VERSION_RANGE.max, + daemonProtocolRange: PROTOCOL_VERSION_RANGE, + version: "0.0.0-test", + role: "admin", + principal: "test-requester", + }, + })}\n`, + ); + // why: a daemon wedged after `hello` neither answers nor ends its side of the socket. + socket.pause(); + } + buffered = buffered.slice(buffered.lastIndexOf("\n") + 1); + }); + }); + await new Promise((resolve) => server.listen(socketPath, resolve)); + try { + const real = new NodeIpcTransport(); + let launched = false; + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher(() => { + launched = true; + }); + let probeSent = false; + const ipc: IpcConnector = { + connect: async () => { + if (!launched) throw new IpcError("endpoint-not-found", "not yet", undefined); + const connection = await real.connect(socketPath); + const write = connection.write.bind(connection); + connection.write = async (contents) => { + await write(contents); + if ((JSON.parse(contents.trim()) as { type: string }).type === "status.get") { + probeSent = true; + } + }; + return connection; + }, + }; + const output = outputCapture({ ...realCliEnvironmentPorts(), ipc, clock, launcher }); + let settled = false; + const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 200 && hellos === 0; tick += 1) { + clock.advance(50); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(hellos).toBe(1); + // why: the server observing hello does not mean the client has read its reply and armed + // the probe deadline; advance the fake clock only after the client sends that probe. + await expect.poll(() => probeSent).toBe(true); + clock.advance(20_000); + await expect.poll(() => settled, { timeout: 2_000 }).toBe(true); + expect(await run).toBe(1); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { code: "DAEMON_STARTUP_FAILED" }, + }); + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it("exits 1 for a socket closed before hello and reports DAEMON_STARTUP_FAILED only after launch", async () => { + for (const scenario of [ + { launch: false, afterExit: false, code: "DAEMON_CONNECTION_LOST", connects: 1 }, + { launch: true, afterExit: false, code: "DAEMON_STARTUP_FAILED", connects: 2 }, + { launch: true, afterExit: true, code: "DAEMON_STARTUP_FAILED", connects: 3 }, + ]) { + const directory = await mkdtemp(join(tmpdir(), "simlock-closed-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + let accepted = 0; + // why: the daemon that refuses its history accepts the CLI's connection, then stops and + // closes it; the CLI is still reading `admin.token` then, so the close lands before `hello`. + const server = createServer((socket) => { + accepted += 1; + socket.destroy(); + }); + await new Promise((resolve) => server.listen(socketPath, resolve)); + try { + const real = new NodeIpcTransport(); + let launched = !scenario.launch; + const clock = new FakeClock(0); + const launcher = new FakeDaemonLauncher( + () => { + launched = true; + }, + { exit: { code: 1, signal: null } }, + ); + let connects = 0; + const ipc: IpcConnector = { + connect: async () => { + connects += 1; + if (!launched || (scenario.afterExit && connects === 2)) { + throw new IpcError("endpoint-not-found", "not yet", undefined); + } + const connection = await real.connect(socketPath); + await expect.poll(() => connection.closed).toBe(true); + return connection; + }, + }; + const output = outputCapture({ ...realCliEnvironmentPorts(), ipc, clock, launcher }); + let settled = false; + const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 100 && !settled; tick += 1) { + clock.advance(50); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + expect(launcher.launches).toBe(scenario.launch ? 1 : 0); + expect(accepted).toBe(1); + expect(connects).toBe(scenario.connects); + expect( + settled, + `simlock daemon start still had not settled at ${String(clock.now())}ms`, + ).toBe(true); + expect(await run).toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toMatchObject({ + error: { code: scenario.code }, + }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + } + }); + + it.each([ + { operation: "pending lease", health: "running" }, + { operation: "pending lease", health: "starting" }, + { operation: "held lease", health: "running" }, + { operation: "in-flight request", health: "running" }, + ] as const)( + "keeps an auto-launched daemon's death after its startup status probe as DAEMON_CONNECTION_LOST during $operation (status: $health)", + async ({ operation, health }) => { + const ipc = new MemoryIpcTransport(); + const clock = new FakeClock(0); + const requests: string[] = []; + const replies: string[] = []; + const connections: IpcConnection[] = []; + const launcher = new FakeDaemonLauncher(async () => { + await ipc.listen("/simlock/daemon.sock", (connection) => { + connections.push(connection); + connection.onData((contents) => { + const frame = JSON.parse(contents) as { id: number; type: string }; + requests.push(frame.type); + const payload = + frame.type === "hello" + ? { + protocolVersion: PROTOCOL_VERSION_RANGE.max, + daemonProtocolRange: PROTOCOL_VERSION_RANGE, + version: "0.0.0-test", + role: "admin", + principal: "test-requester", + } + : frame.type === "lease.request" && operation !== "pending lease" + ? detachedGrant + : frame.type === "status.get" + ? { ...EMPTY_STATUS, daemon: { ...EMPTY_STATUS.daemon, health } } + : undefined; + if (payload === undefined) return; + replies.push(frame.type); + void connection.write(`${JSON.stringify({ id: frame.id, ok: true, payload })}\n`); + }); + }); + }); + const output = outputCapture({ + ...realCliEnvironmentPorts(), + ipc, + clock, + launcher, + }); + let settled = false; + const runPromise = runCli( + operation !== "in-flight request" + ? ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--token", "probe"] + : ["simctl", "list", "--token", "probe"], + output.environmentWith(), + ).then((code) => { + settled = true; + return code; + }); + const ready = (): boolean => + operation === "held lease" + ? output.stdout !== "" + : requests.includes( + operation === "pending lease" ? "lease.request" : "driver.passthrough", + ); + for (let tick = 0; tick < 100 && !ready() && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(ready()).toBe(true); + expect(settled).toBe(false); + expect(launcher.launches).toBe(1); + expect(connections).toHaveLength(1); + expect(requests).toContain( + operation === "in-flight request" ? "driver.passthrough" : "lease.request", + ); + if (operation === "pending lease") expect(replies).not.toContain("lease.request"); + await connections[0]!.close(); + expect(await runPromise).toBe(1); + expect(JSON.parse(output.stderr)).toMatchObject({ + error: { code: "DAEMON_CONNECTION_LOST" }, + }); + expect(replies).toContain("status.get"); + if (operation === "held lease") { + expect(JSON.parse(output.stdout)).toMatchObject({ lease: detachedGrant.lease }); + expect(JSON.parse(output.stderr).error.message).toContain(detachedGrant.lease.id); + expect(requests).toContain("lease.request"); + } else { + expect(output.stdout).toBe(""); + } + }, + ); + + it("preserves the daemon's refusal of the first auto-launched request", async () => { + const filesystem = new MemoryFilesystem(); + const ipc = new MemoryIpcTransport(); + const launcher = new FakeDaemonLauncher(async () => { + await startInMemoryDaemon({ + filesystem, + ipcTransport: ipc, + socketPath: "/simlock/daemon.sock", + adminTokenPath: "/simlock/admin.token", + }); + }); + const output = outputCapture({ + ...realCliEnvironmentPorts(), + filesystem, + ipc, + launcher, + }); + + expect( + await runCli(["lease", "--platform", "android", "--detach"], output.environmentWith()), + ).toBe(12); + expect(launcher.launches).toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "NO_DRIVER" } }); + }); +}); + +class HelloThenDeathIpc implements IpcConnector { + helloReplies = 0; + deaths = 0; + readonly frameTypes: string[] = []; + #listening = false; + + rejectedCredentials = 0; + clientCloses = 0; + + constructor( + readonly closeAfter: "hello" | "request" | "silent" | "stopping" | "starting" = "hello", + readonly rejectCredential = false, + ) {} + + startListening(): void { + this.#listening = true; + } + + async connect(): Promise { + if (!this.#listening) { + throw new IpcError("endpoint-not-found", "No IPC endpoint yet", undefined); + } + return new HelloThenDeathConnection(this); + } +} + +class HelloThenDeathConnection implements IpcConnection { + readonly #dataListeners = new Set<(chunk: string) => void>(); + readonly #closeListeners = new Set<() => void>(); + #closed = false; + + constructor(private readonly daemon: HelloThenDeathIpc) {} + + get closed(): boolean { + return this.#closed; + } + + onData(listener: (chunk: string) => void): () => void { + this.#dataListeners.add(listener); + return () => this.#dataListeners.delete(listener); + } + + onClose(listener: () => void): () => void { + this.#closeListeners.add(listener); + return () => this.#closeListeners.delete(listener); + } + + onError(): () => void { + return () => {}; + } + + async write(contents: string): Promise { + if (this.#closed) return; + let answeredHello = false; + for (const line of contents.split("\n")) { + answeredHello = this.#receiveFrame(line) || answeredHello; + if (this.#closed) return; + } + if ( + this.daemon.closeAfter === "hello" || + (!answeredHello && this.daemon.closeAfter === "request") + ) + this.#shutDown(true); + } + + #receiveFrame(line: string): boolean { + if (line.trim() === "") return false; + const frame = JSON.parse(line) as { + id?: unknown; + type?: unknown; + payload?: { credential?: string }; + }; + if (typeof frame.type === "string") this.daemon.frameTypes.push(frame.type); + if (frame.type === "status.get") this.#answerStatus(frame.id); + if (frame.type !== "hello" || typeof frame.id !== "number") return false; + this.#answerHello(frame.id, frame.payload?.credential); + return true; + } + + #answerStatus(id: unknown): void { + const { closeAfter } = this.daemon; + if (closeAfter !== "stopping" && closeAfter !== "starting") return; + const reply = + closeAfter === "stopping" + ? { + id, + ok: false, + error: { code: "DAEMON_STOPPING", message: "Daemon is stopping", details: {} }, + } + : { + id, + ok: true, + payload: { + daemon: { + health: "starting", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, + }, + }; + for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); + } + + #answerHello(id: number, credential: string | undefined): void { + if (this.daemon.rejectCredential && credential !== undefined) { + this.daemon.rejectedCredentials += 1; + const reply = { + id: id, + ok: false, + error: { + code: "ADMIN_AUTHENTICATION_FAILED", + message: "Rejected credential", + details: {}, + }, + }; + for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); + this.#shutDown(false); + return; + } + this.daemon.helloReplies += 1; + const reply = { + id: id, + ok: true, + payload: { + protocolVersion: PROTOCOL_VERSION_RANGE.max, + daemonProtocolRange: PROTOCOL_VERSION_RANGE, + version: "0.0.0-test", + role: "admin", + principal: "test-requester", + }, + }; + for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); + } + + async close(): Promise { + this.daemon.clientCloses += 1; + this.#shutDown(false); + } + + #shutDown(killedByDaemon: boolean): void { + if (this.#closed) return; + this.#closed = true; + if (killedByDaemon) this.daemon.deaths += 1; + for (const listener of this.#closeListeners) listener(); + } +} diff --git a/src/cli/index-baseline-alerts-warm.test.ts b/src/cli/index-baseline-alerts-warm.test.ts new file mode 100644 index 000000000..ed4d9e8a2 --- /dev/null +++ b/src/cli/index-baseline-alerts-warm.test.ts @@ -0,0 +1,531 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import type { SimlockAdminClient } from "../simlock-client/client.js"; +import { runCli } from "./index.js"; +import { fakeClient, outputCapture } from "./test-index.js"; + +describe("CLI: alerts test (ADR 0035 §1)", () => { + const USAGE = "Usage: simlock alerts test [--json]\n"; + const errorLine = (stderr: string): unknown => JSON.parse(stderr.trim().split("\n").at(-1) ?? ""); + + async function runAlerts(argv: string[], overrides: Partial = {}) { + const output = outputCapture(); + const inputs: unknown[] = []; + let connected = false; + const code = await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient({ + testAlertDestination: (input) => { + inputs.push(input); + return Promise.resolve({ destination: input.destination, status: 204 }); + }, + ...overrides, + }); + }, + }), + ); + return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; + } + + it("simlock alerts test prints the sent line, --json prints the output, a missing name prints the Expected line, simlock alerts and --help print the usage, and an unknown subcommand is a usage error", async () => { + const sent = await runAlerts(["alerts", "test", "ops-slack"]); + expect([sent.code, sent.stdout, sent.inputs]).toEqual([ + 0, + "Sent a test message to ops-slack: HTTP 204\n", + [{ destination: "ops-slack" }], + ]); + + const json = await runAlerts(["alerts", "test", "ops-slack", "--json"]); + expect([json.code, json.stdout]).toEqual([0, '{"destination":"ops-slack","status":204}\n']); + + for (const argv of [["alerts"], ["alerts", "--help"], ["alerts", "test", "--help"]]) { + const usage = await runAlerts(argv); + expect([usage.code, usage.stdout, usage.stderr, usage.connected]).toEqual([ + 0, + USAGE, + "", + false, + ]); + } + + for (const [argv, message] of [ + [["alerts", "test"], "Expected destination (run `simlock --help` for usage)"], + [["alerts", "foo"], "Unknown alerts command: foo (run `simlock --help` for usage)"], + ] as const) { + const refused = await runAlerts([...argv]); + expect(refused.code).toBe(2); + expect(refused.connected).toBe(false); + expect(JSON.parse(refused.stderr)).toEqual({ error: { code: "USAGE", message } }); + } + }); + + it("simlock alerts test exits 1 on ALERT_SEND_FAILED and 12 on UNKNOWN_ALERT_DESTINATION with the error line", async () => { + const failed = await runAlerts(["alerts", "test", "ops-slack"], { + testAlertDestination: () => + Promise.reject( + new SimlockError("ALERT_SEND_FAILED", "transport", "ops-slack answered HTTP 404", { + destination: "ops-slack", + status: 404, + }), + ), + }); + expect([failed.code, failed.stdout, errorLine(failed.stderr)]).toEqual([ + 1, + "", + { error: { code: "ALERT_SEND_FAILED", message: "ops-slack answered HTTP 404" } }, + ]); + + const unknown = await runAlerts(["alerts", "test", "nope"], { + testAlertDestination: () => + Promise.reject( + new SimlockError("UNKNOWN_ALERT_DESTINATION", "domain", "No alert destination named", { + destination: "nope", + }), + ), + }); + expect([unknown.code, errorLine(unknown.stderr)]).toEqual([ + 12, + { error: { code: "UNKNOWN_ALERT_DESTINATION", message: "No alert destination named" } }, + ]); + }); +}); + +describe("CLI: baseline save", () => { + const saved = (overrides: Record = {}) => ({ + baseline: { + mode: "full", + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android", + savedAt: "2026-11-02T13:40:00.000Z", + version: 1, + ...overrides, + }, + leaseId: "L1", + }); + + async function runSave(argv: string[], answer = saved()) { + const output = outputCapture(); + const inputs: unknown[] = []; + let connected = false; + const code = await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient({ + saveBaseline: (input) => { + inputs.push(input); + return Promise.resolve(answer as never); + }, + }); + }, + }), + ); + return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; + } + + it("simlock baseline save sends the name, lease and note and prints the baseline's version, spec and size with the lease it ended, and refuses a missing --lease as USAGE", async () => { + const android = await runSave( + ["baseline", "save", "checkout", "--lease", "L1", "--note", "app 4.2, test user"], + saved({ imageTag: "google_apis", sizeBytes: 4_800_000_000 }), + ); + expect([android.code, android.stdout, android.inputs]).toEqual([ + 0, + "Saved baseline checkout v1: android Pixel 9, Android 36 (google_apis), full, 4.8 GB. Lease L1 ended.\n", + [{ leaseId: "L1", name: "checkout", note: "app 4.2, test user" }], + ]); + + const ios = await runSave( + ["baseline", "save", "login", "--lease", "L5"], + saved({ + mode: "slim", + model: "iPhone 17", + name: "login", + osVersion: "27.0", + platform: "ios", + sizeBytes: 3_100_000_000, + }), + ); + expect(ios.stdout).toBe( + "Saved baseline login v1: ios iPhone 17, iOS 27.0, slim, 3.1 GB. Lease L1 ended.\n", + ); + expect(ios.inputs).toEqual([{ leaseId: "L5", name: "login" }]); + + const small = await runSave( + ["baseline", "save", "tiny", "--lease", "L1"], + saved({ sizeBytes: 250_000_000 }), + ); + expect(small.stdout).toContain("full, 250.0 MB. Lease L1 ended."); + const uncounted = await runSave(["baseline", "save", "tiny", "--lease", "L1"], saved()); + expect(uncounted.stdout).toBe( + "Saved baseline checkout v1: android Pixel 9, Android 36, full. Lease L1 ended.\n", + ); + + const missing = await runSave(["baseline", "save", "checkout"]); + expect([missing.code, missing.connected, missing.stdout]).toEqual([2, false, ""]); + expect(JSON.parse(missing.stderr)).toMatchObject({ error: { code: "USAGE" } }); + + const twoNames = await runSave(["baseline", "save", "checkout", "other", "--lease", "L1"]); + expect([twoNames.code, twoNames.connected, twoNames.inputs]).toEqual([2, false, []]); + }); +}); + +describe("CLI: baseline list", () => { + const checkout = { + imageTag: "google_apis", + mode: "full", + model: "Pixel 9", + name: "checkout", + osVersion: "36", + platform: "android", + savedAt: new Date(2026, 10, 2, 13, 40).toISOString(), + sizeBytes: 4_900_000_000, + version: 2, + } as const; + const login = { + mode: "slim", + model: "iPhone 17", + name: "login", + note: "signed in", + osVersion: "27.0", + platform: "ios", + savedAt: new Date(2026, 10, 1, 9, 12).toISOString(), + staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + version: 1, + } as const; + + async function runList(argv: string[], baselines: readonly (typeof checkout | typeof login)[]) { + const output = outputCapture(); + const code = await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => + fakeClient({ listBaselines: () => Promise.resolve({ baselines: [...baselines] }) }), + }), + ); + return { code, stderr: output.stderr, stdout: output.stdout }; + } + + it("simlock baseline list prints a table of each baseline with a - for an uncounted size and its stale reason under the row, --json the baselines, and says when there are none", async () => { + const table = await runList(["baseline", "list"], [checkout, login]); + expect([table.code, table.stdout]).toEqual([ + 0, + [ + "NAME VERSION PLATFORM MODEL OS MODE IMAGE SIZE SAVED NOTE", + "checkout 2 android Pixel 9 36 full google_apis 4.9 GB 2026-11-02 13:40", + "login 1 ios iPhone 17 27.0 slim - - 2026-11-01 09:12 signed in", + " stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + "", + ].join("\n"), + ]); + + const json = await runList(["baseline", "list", "--json"], [checkout, login]); + expect([json.code, JSON.parse(json.stdout)]).toEqual([0, { baselines: [checkout, login] }]); + + const none = await runList(["baseline", "list"], []); + expect([none.code, none.stdout]).toEqual([0, "No baselines on this worker.\n"]); + }); +}); + +describe("CLI: baselines through a gateway (#1208)", () => { + const GATEWAY = { + daemonHome: { dataDirectory: "/simlock/gateway", gatewayPresent: true, role: "gateway" }, + } as const; + const checkout = { + imageTag: "google_apis", + mode: "full", + model: "Pixel 9", + name: "checkout", + note: "app 4.2, test user", + osVersion: "36", + platform: "android", + savedAt: new Date(2026, 10, 2, 13, 10).toISOString(), + sizeBytes: 4_800_000_000, + version: 2, + workerId: "wrk_a", + } as const; + const smoke = { + mode: "slim", + model: "iPhone 17", + name: "smoke", + osVersion: "27.0", + platform: "ios", + savedAt: new Date(2026, 9, 30, 9, 2).toISOString(), + staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + version: 1, + workerId: "wrk_a", + } as const; + + it("simlock worker list prints a worker's Baselines line, its image tag in brackets and a stale one marked, none for a worker without any, and no line for one that has not read them", async () => { + const output = outputCapture(); + const worker = { connection: "connected", drained: false, lastSeenAt: 1 } as const; + await runCli( + ["worker", "list"], + output.environmentWith({ + clock: new FakeClock(1_000), + connectAdmin: async () => + fakeClient({ + listWorkers: () => + Promise.resolve({ + workers: [ + { ...worker, baselines: [checkout, smoke], id: "wrk_a" }, + { ...worker, baselines: [], id: "wrk_b" }, + { ...worker, health: "starting", id: "wrk_c" }, + ], + } as never), + }), + }), + ); + + expect(output.stdout.split("\n").filter((line) => /^\s+Baselines:/.test(line))).toEqual([ + " Baselines: checkout v2: android Pixel 9, Android 36 (google_apis), full; smoke v1: ios iPhone 17, iOS 27.0, slim, stale", + " Baselines: none", + ]); + }); + + it("simlock --gateway baseline list puts the worker first in each row, and says when no worker holds a baseline", async () => { + const run = async (baselines: readonly (typeof checkout | typeof smoke)[]) => { + const output = outputCapture(); + await runCli( + ["--gateway", "baseline", "list"], + output.environmentWith({ + ...GATEWAY, + connectAdmin: async () => + fakeClient({ listBaselines: () => Promise.resolve({ baselines: [...baselines] }) }), + }), + ); + return output.stdout; + }; + + expect(await run([checkout, smoke])).toBe( + [ + "WORKER NAME VERSION PLATFORM MODEL OS MODE IMAGE SIZE SAVED NOTE", + "wrk_a checkout 2 android Pixel 9 36 full google_apis 4.8 GB 2026-11-02 13:10 app 4.2, test user", + "wrk_a smoke 1 ios iPhone 17 27.0 slim - - 2026-10-30 09:02", + " stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", + "", + ].join("\n"), + ); + expect(await run([])).toBe("No baselines on any worker.\n"); + }); + + it("simlock --gateway baseline remove --worker sends the worker, asks and prints ending in from , and without --worker asks nothing and sends the remove", async () => { + const run = async (argv: string[]) => { + const output = outputCapture(); + const sent: unknown[] = []; + const questions: string[] = []; + await runCli( + ["--gateway", ...argv], + output.environmentWith({ + ...GATEWAY, + confirm: async (question) => { + questions.push(question); + return true; + }, + connectAdmin: async () => + fakeClient({ + listBaselines: () => + Promise.resolve({ + baselines: [{ ...smoke, version: 3, workerId: "wrk_b" }, smoke], + }), + removeBaseline: (input) => { + sent.push(input); + return Promise.resolve({ deletedDevices: [], name: input.name, versions: [1] }); + }, + }), + }), + ); + return { questions, sent, stdout: output.stdout }; + }; + + expect(await run(["baseline", "remove", "smoke", "--worker", "wrk_a"])).toEqual({ + questions: ["Remove baseline smoke v1 from wrk_a? [y/N] "], + sent: [{ name: "smoke", workerId: "wrk_a" }], + stdout: "Removed baseline smoke v1 from wrk_a.\n", + }); + expect(await run(["baseline", "remove", "smoke"])).toMatchObject({ + questions: [], + sent: [{ name: "smoke" }], + }); + }); +}); + +describe("CLI: baseline remove", () => { + const login = { + mode: "slim", + model: "iPhone 17", + name: "login", + osVersion: "27.0", + platform: "ios", + savedAt: new Date(2026, 10, 1, 9, 12).toISOString(), + version: 1, + } as const; + + async function runRemove( + argv: string[], + options: { confirm?: ((question: string) => Promise) | undefined; deleted?: string[] }, + ) { + const output = outputCapture(); + const removed: unknown[] = []; + const code = await runCli( + argv, + output.environmentWith({ + confirm: options.confirm, + connectAdmin: async () => + fakeClient({ + listBaselines: () => Promise.resolve({ baselines: [login] }), + removeBaseline: (input) => { + removed.push(input); + return Promise.resolve({ + deletedDevices: options.deleted ?? [], + name: input.name, + versions: [1], + }); + }, + }), + }), + ); + return { code, removed, stderr: output.stderr, stdout: output.stdout }; + } + + it("simlock baseline remove asks Remove baseline v? first, removes with --yes or an answered yes and prints how many idle devices it deleted, and refuses with USAGE when there is no terminal or the answer is no", async () => { + const questions: string[] = []; + const asked = await runRemove(["baseline", "remove", "login"], { + confirm: async (question) => { + questions.push(question); + return true; + }, + }); + expect([asked.code, asked.stdout, asked.removed, questions]).toEqual([ + 0, + "Removed baseline login v1.\n", + [{ name: "login" }], + ["Remove baseline login v1? [y/N] "], + ]); + + const yes = await runRemove(["baseline", "remove", "login", "--yes"], { + confirm: async () => { + throw new Error("--yes must not ask"); + }, + deleted: ["dev_1"], + }); + expect(yes.stdout).toBe("Removed baseline login v1. Deleted 1 idle device made from it.\n"); + const many = await runRemove(["baseline", "remove", "login", "--yes"], { + deleted: ["dev_1", "dev_2"], + }); + expect(many.stdout).toBe("Removed baseline login v1. Deleted 2 idle devices made from it.\n"); + + for (const confirm of [undefined, async () => false]) { + const refused = await runRemove(["baseline", "remove", "login"], { confirm }); + expect([refused.code, refused.removed, refused.stdout]).toEqual([2, [], ""]); + expect(JSON.parse(refused.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: "baseline remove requires confirmation or --yes" }, + }); + } + }); +}); + +describe("CLI: warm forget", () => { + const USAGE = + "Usage: simlock warm forget [] [--mode ] [--image-tag ] [--json]\n"; + const pixel = { model: "Pixel", mode: "full", osVersion: "36", platform: "android" } as const; + + const checkoutKind = { ...pixel, baseline: "checkout" } as const; + + async function runWarm( + argv: string[], + forgotten: (typeof pixel | typeof checkoutKind)[] = [pixel], + ) { + const output = outputCapture(); + const inputs: unknown[] = []; + let connected = false; + const code = await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient({ + forgetWarmKind: (input) => { + inputs.push(input); + return Promise.resolve({ forgotten }); + }, + }); + }, + }), + ); + return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; + } + + it("simlock warm forget sends the platform, model, OS, mode and image tag it was given and prints each kind forgotten, or that none was learned", async () => { + const forgot = await runWarm(["warm", "forget", "android", "Pixel"]); + expect([forgot.code, forgot.stdout, forgot.inputs]).toEqual([ + 0, + "Forgot android Pixel 36 (full).\nIts idle devices are no longer kept warm: the idle rules shut them down and delete them, unless a configured warmPool.targets entry keeps them.\n", + [{ model: "Pixel", platform: "android" }], + ]); + + const narrowed = await runWarm( + ["warm", "forget", "android", "Pixel", "36", "--mode", "full", "--image-tag", "google_apis"], + [], + ); + expect(narrowed.stdout).toBe( + "The warm pool has learned no kind matching android Pixel 36 (full, google_apis).\n", + ); + expect(narrowed.inputs).toEqual([ + { + imageTag: "google_apis", + mode: "full", + model: "Pixel", + osVersion: "36", + platform: "android", + }, + ]); + + const named = await runWarm(["warm", "forget", "android", "Pixel"], [checkoutKind]); + expect(named.stdout).toContain("Forgot android Pixel 36 (full, baseline checkout).\n"); + + const none = await runWarm(["warm", "forget", "ios", "iPhone 16"], []); + expect([none.code, none.stdout]).toEqual([ + 0, + "The warm pool has learned no kind matching ios iPhone 16.\n", + ]); + + const json = await runWarm(["warm", "forget", "android", "Pixel", "--json"]); + expect([json.code, JSON.parse(json.stdout)]).toEqual([0, { forgotten: [pixel] }]); + }); + + it("simlock warm and --help print the usage without connecting; a bad platform, a missing model, extra words, a bad --mode and an unknown subcommand are usage errors", async () => { + for (const argv of [["warm"], ["warm", "--help"], ["warm", "forget", "--help"]]) { + const usage = await runWarm(argv); + expect([usage.code, usage.stdout, usage.stderr, usage.connected]).toEqual([ + 0, + USAGE, + "", + false, + ]); + } + + for (const [argv, message] of [ + [["warm", "forget", "tvos", "Pixel"], "warm forget needs a platform first: ios or android"], + [["warm", "forget", "android"], "warm forget takes and an optional "], + [ + ["warm", "forget", "android", "Pixel", "36", "extra"], + "warm forget takes and an optional ", + ], + [["warm", "forget", "android", "Pixel", "--mode", "tiny"], "--mode is slim or full"], + [["warm", "boot"], "Unknown warm command: boot"], + ] as const) { + const refused = await runWarm([...argv]); + expect([refused.code, refused.connected]).toEqual([2, false]); + expect(JSON.parse(refused.stderr)).toEqual({ + error: { code: "USAGE", message: `${message} (run \`simlock --help\` for usage)` }, + }); + } + }); +}); diff --git a/src/cli/index-catalog-components.test.ts b/src/cli/index-catalog-components.test.ts new file mode 100644 index 000000000..6fa87eb6a --- /dev/null +++ b/src/cli/index-catalog-components.test.ts @@ -0,0 +1,564 @@ +import { describe, expect, it } from "vitest"; +import { SimlockError } from "../contract/index.js"; +import { runCli } from "./index.js"; +import { fakeClient, outputCapture } from "./test-index.js"; + +describe("CLI: catalog", () => { + it("prints a model named after an Object.prototype member with no paired runtime", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getCatalog: () => + Promise.resolve({ + platforms: [ + { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: {}, + models: ["constructor"], + platform: "ios", + runtimes: [], + }, + ], + }), + }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toContain(" (no class):\n constructor: (no paired runtime)\n"); + }); + + it("prints (none) for a platform with no models", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getCatalog: () => + Promise.resolve({ + platforms: [ + { + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: {}, + models: [], + platform: "android", + runtimes: ["35"], + }, + ], + }), + }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toBe( + ["Platform: android", " Runtimes: 35 (default: (none))", " Models: (none)", ""].join("\n"), + ); + }); + + it("prints each model with the runtimes it pairs with", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getCatalog: () => + Promise.resolve({ + platforms: [ + { + defaultRuntime: "26.0", + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: { + "iPhone 16": ["18.4", "26.0"], + "iPhone 8": [], + "iPhone XS": ["18.4"], + }, + models: ["iPhone 16", "iPhone XS", "iPhone 8"], + platform: "ios", + runtimes: ["18.4", "26.0"], + }, + ], + }), + }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toBe( + [ + "Platform: ios", + " Runtimes: 18.4, 26.0 (default: 26.0)", + " Models:", + " (no class):", + " iPhone 16: 18.4, 26.0", + " iPhone XS: 18.4", + " iPhone 8: (no paired runtime)", + "", + ].join("\n"), + ); + }); + + it("prints each model under its class in the order of the enum, and an unclassed model under (no class)", async () => { + const catalog = { + defaultRuntime: "26.0", + classDefaults: { phone: "iPhone 17", tablet: "iPad Pro" }, + modelAliases: { "iPhone 17": ["iphone-17", "i17"] }, + modelClasses: { + "Apple TV 4K": "tv" as const, + "Apple Watch Series 11 (46mm)": "watch" as const, + "iPad Pro": "tablet" as const, + "iPhone 17": "phone" as const, + "iPhone 16": "phone" as const, + }, + modelRuntimes: { + "Apple TV 4K": ["26.0"], + "Apple Watch Series 11 (46mm)": ["26.0"], + "iPad Pro": ["26.0"], + "iPhone 16": ["26.0"], + "iPhone 17": ["26.0"], + Mystery: [], + }, + models: [ + "Mystery", + "Apple Watch Series 11 (46mm)", + "iPhone 17", + "Apple TV 4K", + "iPad Pro", + "iPhone 16", + ], + platform: "ios" as const, + runtimes: ["26.0"], + customModels: ["iPhone 16"], + }; + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ getCatalog: () => Promise.resolve({ platforms: [catalog] }) }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toBe( + [ + "Platform: ios", + " Runtimes: 26.0 (default: 26.0)", + " Models:", + " phone (default: iPhone 17):", + " iPhone 17: 26.0", + " Other names: iphone-17, i17", + " iPhone 16 (custom): 26.0", + " tablet (default: iPad Pro):", + " iPad Pro: 26.0", + " watch (default: none):", + " Apple Watch Series 11 (46mm): 26.0", + " tv (default: none):", + " Apple TV 4K: 26.0", + " (no class):", + " Mystery: (no paired runtime)", + "", + ].join("\n"), + ); + }); + + it.each([ + { + field: "classDefaults", + catalog: { + classDefaults: { phone: "Pixel 9", tv: "Not Even Listed" }, + modelAliases: {}, + modelClasses: {}, + modelRuntimes: {}, + models: [], + platform: "android" as const, + runtimes: [], + }, + }, + { + field: "modelClasses", + catalog: { + modelAliases: {}, + classDefaults: {}, + modelClasses: { "Pixel 8": "phone" as const, "Odd One": "tv" as const }, + modelRuntimes: {}, + models: ["Pixel 8"], + platform: "android" as const, + runtimes: [], + }, + }, + ])("carries $field in --json as received", async ({ catalog }) => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ getCatalog: () => Promise.resolve({ platforms: [catalog] }) }), + }); + + await expect(runCli(["catalog", "--json"], environment)).resolves.toBe(0); + + expect(JSON.parse(output.stdout)).toEqual({ platforms: [catalog] }); + }); + + it("prints (none) for a platform that has images but none installed", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getCatalog: () => + Promise.resolve({ + platforms: [ + { + images: [], + modelAliases: {}, + classDefaults: {}, + modelClasses: {}, + modelRuntimes: {}, + models: [], + platform: "android", + runtimes: [], + }, + ], + }), + }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toBe( + [ + "Platform: android", + " Runtimes: (none) (default: (none))", + " Models: (none)", + " Images: (none)", + "", + ].join("\n"), + ); + }); + + it("prints a model's other names and each image", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getCatalog: () => + Promise.resolve({ + platforms: [ + { + defaultRuntime: "35", + images: [ + { abi: "x86_64", runtime: "34", tag: "default" }, + { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, + ], + modelAliases: { "Pixel 8": ["pixel_8"] }, + classDefaults: {}, + modelClasses: { "Pixel 8": "phone" }, + modelRuntimes: { "My Tablet": ["34", "35"], "Pixel 8": ["34", "35"] }, + models: ["My Tablet", "Pixel 8"], + platform: "android", + runtimes: ["34", "35"], + }, + ], + }), + }), + }); + + await expect(runCli(["catalog"], environment)).resolves.toBe(0); + + expect(output.stdout).toBe( + [ + "Platform: android", + " Runtimes: 34, 35 (default: 35)", + " Models:", + " phone (default: none):", + " Pixel 8: 34, 35", + " Other names: pixel_8", + " (no class):", + " My Tablet: 34, 35", + " Images (runtime, tag, ABI):", + " 34 default x86_64", + " 35 google_apis arm64-v8a", + "", + ].join("\n"), + ); + }); + + it("exits 12 when the daemon refuses with DOWNLOADS_DISABLED", async () => { + const output = outputCapture(); + const client = fakeClient({ + installComponent: () => + Promise.reject( + new SimlockError("DOWNLOADS_DISABLED", "domain", "downloads.policy is never", { + policy: "never", + }), + ), + }); + + await expect( + runCli( + ["component", "install", "ios", "26.4"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(12); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "DOWNLOADS_DISABLED", message: expect.any(String) }, + }); + }); + + it.each([ + ["a missing version", ["component", "install", "ios"]], + ["an extra argument", ["component", "install", "ios", "26.4", "26.5"]], + ["an unknown platform", ["component", "install", "tvos", "26.4"]], + ["--json", ["component", "install", "ios", "26.4", "--json"]], + ["an unknown subcommand", ["component", "uninstall", "ios", "26.4"]], + ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { + const output = outputCapture(); + let installs = 0; + const client = fakeClient({ + installComponent: (input) => { + installs += 1; + return Promise.resolve({ ...input, component: input.version, outcome: "installed" }); + }, + }); + + await expect( + runCli(argv, output.environmentWith({ connectAdmin: async () => client })), + ).resolves.toBe(2); + expect(installs).toBe(0); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ + error: { code: "USAGE" }, + }); + }); + + it.each([[["component"]], [["component", "--help"]], [["component", "install", "--help"]]])( + "prints the command's usage for %j without connecting to the daemon", + async (argv) => { + const output = outputCapture(); + let connected = false; + + await expect( + runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(0); + expect(output.stdout).toBe( + "Usage: simlock component install [--worker ... | --all-workers]\n" + + " simlock component list [--platform ]\n" + + " simlock component remove [--dry-run] [--yes]\n", + ); + expect(connected).toBe(false); + }, + ); +}); + +describe("CLI: component install on a gateway's workers (ADR 0010 §7)", () => { + function progressLines(stderr: string): unknown[] { + return stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as unknown) + .filter((line) => !Object.hasOwn(line as object, "notice")); + } + + it("--all-workers prints progress with each worker's id, one result line, and exits 15 when one worker is refused", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const results = [ + { outcome: "installed" as const, version: "35", workerId: "wrk_a" }, + { + error: { code: "DOWNLOADS_DISABLED", message: "Downloads are disabled" }, + outcome: "refused" as const, + workerId: "wrk_b", + }, + ]; + const client = fakeClient({ + installComponentOnWorkers: async (input, options) => { + asked.push(input); + options?.onProgress?.({ stage: "waiting", workerId: "wrk_a" }); + options?.onProgress?.({ fraction: 0.5, stage: "downloading", workerId: "wrk_b" }); + return { results }; + }, + }); + + await expect( + runCli( + ["component", "install", "android", "35", "--all-workers"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(15); + + expect(asked).toEqual([{ platform: "android", version: "35", workers: "all" }]); + expect(progressLines(output.stderr)).toEqual([ + { stage: "waiting", workerId: "wrk_a" }, + { fraction: 0.5, stage: "downloading", workerId: "wrk_b" }, + ]); + expect(output.stdout.trim().split("\n")).toHaveLength(1); + expect(JSON.parse(output.stdout)).toEqual({ results }); + }); + + it("exits 0 when every named worker ends installed or already-installed, and names each --worker given", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const client = fakeClient({ + installComponentOnWorkers: async (input) => { + asked.push(input); + return { + results: [ + { outcome: "installed", version: "26.4", workerId: "wrk_a" }, + { outcome: "already-installed", version: "26.4", workerId: "wrk_b" }, + ], + }; + }, + }); + + await expect( + runCli( + ["component", "install", "ios", "26.4", "--worker", "wrk_a", "--worker", "wrk_b"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(0); + + expect(asked).toEqual([{ platform: "ios", version: "26.4", workers: ["wrk_a", "wrk_b"] }]); + }); + + it.each(["skipped", "failed", "unknown"] as const)( + "exits 15 when a worker ends %s", + async (outcome) => { + const output = outputCapture(); + const client = fakeClient({ + installComponentOnWorkers: async () => ({ + results: [ + { outcome: "installed", version: "35", workerId: "wrk_a" }, + { error: { code: "WORKER_UNREACHABLE", message: "gone" }, outcome, workerId: "wrk_b" }, + ], + }), + }); + + await expect( + runCli( + ["component", "install", "android", "35", "--all-workers"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(15); + }, + ); + + it("refuses --worker together with --all-workers as USAGE, without connecting", async () => { + const output = outputCapture(); + let connected = false; + + await expect( + runCli( + ["component", "install", "android", "35", "--worker", "wrk_a", "--all-workers"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(2); + expect(connected).toBe(false); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ + error: { code: "USAGE" }, + }); + }); + + it.each([ + "component install ", + "events, stats,", + "component list [--platform ]", + ])("lists the command in the usage banner: %s", async (fragment) => { + const output = outputCapture(); + await runCli(["--help"], output.environmentWith({})); + expect(output.stdout).toContain(fragment); + }); +}); + +describe("CLI: component list (ADR 0010 §8)", () => { + const listing = { + components: [ + { + devices: 2, + foreignDevices: 1, + installedAt: 1_790_864_071_200, + installedBySimlock: true, + platform: "android" as const, + sizeBytes: 4_200_000_000, + variant: "google_apis/arm64-v8a", + version: "35", + }, + ], + }; + + it("prints the daemon's component.list answer as one JSON object on stdout, and exits 0", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const client = fakeClient({ + listComponents: async (input) => { + asked.push(input); + return listing; + }, + }); + + await expect( + runCli(["component", "list"], output.environmentWith({ connectAdmin: async () => client })), + ).resolves.toBe(0); + + expect(asked).toEqual([{}]); + expect(output.stdout.trim().split("\n")).toHaveLength(1); + expect(JSON.parse(output.stdout)).toEqual(listing); + }); + + it("asks for one platform with --platform", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const client = fakeClient({ + listComponents: async (input) => { + asked.push(input); + return { components: [] }; + }, + }); + + await expect( + runCli( + ["component", "list", "--platform", "ios"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(0); + + expect(asked).toEqual([{ platform: "ios" }]); + }); + + it.each([ + ["an unknown platform", ["component", "list", "--platform", "tvos"]], + ["a positional argument", ["component", "list", "ios"]], + ["an unknown flag", ["component", "list", "--json"]], + ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { + const output = outputCapture(); + let listed = 0; + const client = fakeClient({ + listComponents: () => { + listed += 1; + return Promise.resolve({ components: [] }); + }, + }); + + await expect( + runCli(argv, output.environmentWith({ connectAdmin: async () => client })), + ).resolves.toBe(2); + expect(listed).toBe(0); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ + error: { code: "USAGE" }, + }); + }); +}); diff --git a/src/cli/index-daemon-service.test.ts b/src/cli/index-daemon-service.test.ts new file mode 100644 index 000000000..44eca101b --- /dev/null +++ b/src/cli/index-daemon-service.test.ts @@ -0,0 +1,763 @@ +import { appendFile, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + FakeClock, + FakeDaemonLauncher, + FakeServiceManager, + IpcError, + launcherFor, + LaunchdServiceManager, + ScriptedProcessRunner, + MemoryFilesystem, + MemoryIpcTransport, + serviceDefinition, + type ServiceState, + FakeSystemStats, +} from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; +import { + fakeClient, + gibibyte, + outputCapture, + runningDaemons, + settle, + startInMemoryDaemon, + temporaryDirectories, +} from "./test-index.js"; + +afterEach(async () => { + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("CLI: daemon install (#830)", () => { + const home = "/Users/me"; + const nodePath = "/opt/node/bin/node"; + const mainPath = "/opt/simlock/dist/daemon/main.js"; + + async function install(options: { + readonly argv: readonly string[]; + readonly role: "worker" | "gateway"; + readonly root: string; + readonly env?: NodeJS.ProcessEnv; + readonly linux?: boolean; + readonly user?: () => string; + }): Promise<{ + readonly exitCode: number; + readonly output: ReturnType; + readonly serviceManager: FakeServiceManager; + }> { + const { root } = options; + const dataDirectory = options.role === "gateway" ? join(root, "gateway") : root; + const filesystem = new MemoryFilesystem(); + const ipcTransport = new MemoryIpcTransport(); + const serviceManager = new FakeServiceManager({ + ...(options.linux === true ? { manager: "systemd" as const } : {}), + onInstall: async () => { + await startInMemoryDaemon({ + adminTokenPath: join(dataDirectory, "admin.token"), + filesystem, + ipcTransport, + socketPath: join(dataDirectory, "daemon.sock"), + }); + }, + }); + const output = outputCapture( + { + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: ipcTransport, + launcher: new FakeDaemonLauncher(), + daemonHome: { role: options.role, dataDirectory, gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + service: { + manager: serviceManager, + platform: options.linux === true ? "linux" : "darwin", + home: options.linux === true ? "/home/me" : home, + user: options.user ?? (() => "me"), + root, + nodePath, + mainPath, + }, + }, + options.env, + ); + const exitCode = await runCli(options.argv, output.environmentWith()); + return { exitCode, output, serviceManager }; + } + + it("an operator runs daemon install on macOS: the service is installed, the daemon answers, install prints the name, Daemon running and the hint, and --json prints the status and service", async () => { + const cases = [ + { + label: "worker, default home", + argv: ["daemon", "install"], + role: "worker" as const, + root: join(home, ".simlock"), + name: "com.callstack.simlock.worker", + args: [mainPath], + logPath: "/Users/me/.simlock/daemon.log", + }, + { + label: "gateway on a host with no gateway folder yet", + argv: ["--gateway", "daemon", "install"], + role: "gateway" as const, + root: join(home, ".simlock"), + name: "com.callstack.simlock.gateway", + args: [mainPath, "--gateway"], + logPath: "/Users/me/.simlock/gateway/daemon.log", + }, + { + label: "another SIMLOCK_HOME", + argv: ["daemon", "install"], + role: "worker" as const, + root: "/tmp/x", + name: "com.callstack.simlock.worker.2e56aa", + args: [mainPath], + logPath: "/tmp/x/daemon.log", + }, + ]; + + for (const testCase of cases) { + const { exitCode, output, serviceManager } = await install(testCase); + const label = testCase.label; + expect(output.stderr, label).toBe(""); + expect(exitCode, label).toBe(0); + expect(serviceManager.installs, label).toHaveLength(1); + expect(serviceManager.installs[0], label).toMatchObject({ + name: testCase.name, + path: `/Users/me/Library/LaunchAgents/${testCase.name}.plist`, + command: nodePath, + args: testCase.args, + logPath: testCase.logPath, + }); + expect(output.stdout.split("\n").slice(0, 2), label).toEqual([ + `Service installed: ${testCase.name} (launchd)`, + "Daemon running", + ]); + expect(output.stdout.trimEnd().split("\n").at(-1), label).toBe( + "To start after a reboot, turn on automatic login for this user.", + ); + } + + const withShell = await install({ + argv: ["daemon", "install"], + role: "worker", + root: join(home, ".simlock"), + env: { PATH: "/opt/bin:/usr/bin", JAVA_HOME: "/jdk", SIMLOCK_ADMIN_TOKEN: "secret" }, + }); + expect(withShell.exitCode, withShell.output.stderr).toBe(0); + expect(withShell.serviceManager.installs[0]?.environment).toEqual({ + SIMLOCK_HOME: join(home, ".simlock"), + PATH: "/opt/bin:/usr/bin", + JAVA_HOME: "/jdk", + }); + + const json = await install({ + argv: ["daemon", "install", "--json"], + role: "worker", + root: join(home, ".simlock"), + }); + expect(json.exitCode).toBe(0); + expect(JSON.parse(json.output.stdout)).toEqual({ + status: "installed", + service: { + state: "installed", + manager: "launchd", + name: "com.callstack.simlock.worker", + path: "/Users/me/Library/LaunchAgents/com.callstack.simlock.worker.plist", + }, + }); + }); + + it("an operator on Linux runs daemon install: the unit lands in the user config directory, and install prints systemd and the linger hint", async () => { + const cases = [ + { + label: "default config directory", + env: {}, + unit: "/home/me/.config/systemd/user/com.callstack.simlock.worker.service", + }, + { + label: "absolute XDG_CONFIG_HOME", + env: { XDG_CONFIG_HOME: "/xdg" }, + unit: "/xdg/systemd/user/com.callstack.simlock.worker.service", + }, + ]; + for (const { label, env, unit } of cases) { + const { exitCode, output, serviceManager } = await install({ + argv: ["daemon", "install"], + role: "worker", + root: "/home/me/.simlock", + env, + linux: true, + }); + + expect(output.stderr, label).toBe(""); + expect(exitCode, label).toBe(0); + expect(serviceManager.installs[0]?.path, label).toBe(unit); + expect(output.stdout.split("\n").slice(0, 2), label).toEqual([ + "Service installed: com.callstack.simlock.worker (systemd)", + "Daemon running", + ]); + expect(output.stdout.trimEnd().split("\n").at(-1), label).toBe( + "To start after a reboot without logging in, run: loginctl enable-linger me", + ); + } + + // why: a uid with no passwd entry makes the user lookup throw; install must still succeed. + const unknownUser = (): string => { + throw new Error("ENOENT: no passwd entry for uid 1234"); + }; + const lookupFails = await install({ + argv: ["daemon", "install"], + role: "worker", + root: "/home/me/.simlock", + linux: true, + user: unknownUser, + }); + expect(lookupFails.output.stderr).toBe(""); + expect(lookupFails.exitCode).toBe(0); + expect(lookupFails.output.stdout.trimEnd().split("\n").at(-1)).toBe( + "To start after a reboot without logging in, run: loginctl enable-linger ", + ); + }); + + async function runWithService(options: { + readonly argv: readonly string[]; + readonly running: boolean; + readonly socketHeldPolls: number; + readonly state?: ServiceState; + readonly refuse?: boolean; + }): Promise<{ + readonly exitCode: number; + readonly output: ReturnType; + readonly stops: number; + readonly serviceManager: FakeServiceManager; + readonly clock: FakeClock; + }> { + const root = join(home, ".simlock"); + const clock = new FakeClock(0); + const serviceManager = new FakeServiceManager( + options.state === undefined ? {} : { state: options.state }, + ); + let stops = 0; + let polls = 0; + const connectExistingAdmin: CliEnvironment["connectExistingAdmin"] = async ( + resolveCredential, + ) => { + await resolveCredential(); + if (!options.running) + throw new IpcError("endpoint-not-found", "No IPC endpoint yet", undefined); + if (options.refuse === true) { + throw new SimlockError("FORBIDDEN", "protocol", "The admin token was rejected", {}); + } + // why: a stopping daemon refuses new connections while it still holds its socket. + if (stops > 0) throw new IpcError("connection-refused", "Connection refused", undefined); + return fakeClient({ + stopDaemon: async () => { + stops += 1; + return { stopping: true }; + }, + }); + }; + const output = outputCapture(); + const environment = output.environmentWith({ + clock, + daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, + connectExistingAdmin, + daemonEndpointPresent: async () => { + if (stops === 0) return options.running; + polls += 1; + return polls <= options.socketHeldPolls; + }, + readAdminTokenFile: async () => "admin-token", + sleep: async (milliseconds) => { + clock.advance(milliseconds); + }, + service: { + manager: serviceManager, + platform: "darwin", + user: () => "me", + definition: async () => + serviceDefinition({ + role: "worker", + root, + dataDirectory: root, + home, + nodePath, + mainPath, + env: {}, + }), + }, + }); + const exitCode = await runCli(options.argv, environment); + return { exitCode, output, stops, serviceManager, clock }; + } + + it('daemon stop with no socket exits 0 and prints Daemon not running; with --json it prints {"status":"not-running"}', async () => { + const text = await runWithService({ + argv: ["daemon", "stop"], + running: false, + socketHeldPolls: 0, + }); + expect(text.output.stderr).toBe(""); + expect(text.exitCode).toBe(0); + expect(text.output.stdout).toBe("Daemon not running\n"); + + const json = await runWithService({ + argv: ["daemon", "stop", "--json"], + running: false, + socketHeldPolls: 0, + }); + expect(json.output.stderr).toBe(""); + expect(json.exitCode).toBe(0); + expect(JSON.parse(json.output.stdout)).toEqual({ status: "not-running" }); + }); + + it("daemon uninstall removes the service", async () => { + const installedState: ServiceState = { + state: "installed", + path: "/Users/me/Library/LaunchAgents/com.callstack.simlock.worker.plist", + }; + const name = "com.callstack.simlock.worker"; + const cases = [ + { + label: "installed", + argv: ["daemon", "uninstall"], + state: installedState, + stdout: `Service removed: ${name} (launchd)\n`, + removed: 1, + }, + { + label: "installed, --json", + argv: ["daemon", "uninstall", "--json"], + state: installedState, + json: { status: "removed", service: { state: "not-installed", manager: "launchd", name } }, + removed: 1, + }, + { + label: "installed, a daemon running, --json", + argv: ["daemon", "uninstall", "--json"], + state: installedState, + running: true, + json: { + status: "removed", + stopped: true, + service: { state: "not-installed", manager: "launchd", name }, + }, + removed: 1, + }, + { + label: "installed, gateway flag, no gateway folder", + argv: ["--gateway", "daemon", "uninstall"], + state: installedState, + stdout: `Service removed: ${name} (launchd)\n`, + removed: 1, + }, + { + label: "not installed", + argv: ["daemon", "uninstall"], + state: { state: "not-installed" } as ServiceState, + stdout: `Service not installed: ${name}\n`, + removed: 0, + }, + { + label: "not installed, --json", + argv: ["daemon", "uninstall", "--json"], + state: { state: "not-installed" } as ServiceState, + json: { + status: "not-installed", + service: { state: "not-installed", manager: "launchd", name }, + }, + removed: 0, + }, + ]; + for (const testCase of cases) { + const result = await runWithService({ + argv: testCase.argv, + running: testCase.running ?? false, + socketHeldPolls: 0, + state: testCase.state, + }); + expect(result.output.stderr, testCase.label).toBe(""); + expect(result.exitCode, testCase.label).toBe(0); + expect(result.serviceManager.uninstalls, testCase.label).toHaveLength(testCase.removed); + if (testCase.json === undefined) + expect(result.output.stdout, testCase.label).toBe(testCase.stdout); + else expect(JSON.parse(result.output.stdout), testCase.label).toEqual(testCase.json); + } + }); + + it("daemon install stops a running daemon first and waits until its socket is released, even while it refuses connections", async () => { + const none = await runWithService({ + argv: ["daemon", "install"], + running: false, + socketHeldPolls: 0, + }); + expect(none.exitCode).toBe(0); + expect(none.stops).toBe(0); + expect(none.output.stdout).not.toContain("Stopped"); + + const running = await runWithService({ + argv: ["daemon", "install"], + running: true, + socketHeldPolls: 3, + }); + expect(running.exitCode).toBe(0); + expect(running.stops).toBe(1); + expect(running.output.stdout.split("\n").slice(0, 2)).toEqual([ + "Stopped the running daemon; leases are kept.", + "Service installed: com.callstack.simlock.worker (launchd)", + ]); + + const json = await runWithService({ + argv: ["daemon", "install", "--json"], + running: true, + socketHeldPolls: 0, + }); + expect(JSON.parse(json.output.stdout)).toMatchObject({ status: "installed", stopped: true }); + + const stubborn = await runWithService({ + argv: ["daemon", "install"], + running: true, + socketHeldPolls: Number.POSITIVE_INFINITY, + }); + expect(stubborn.exitCode).toBe(1); + expect(JSON.parse(stubborn.output.stderr)).toEqual({ + error: { + code: "SERVICE_FAILED", + message: "The running daemon did not stop within 300 s.", + }, + }); + expect(stubborn.serviceManager.installs).toEqual([]); + expect(stubborn.clock.now()).toBeGreaterThanOrEqual(300_000); + expect(stubborn.clock.now()).toBeLessThan(301_000); + + const refused = await runWithService({ + argv: ["daemon", "install"], + running: true, + socketHeldPolls: 0, + refuse: true, + }); + expect(refused.exitCode).toBe(1); + expect(JSON.parse(refused.output.stderr)).toEqual({ + error: { code: "FORBIDDEN", message: "The admin token was rejected" }, + }); + expect(refused.serviceManager.installs).toEqual([]); + + const uninstall = await runWithService({ + argv: ["daemon", "uninstall"], + running: true, + socketHeldPolls: 3, + state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, + }); + expect(uninstall.exitCode).toBe(0); + expect(uninstall.stops).toBe(1); + expect(uninstall.output.stdout.split("\n").slice(0, 2)).toEqual([ + "Daemon stopping", + "Service removed: com.callstack.simlock.worker (launchd)", + ]); + }); +}); + +describe("CLI: a start goes through the service (#830)", () => { + it("daemon start and a command's auto-start ask a loaded service to start the daemon and spawn nothing", async () => { + const home = "/Users/me"; + const root = join(home, ".simlock"); + const cases = [ + { label: "daemon start", argv: ["daemon", "start", "--json"] }, + { label: "a command's auto-start", argv: ["status", "--json"] }, + ]; + for (const testCase of cases) { + const filesystem = new MemoryFilesystem(); + const ipcTransport = new MemoryIpcTransport(); + const serviceManager = new FakeServiceManager({ + state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, + onStart: async () => { + await startInMemoryDaemon({ + adminTokenPath: join(root, "admin.token"), + filesystem, + ipcTransport, + socketPath: join(root, "daemon.sock"), + }); + }, + }); + const spawner = new FakeDaemonLauncher(() => undefined, { + exit: { code: 1, signal: null }, + }); + const notices: string[] = []; + const output = outputCapture({ + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: ipcTransport, + launcher: launcherFor({ + role: "worker", + root, + dataDirectory: root, + mainPath: "/opt/simlock/dist/daemon/main.js", + nodePath: "/opt/node/bin/node", + home, + env: {}, + serviceManager, + notify: (line) => notices.push(line), + spawner, + }), + daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + + const exitCode = await runCli(testCase.argv, output.environmentWith()); + + expect(exitCode, testCase.label).toBe(0); + expect(serviceManager.starts, testCase.label).toHaveLength(1); + expect(serviceManager.starts[0]?.name, testCase.label).toBe("com.callstack.simlock.worker"); + expect(spawner.launches, testCase.label).toBe(0); + expect(notices, testCase.label).toEqual([]); + } + }); +}); + +describe("CLI: a start through a service that fails (#830)", () => { + it("fails at the 20 s deadline with DAEMON_STARTUP_FAILED and the daemon.log lines written since the start", async () => { + const startError = + '{"message":"Daemon failed to start","fields":{"message":"EADDRINUSE: socket taken"}}\n'; + const cases = [ + { label: "the log holds the start error", written: startError, expected: "EADDRINUSE" }, + { label: "empty log", written: "", expected: undefined }, + { + label: "25 lines: the last 20 are kept", + written: Array.from({ length: 25 }, (_, index) => `line ${String(index + 1)}\n`).join(""), + expected: "line 6\n", + notExpected: "line 5\n", + }, + ]; + for (const testCase of cases) { + const directory = await mkdtemp(join(tmpdir(), "simlock-service-start-")); + try { + const logPath = join(directory, "daemon.log"); + await writeFile(logPath, "older line\n"); + const clock = new FakeClock(0); + const serviceManager = new FakeServiceManager({ + state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, + onStart: async () => { + await appendFile(logPath, testCase.written); + }, + }); + const output = outputCapture({ + filesystem: new MemoryFilesystem(), + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: new MemoryIpcTransport(), + launcher: launcherFor({ + role: "worker", + root: directory, + dataDirectory: directory, + mainPath: "/opt/simlock/dist/daemon/main.js", + nodePath: "/opt/node/bin/node", + home: "/Users/me", + env: {}, + serviceManager, + notify: () => undefined, + spawner: new FakeDaemonLauncher(), + }), + daemonHome: { role: "worker", dataDirectory: directory, gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli(["daemon", "start", "--json"], output.environmentWith()).then( + (code) => { + settled = true; + return code; + }, + ); + for (let tick = 0; tick < 500 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(settled, `${testCase.label}: not settled at ${String(clock.now())}ms`).toBe(true); + expect(await exitCode, testCase.label).toBe(1); + const error = ( + JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "") as { + error: { code: string; message: string }; + } + ).error; + expect(error.code, testCase.label).toBe("DAEMON_STARTUP_FAILED"); + expect(error.message, testCase.label).toContain("Timed out starting simlock daemon"); + expect(error.message, testCase.label).not.toContain("older line"); + if (testCase.expected === undefined) { + expect(error.message, testCase.label).not.toContain("\n"); + } else { + expect(error.message, testCase.label).toContain(testCase.expected); + } + if (testCase.notExpected !== undefined) { + expect(error.message, testCase.label).not.toContain(testCase.notExpected); + } + } finally { + await rm(directory, { force: true, recursive: true }); + } + } + }); +}); + +describe("CLI: a service manager call fails, or the platform has none (#830)", () => { + const home = "/Users/me"; + const root = join(home, ".simlock"); + const name = "com.callstack.simlock.worker"; + const plist = `${home}/Library/LaunchAgents/${name}.plist`; + const nodePath = "/opt/node/bin/node"; + const mainPath = "/opt/simlock/dist/daemon/main.js"; + const done = (code: number | null, stderr = "") => ({ code, stdout: "", stderr }); + + async function run(options: { + readonly argv: readonly string[]; + readonly platform?: NodeJS.Platform; + readonly script?: ConstructorParameters[0]; + readonly present?: readonly string[]; + }): Promise<{ + readonly exitCode: number; + readonly stderr: string; + readonly filesystem: MemoryFilesystem; + }> { + const filesystem = new MemoryFilesystem(); + for (const path of options.present ?? []) { + await filesystem.mkdirp(dirname(path)); + await filesystem.writeFileAtomic(path, ""); + } + const manager = + options.script === undefined + ? undefined + : new LaunchdServiceManager({ + runner: new ScriptedProcessRunner(options.script), + filesystem, + uid: 501, + }); + const ipc = new MemoryIpcTransport(); + const ports: CliEnvironmentPorts = { + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc, + launcher: launcherFor({ + role: "worker", + root, + dataDirectory: root, + mainPath, + nodePath, + home, + env: {}, + serviceManager: manager, + notify: () => undefined, + spawner: new FakeDaemonLauncher(() => undefined, { exit: { code: 1, signal: null } }), + }), + daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + service: { + manager, + platform: options.platform ?? "darwin", + user: () => "me", + home, + root, + nodePath, + mainPath, + }, + }; + const output = outputCapture(ports); + const exitCode = await runCli(options.argv, output.environmentWith()); + return { exitCode, stderr: output.stderr, filesystem }; + } + + const errorOf = (stderr: string): { code: string; message: string } => + ( + JSON.parse(stderr.trim().split("\n").at(-1) ?? "") as { + error: { code: string; message: string }; + } + ).error; + + it("a failed or stuck launchctl call is SERVICE_FAILED, a failed load removes the plist, and a platform without a service manager is a usage error", async () => { + for (const command of ["install", "uninstall"]) { + const result = await run({ argv: ["daemon", command], platform: "win32" }); + expect(result.exitCode, `win32 ${command}`).toBe(2); + expect(errorOf(result.stderr), `win32 ${command}`).toEqual({ + code: "USAGE", + message: `daemon ${command} is supported on macOS and Linux only`, + }); + } + + const bootout = { + match: { command: "launchctl", args: ["bootout", `gui/501/${name}`] }, + result: done(113), + }; + const bootstrap = { command: "launchctl", args: ["bootstrap", "gui/501", plist] }; + const failedLoad = await run({ + argv: ["daemon", "install"], + script: [ + bootout, + { + match: bootstrap, + result: done(5, "Bootstrap failed: 5: Input/output error\nsecond line"), + }, + ], + }); + expect(failedLoad.exitCode).toBe(1); + expect(errorOf(failedLoad.stderr)).toEqual({ + code: "SERVICE_FAILED", + message: `launchctl bootstrap gui/501 ${plist} failed: Bootstrap failed: 5: Input/output error`, + }); + expect(await failedLoad.filesystem.exists(plist)).toBe(false); + + const stuckLoad = await run({ + argv: ["daemon", "install"], + script: [bootout, { match: bootstrap, result: done(null) }], + }); + expect(stuckLoad.exitCode).toBe(1); + expect(errorOf(stuckLoad.stderr)).toEqual({ + code: "SERVICE_FAILED", + message: `launchctl bootstrap gui/501 ${plist} did not finish within 10 s`, + }); + expect(await stuckLoad.filesystem.exists(plist)).toBe(false); + + const failedKickstart = await run({ + argv: ["daemon", "start"], + present: [plist, nodePath, mainPath], + script: [ + { + match: { command: "plutil", args: ["-convert", "json", "-o", "-", plist] }, + result: { + code: 0, + stdout: JSON.stringify({ ProgramArguments: [nodePath, mainPath] }), + stderr: "", + }, + }, + { match: { command: "launchctl", args: ["print", `gui/501/${name}`] }, result: done(0) }, + { + match: { command: "launchctl", args: ["kickstart", `gui/501/${name}`] }, + result: done(1, "Could not find service"), + }, + ], + }); + expect(failedKickstart.exitCode).toBe(1); + expect(errorOf(failedKickstart.stderr)).toEqual({ + code: "SERVICE_FAILED", + message: `launchctl kickstart gui/501/${name} failed: Could not find service`, + }); + }); +}); diff --git a/src/cli/index-daemon-status.test.ts b/src/cli/index-daemon-status.test.ts new file mode 100644 index 000000000..cda718931 --- /dev/null +++ b/src/cli/index-daemon-status.test.ts @@ -0,0 +1,560 @@ +import { describe, expect, it } from "vitest"; +import { + FakeServiceManager, + serviceDefinition, + type ServiceManager, + type ServiceState, +} from "../ports/index.js"; +import { runCli } from "./index.js"; +import { fakeClient, outputCapture, simlockError } from "./test-index.js"; + +async function runDaemonStatus( + testCase: { + readonly role: "worker" | "gateway"; + readonly running: boolean; + readonly state: ServiceState | undefined; + }, + json: boolean, +): Promise<{ readonly stdout: string }> { + const home = "/Users/me"; + const output = outputCapture(); + const manager = + testCase.state === undefined ? undefined : new FakeServiceManager({ state: testCase.state }); + const environment = output.environmentWith({ + daemonHome: { role: testCase.role, dataDirectory: "/simlock", gatewayPresent: false }, + connectExistingAdmin: async () => { + if (!testCase.running) throw new Error("connect ECONNREFUSED"); + return fakeClient(); + }, + service: { + manager, + platform: manager === undefined ? "linux" : "darwin", + user: () => "me", + definition: async () => + serviceDefinition({ + role: testCase.role, + root: `${home}/.simlock`, + dataDirectory: "/simlock", + home, + nodePath: "/Users/me/.nvm/versions/node/v22.13.0/bin/node", + mainPath: "/Users/me/simlock/dist/daemon/main.js", + env: {}, + }), + }, + }); + await expect( + runCli(["daemon", "status", ...(json ? ["--json"] : [])], environment), + ).resolves.toBe(0); + return output; +} + +describe("CLI: daemon status (ADR 0003 §11)", () => { + it.each([ + { name: "the socket cannot be reached at all", error: () => new Error("connect ECONNREFUSED") }, + { + name: "a transport-kind SimlockError (connection lost)", + error: () => simlockError("DAEMON_CONNECTION_LOST"), + }, + ])("reports stopped when $name", async ({ error }) => { + const output = outputCapture(); + await expect( + runCli( + ["daemon", "status", "--json"], + output.environmentWith({ + connectExistingAdmin: async () => { + throw error(); + }, + }), + ), + ).resolves.toBe(0); + expect(JSON.parse(output.stdout)).toEqual({ status: "stopped" }); + }); + + it("distinguishes a refused handshake (socket reachable) from an absent socket", async () => { + const output = outputCapture(); + await expect( + runCli( + ["daemon", "status", "--json"], + output.environmentWith({ + connectExistingAdmin: async () => { + throw simlockError("ADMIN_AUTHENTICATION_FAILED"); + }, + }), + ), + ).resolves.toBe(1); + const parsed = JSON.parse(output.stdout) as { status: string; error: { code: string } }; + expect(parsed.status).toBe("handshake-refused"); + expect(parsed.error.code).toBe("ADMIN_AUTHENTICATION_FAILED"); + }); + + it("never auto-launches the daemon for daemon status", async () => { + const output = outputCapture(); + let launched = false; + await runCli( + ["daemon", "status"], + output.environmentWith({ + connectAdmin: async () => { + launched = true; + return fakeClient(); + }, + connectExistingAdmin: async () => fakeClient(), + }), + ); + expect(launched).toBe(false); + }); + + it("daemon status prints the Service: line and the service field in --json, for a worker or a gateway, running or stopped, and none on an unsupported platform", async () => { + const home = "/Users/me"; + const node = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; + const main = "/Users/me/simlock/dist/daemon/main.js"; + const worker = "com.callstack.simlock.worker"; + const gateway = "com.callstack.simlock.gateway"; + const plist = (name: string) => `${home}/Library/LaunchAgents/${name}.plist`; + const cases: readonly { + readonly label: string; + readonly role: "worker" | "gateway"; + readonly running: boolean; + readonly state: ServiceState | undefined; + readonly line: string | undefined; + readonly service: Record | undefined; + }[] = [ + { + label: "not installed, stopped", + role: "worker", + running: false, + state: { state: "not-installed" }, + line: "Service: not installed", + service: { state: "not-installed", manager: "launchd", name: worker }, + }, + { + label: "installed, running", + role: "worker", + running: true, + state: { state: "installed", path: plist(worker) }, + line: `Service: installed (launchd, ${worker})`, + service: { state: "installed", manager: "launchd", name: worker, path: plist(worker) }, + }, + { + label: "not loaded, running", + role: "worker", + running: true, + state: { state: "not-loaded", path: plist(worker) }, + line: `Service: not loaded (launchd, ${worker}): commands start the daemon without it`, + service: { state: "not-loaded", manager: "launchd", name: worker, path: plist(worker) }, + }, + { + label: "Node path gone, stopped", + role: "worker", + running: false, + state: { state: "broken", path: plist(worker), missing: [node] }, + line: `Service: broken (launchd, ${worker}): ${node} is gone; run simlock daemon install`, + service: { + state: "broken", + manager: "launchd", + name: worker, + path: plist(worker), + missing: [node], + }, + }, + { + label: "both paths gone, stopped", + role: "worker", + running: false, + state: { state: "broken", path: plist(worker), missing: [node, main] }, + line: `Service: broken (launchd, ${worker}): ${node}, ${main} are gone; run simlock daemon install`, + service: { + state: "broken", + manager: "launchd", + name: worker, + path: plist(worker), + missing: [node, main], + }, + }, + { + label: "gateway, running", + role: "gateway", + running: true, + state: { state: "installed", path: plist(gateway) }, + line: `Service: installed (launchd, ${gateway})`, + service: { state: "installed", manager: "launchd", name: gateway, path: plist(gateway) }, + }, + { + label: "unsupported platform", + role: "worker", + running: true, + state: undefined, + line: undefined, + service: undefined, + }, + ]; + for (const testCase of cases) { + const human = await runDaemonStatus(testCase, false); + const lines = human.stdout.trimEnd().split("\n"); + expect( + [lines[0], ...lines.filter((line) => line.startsWith("Service:"))], + testCase.label, + ).toEqual([ + testCase.running ? "Daemon: running (worker)" : "Daemon stopped", + ...(testCase.line === undefined ? [] : [testCase.line]), + ]); + if (testCase.line !== undefined) + expect(lines.at(-1), `${testCase.label} last line`).toBe(testCase.line); + const json = JSON.parse((await runDaemonStatus(testCase, true)).stdout) as { + readonly service?: unknown; + }; + expect(json.service, `${testCase.label} --json`).toEqual(testCase.service); + } + }); + + it("daemon status fails with the service manager's error instead of printing Daemon stopped", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + connectExistingAdmin: async () => { + throw new Error("connect ECONNREFUSED"); + }, + service: { + manager: { + manager: "launchd", + state: async () => { + throw new Error("launchctl timed out"); + }, + } as unknown as ServiceManager, + platform: "darwin", + user: () => "me", + definition: async () => + serviceDefinition({ + role: "worker", + root: "/Users/me/.simlock", + dataDirectory: "/simlock", + home: "/Users/me", + nodePath: "/n", + mainPath: "/m.js", + env: {}, + }), + }, + }); + + await expect(runCli(["daemon", "status"], environment)).resolves.toBe(1); + + expect(output.stdout).not.toContain("Daemon stopped"); + expect(output.stderr).toContain("launchctl timed out"); + }); + + it("asks the daemon to purge orphans only once the operator has confirmed", async () => { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["doctor", "--purge-orphans"], + output.environmentWith({ + confirm: async () => true, + connectAdmin: async () => + fakeClient({ + runDoctor: (input) => { + seen.push(input); + return Promise.resolve({ findings: [], sources: [] }); + }, + }), + }), + ), + ).resolves.toBe(0); + + expect(seen).toEqual([{ fix: false, purgeOrphans: true }]); + }); + + it("prints one stderr line per prerequisite finding, keeps the JSON report on stdout, and exits 0", async () => { + const output = outputCapture(); + const findings = [ + { + kind: "prerequisite-missing" as const, + message: "The Android emulator package is not installed.", + platform: "android" as const, + prerequisite: "android-emulator", + remedy: "Run `sdkmanager --install emulator`.", + }, + { + kind: "prerequisite-missing" as const, + message: "Xcode is not installed.", + platform: "ios" as const, + prerequisite: "xcode", + remedy: "Run `xcode-select`.", + }, + { deviceId: "d-1", kind: "expired-live-lease" as const, leaseId: "l-1" }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), + }), + ), + ).resolves.toBe(0); + + // why: The harness has no admin credential, so a connection notice precedes these lines. + expect(output.stderr.split("\n").filter((line) => line.startsWith("Missing"))).toEqual([ + "Missing [android] android-emulator: The Android emulator package is not installed. " + + "Run `sdkmanager --install emulator`.", + "Missing [ios] xcode: Xcode is not installed. Run `xcode-select`.", + ]); + expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); + }); + + it("simlock doctor prints a service-broken finding to stderr, naming each missing path, and keeps the finding in the JSON report", async () => { + const node = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; + const main = "/Users/me/simlock/dist/daemon/main.js"; + for (const testCase of [ + { + missing: [node], + line: `Service broken: com.callstack.simlock.worker (launchd) names missing ${node}. Run \`simlock daemon install\` again.`, + }, + { + missing: [node, main], + line: `Service broken: com.callstack.simlock.worker (launchd) names missing ${node}, ${main}. Run \`simlock daemon install\` again.`, + }, + ]) { + const output = outputCapture(); + const findings = [ + { + kind: "service-broken" as const, + manager: "launchd" as const, + name: "com.callstack.simlock.worker", + missing: testCase.missing, + remedy: "Run `simlock daemon install` again.", + }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), + }), + ), + ).resolves.toBe(0); + + expect(output.stderr.split("\n").filter((line) => line.startsWith("Service broken"))).toEqual( + [testCase.line], + ); + expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); + } + }); + + it("simlock doctor prints a driver-advisory finding to stderr as `Warning [] : `", async () => { + const output = outputCapture(); + const findings = [ + { + code: "device-profile-hidden", + kind: "driver-advisory" as const, + message: '"Pixel 9" in devices.xml is not used', + platform: "android" as const, + }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), + }), + ), + ).resolves.toBe(0); + + expect(output.stderr.split("\n")).toContain( + 'Warning [android] device-profile-hidden: "Pixel 9" in devices.xml is not used', + ); + }); + + it("simlock doctor writes `Unavailable [] : ` on stderr for a driver-unavailable finding and prints the report on stdout unchanged", async () => { + const output = outputCapture(); + const findings = [ + { + detail: "Refusing the ios device set: bad. Restart the daemon.", + kind: "driver-unavailable" as const, + platform: "ios" as const, + reason: "invalid-device-set", + }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), + }), + ), + ).resolves.toBe(0); + + expect(output.stderr.split("\n")).toContain( + "Unavailable [ios] invalid-device-set: Refusing the ios device set: bad. Restart the daemon.", + ); + expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); + }); + + it("simlock doctor writes agent-device fail ; on stderr for a module-unavailable finding, one line of at most 500 characters per part, after the advisory, prerequisite and warm-pool lines, prints the report on stdout unchanged, and exits 0", async () => { + const output = outputCapture(); + const findings = [ + { + kind: "module-unavailable" as const, + module: "agent-device", + message: `found 0.20.3,\n need ${"x".repeat(600)}`, + remedy: `install agent-device 0.21.28 or later,\nthen ${"y".repeat(600)}`, + }, + { + kind: "warm-pool-target-unreachable" as const, + message: "iOS 27.0 is not installed", + platform: "ios" as const, + reason: "runtime-missing" as const, + remedy: "run simlock component install ios 27.0", + target: "iPhone 17 / 27.0 / full", + }, + { + kind: "prerequisite-missing" as const, + message: "Xcode is not installed.", + platform: "ios" as const, + prerequisite: "xcode", + remedy: "Run `xcode-select`.", + }, + { + code: "slim-runtime-unsupported", + kind: "driver-advisory" as const, + message: "detail", + platform: "ios" as const, + }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), + }), + ), + ).resolves.toBe(0); + + const lines = output.stderr.split("\n"); + const index = lines.findIndex((line) => line.startsWith("agent-device fail ")); + expect(index).toBeGreaterThan(-1); + expect(lines[index]).toBe( + `agent-device fail found 0.20.3, need ${"x".repeat(500 - "found 0.20.3, need ".length)}; ${("install agent-device 0.21.28 or later, then " + "y".repeat(600)).slice(0, 500)}`, + ); + expect(lines.filter((line) => line.startsWith("agent-device"))).toHaveLength(1); + for (const earlier of ["Warning [ios]", "Missing [ios]", "warm-pool-target-unreachable"]) { + expect(lines.findIndex((line) => line.startsWith(earlier))).toBeLessThan(index); + expect(lines.findIndex((line) => line.startsWith(earlier))).toBeGreaterThan(-1); + } + expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); + }); + + it("simlock doctor writes fail … for a finding of another module", async () => { + const output = outputCapture(); + const findings = [ + { + kind: "module-unavailable" as const, + message: "needs\n tmux", + module: "other-mod", + remedy: "install tmux", + }, + ]; + + await runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), + }), + ); + + expect(output.stderr.split("\n")).toContain("other-mod fail needs tmux; install tmux"); + expect(output.stderr).not.toContain("agent-device"); + }); + + it("keeps --purge-orphans off the wire when --fix is all that was asked for", async () => { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["doctor", "--fix"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + runDoctor: (input) => { + seen.push(input); + return Promise.resolve({ findings: [], sources: [] }); + }, + }), + }), + ), + ).resolves.toBe(0); + + // why: The upgrade contract: `doctor --fix` running unattended in CI never becomes + // destructive on its own (ADR 0001, decision 6). + expect(seen).toEqual([{ fix: true, purgeOrphans: false }]); + }); + + it.each([ + ["declined", async () => false], + // why: Explicitly absent: the shared harness defaults `confirm` to an auto-yes, so leaving it + // out would test the default rather than the no-terminal case. + ["unavailable", undefined], + ] as const)( + "refuses --purge-orphans and contacts nobody when confirmation is %s", + async (_case, confirm) => { + const output = outputCapture(); + let connected = false; + + await expect( + runCli( + ["doctor", "--purge-orphans"], + output.environmentWith({ + // why: `exactOptionalPropertyTypes`: an explicitly-absent confirm has to be spread in, + // not passed as `undefined`. + ...(confirm === undefined ? { confirm: undefined } : { confirm }), + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(2); + + expect(connected).toBe(false); + expect(JSON.parse(output.stderr)).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("--yes") }, + }); + }, + ); + + it("takes --yes as the confirmation, so an unattended purge needs no terminal", async () => { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["doctor", "--purge-orphans", "--yes"], + output.environmentWith({ + confirm: async () => { + throw new Error("--yes must not ask"); + }, + connectAdmin: async () => + fakeClient({ + runDoctor: (input) => { + seen.push(input); + return Promise.resolve({ findings: [], sources: [] }); + }, + }), + }), + ), + ).resolves.toBe(0); + + expect(seen).toEqual([{ fix: false, purgeOrphans: true }]); + }); +}); diff --git a/src/cli/index-device-config.test.ts b/src/cli/index-device-config.test.ts new file mode 100644 index 000000000..0ad0655b9 --- /dev/null +++ b/src/cli/index-device-config.test.ts @@ -0,0 +1,802 @@ +import { describe, expect, it } from "vitest"; +import { FakeDaemonLauncher, MemoryFilesystem } from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import type { LeaseGrant, StatusGetOutput } from "../simlock-client/client.js"; +import { runCli } from "./index.js"; +import { fakeClient, outputCapture, realCliEnvironmentPorts } from "./test-index.js"; + +describe("CLI: device add and device remove (ADR 0022 §2)", () => { + async function runDevice(argv: string[]) { + const output = outputCapture(); + let connected = false; + const code = await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ); + return { code, connected, stderr: output.stderr, stdout: output.stdout }; + } + + it.each([ + ["an unknown subcommand", ["device", "enroll", "X"]], + ["two IDs", ["device", "add", "--platform", "ios", "A", "B"]], + ["--yes, which remove does not take", ["device", "remove", "--yes", "X"]], + ["no --platform", ["device", "add", "X"]], + ])("exits 2 with USAGE and contacts nobody for %s", async (_case, argv) => { + const result = await runDevice(argv); + + expect(result.code).toBe(2); + expect(result.connected).toBe(false); + expect(JSON.parse(result.stderr)).toEqual({ + error: { code: "USAGE", message: expect.any(String) }, + }); + }); + + it("prints one line for an enrollment, naming the platform's OS, and the JSON with --json", async () => { + const line = await runDevice(["device", "add", "--platform", "ios", "UDID-1"]); + const json = await runDevice(["device", "add", "--platform", "android", "SER-1", "--json"]); + + expect(line.stdout).toBe( + "Enrolled iPhone 15 Pro (iOS 18.4) as UDID-1, 2 user apps recorded.\n", + ); + expect(JSON.parse(json.stdout)).toMatchObject({ driverDeviceId: "SER-1", platform: "android" }); + }); + + it.each([ + ["device", ["device"]], + ["device --help", ["device", "--help"]], + ["device -h", ["device", "-h"]], + ["device add --help", ["device", "add", "--help"]], + ["device remove -h", ["device", "remove", "-h"]], + ])("prints the usage and contacts nobody for %s", async (_case, argv) => { + const result = await runDevice(argv); + + expect(result.code).toBe(0); + expect(result.connected).toBe(false); + expect(result.stdout).toBe( + "Usage: simlock device add --platform [--json]\n" + + " simlock device remove [--json]\n", + ); + }); + + it("says what is wrong with a usage error, and refuses --platform on remove", async () => { + const messages = async (argv: string[]) => { + const result = await runDevice(argv); + expect(result.code).toBe(2); + return (JSON.parse(result.stderr) as { error: { message: string } }).error.message; + }; + + expect(await messages(["device", "enroll", "X"])).toContain("Unknown device command: enroll"); + expect(await messages(["device", "add", "--platform", "ios"])).toContain( + "device add takes exactly one ", + ); + expect(await messages(["device", "remove", "A", "B"])).toContain( + "device remove takes exactly one ", + ); + expect(await messages(["device", "add", "--platform", "windows", "X"])).toContain( + "device add needs --platform ios or --platform android", + ); + expect(await messages(["device", "remove", "--platform", "ios", "X"])).toContain( + "Unknown option '--platform'", + ); + }); + + it("closes the connection after an enrollment and after a removal", async () => { + const closed: string[] = []; + for (const argv of [ + ["device", "add", "--platform", "ios", "U"], + ["device", "remove", "U"], + ]) { + const output = outputCapture(); + await runCli( + argv, + output.environmentWith({ + connectAdmin: async () => ({ + ...fakeClient(), + close: () => { + closed.push(argv[1] ?? ""); + return Promise.resolve(); + }, + }), + }), + ); + } + + expect(closed).toEqual(["add", "remove"]); + }); + + it("says again for a re-enrollment, one user app in the singular, and the Android OS name", async () => { + const output = outputCapture(); + await runCli( + ["device", "add", "--platform", "android", "SER-1"], + output.environmentWith({ + connectAdmin: async () => ({ + ...fakeClient(), + addDevice: () => + Promise.resolve({ + appCount: 1, + class: "tablet" as const, + driverDeviceId: "SER-1", + id: "dev_1", + model: "Pixel Tablet", + osVersion: "15", + platform: "android" as const, + reenrolled: true, + }), + }), + }), + ); + + expect(output.stdout).toBe( + "Enrolled Pixel Tablet (Android 15) as SER-1 again, 1 user app recorded.\n", + ); + }); + + it("prints the operation's output for remove with --json", async () => { + const result = await runDevice(["device", "remove", "UDID-1", "--json"]); + + expect(JSON.parse(result.stdout)).toEqual({ driverDeviceId: "UDID-1", id: "UDID-1" }); + }); + + it("prints one line for a removal, with no confirmation", async () => { + const result = await runDevice(["device", "remove", "UDID-1"]); + + expect(result.code).toBe(0); + expect(result.stdout).toBe( + "Removed UDID-1. It is no longer enrolled; nothing on the device changed.\n", + ); + }); +}); + +describe("CLI: component remove (ADR 0010 §8)", () => { + function removingClient(asked: unknown[]) { + return fakeClient({ + removeComponent: async (input) => { + asked.push(input); + return { + outcome: input.dryRun === true ? "would-remove" : "removed", + platform: input.platform, + sizeBytes: 7_000_000_000, + version: input.version, + }; + }, + }); + } + + it("removes with --yes and prints the daemon's answer as one JSON line on stdout", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + + await expect( + runCli( + ["component", "remove", "ios", "26.4", "--yes"], + output.environmentWith({ + confirm: async () => { + throw new Error("--yes must not ask"); + }, + connectAdmin: async () => removingClient(asked), + }), + ), + ).resolves.toBe(0); + + expect(asked).toEqual([{ platform: "ios", version: "26.4" }]); + expect(output.stdout.trim().split("\n")).toHaveLength(1); + expect(JSON.parse(output.stdout)).toEqual({ + outcome: "removed", + platform: "ios", + sizeBytes: 7_000_000_000, + version: "26.4", + }); + }); + + it("asks for confirmation without --yes, and removes once it is given", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const questions: string[] = []; + + await expect( + runCli( + ["component", "remove", "android", "35"], + output.environmentWith({ + confirm: async (question) => { + questions.push(question); + return true; + }, + connectAdmin: async () => removingClient(asked), + }), + ), + ).resolves.toBe(0); + + expect(questions).toEqual(["Remove android 35? [y/N] "]); + expect(asked).toEqual([{ platform: "android", version: "35" }]); + }); + + it.each([ + ["declined", async () => false], + // why: No terminal to ask on: the CLI's own `confirmTerminal` is not installed, so this is the + // no-terminal case rather than the default. + ["unavailable", undefined], + ] as const)( + "refuses without --yes when confirmation is %s, and contacts nobody", + async (_case, confirm) => { + const output = outputCapture(); + let connected = false; + + await expect( + runCli( + ["component", "remove", "ios", "26.4"], + output.environmentWith({ + ...(confirm === undefined ? { confirm: undefined } : { confirm }), + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(2); + + expect(connected).toBe(false); + expect(JSON.parse(output.stderr)).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("--yes") }, + }); + }, + ); + + it("does not ask under --dry-run, and asks the daemon for a dry run", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + + await expect( + runCli( + ["component", "remove", "ios", "26.4", "--dry-run"], + output.environmentWith({ + confirm: async () => { + throw new Error("--dry-run must not ask"); + }, + connectAdmin: async () => removingClient(asked), + }), + ), + ).resolves.toBe(0); + + expect(asked).toEqual([{ dryRun: true, platform: "ios", version: "26.4" }]); + expect(JSON.parse(output.stdout)).toMatchObject({ outcome: "would-remove" }); + }); + + it.each([ + ["COMPONENT_NOT_OWNED", {}, 12], + ["COMPONENT_IN_USE", { devices: 1, foreignDevices: 2 }, 12], + ["COMPONENT_BUSY", {}, 11], + ] as const)( + "exits with the table's code when the daemon refuses with %s", + async (code, details, exit) => { + const output = outputCapture(); + const client = fakeClient({ + removeComponent: () => + Promise.reject(new SimlockError(code, "domain", "refused", details as never)), + }); + + await expect( + runCli( + ["component", "remove", "ios", "26.4", "--yes"], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(exit); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ + error: { code }, + }); + }, + ); + + it.each([ + ["no version", ["component", "remove", "ios"]], + ["an unknown platform", ["component", "remove", "tvos", "1.0", "--yes"]], + ["an extra argument", ["component", "remove", "ios", "26.4", "extra", "--yes"]], + ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { + const output = outputCapture(); + let connected = false; + + await expect( + runCli( + argv, + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(2); + expect(connected).toBe(false); + }); +}); + +describe("CLI: config set validates with the real config loader (ADR 0003 §11, B9)", () => { + it.each([ + ["an unknown key", "capacty.limits.ios.maxDevices", "2"], + ["a key missing the Ms suffix", "lease.defaultTtl", "60000"], + ])( + "rejects %s instead of silently dropping it or validating clean", + async (_name, key, value) => { + const output = outputCapture(realCliEnvironmentPorts()); + let wrote = false; + const exitCode = await runCli( + ["config", "set", key, value], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async () => { + wrote = true; + }, + }), + ); + expect(exitCode).toBe(2); + expect(wrote).toBe(false); + expect(JSON.parse(output.stderr).error.code).toBe("USAGE"); + }, + ); + + it.each([ + ["ios.simulator.audio", "false", "loud"], + ["android.emulator.headless", "true", "yes"], + ])( + "writes a good value of %s, a key its driver declares, and refuses a bad one naming the key", + async (key, good, bad) => { + const run = async (value: string) => { + const output = outputCapture(realCliEnvironmentPorts()); + let wrote = false; + const exitCode = await runCli( + ["config", "set", key, value], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async () => { + wrote = true; + }, + }), + ); + return { exitCode, stderr: output.stderr, wrote }; + }; + + const accepted = await run(good); + const refused = await run(bad); + + expect([accepted.exitCode, accepted.wrote]).toEqual([0, true]); + expect([refused.exitCode, refused.wrote]).toEqual([2, false]); + expect(refused.stderr).toContain(key); + }, + ); + + it("--gateway config set validates against the gateway's rules, not the worker's", async () => { + const gatewayHome = { + role: "gateway", + dataDirectory: "/simlock/gateway", + gatewayPresent: true, + } as const; + const output = outputCapture({ ...realCliEnvironmentPorts(), daemonHome: gatewayHome }); + let wrote = false; + const exitCode = await runCli( + ["--gateway", "config", "set", "http.enabled", "false"], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async () => { + wrote = true; + }, + }), + ); + expect(exitCode).toBe(2); + expect(wrote).toBe(false); + expect(output.stderr).toContain("http.enabled"); + }); + + it("--gateway daemon start refuses a config.json that fails gateway validation with exit 2 and USAGE, before any launch", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock/gateway"); + await filesystem.writeFileAtomic( + "/simlock/gateway/config.json", + JSON.stringify({ http: { enabled: false } }), + ); + const launcher = new FakeDaemonLauncher(); + const output = outputCapture({ + ...realCliEnvironmentPorts(filesystem), + daemonHome: { role: "gateway", dataDirectory: "/simlock/gateway", gatewayPresent: true }, + launcher, + }); + const exitCode = await runCli( + ["--gateway", "daemon", "start", "--token", "probe"], + output.environmentWith(), + ); + expect(exitCode).toBe(2); + expect(launcher.launches).toBe(0); + expect(JSON.parse(output.stderr).error).toEqual({ + code: "USAGE", + message: + 'Invalid config value for "http.enabled": expected true in gateway mode (a gateway is reached over HTTP)', + }); + }); + + describe("a joined worker's own HTTP listener (#811)", () => { + const joined = { url: "wss://fleet.example", token: "join-secret" }; + const homeWithGateway = { + role: "worker", + dataDirectory: "/simlock", + gatewayPresent: true, + } as const; + + async function workerFiles(contents: unknown) { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic("/simlock/config.json", JSON.stringify(contents)); + return filesystem; + } + + it.each([ + [ + "enabling HTTP on a joined worker", + ["http.enabled", "true"], + { gateway: joined }, + 'Invalid config after setting http.enabled: Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', + ], + [ + "adding the gateway object to a standalone worker with HTTP on", + ["gateway", JSON.stringify(joined)], + { http: { enabled: true } }, + 'Invalid config after setting gateway: Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', + ], + ] as const)( + "refuses a config set that creates the conflict with exit 2, USAGE and the file untouched: %s", + async (_name, [key, value], contents, message) => { + const filesystem = await workerFiles(contents); + const before = await filesystem.readFile("/simlock/config.json"); + const output = outputCapture({ + ...realCliEnvironmentPorts(filesystem), + daemonHome: homeWithGateway, + }); + const exitCode = await runCli( + ["--local", "config", "set", key, value], + output.environmentWith(), + ); + expect(exitCode).toBe(2); + expect(JSON.parse(output.stderr).error).toMatchObject({ code: "USAGE", message }); + expect(await filesystem.readFile("/simlock/config.json")).toBe(before); + }, + ); + + it("repairs a conflicting file with config set http.enabled false and prints the restart notice", async () => { + const filesystem = await workerFiles({ gateway: joined, http: { enabled: true } }); + const output = outputCapture({ + ...realCliEnvironmentPorts(filesystem), + daemonHome: homeWithGateway, + }); + const exitCode = await runCli( + ["--local", "config", "set", "http.enabled", "false"], + output.environmentWith(), + ); + expect(exitCode).toBe(0); + expect(output.stdout).toBe( + "Updated http.enabled in /simlock/config.json; takes effect on daemon restart.\n", + ); + expect(JSON.parse(await filesystem.readFile("/simlock/config.json"))).toMatchObject({ + http: { enabled: false }, + }); + }); + + it("--local daemon start with no daemon listening refuses a joined worker's HTTP conflict with exit 2 and USAGE, before any launch", async () => { + const filesystem = await workerFiles({ + gateway: { url: "ws://127.0.0.1:1", token: "join-secret" }, + http: { enabled: true }, + }); + // why: a launcher that exits at once, so a start that wrongly launches fails the launches assertion instead of waiting out the test. + const launcher = new FakeDaemonLauncher(undefined, { exit: { code: 1, signal: null } }); + const output = outputCapture({ + ...realCliEnvironmentPorts(filesystem), + daemonHome: homeWithGateway, + launcher, + }); + const exitCode = await runCli(["--local", "daemon", "start"], output.environmentWith()); + expect(exitCode).toBe(2); + expect(launcher.launches).toBe(0); + expect(JSON.parse(output.stderr).error).toEqual({ + code: "USAGE", + message: + 'Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', + }); + }); + }); + + it("simlock config set eventLog.retention rejects a non-positive value, naming the key", async () => { + for (const value of ["0", "1.5"]) { + const output = outputCapture(realCliEnvironmentPorts()); + let wrote = false; + const exitCode = await runCli( + ["config", "set", "eventLog.retention", value], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async () => { + wrote = true; + }, + }), + ); + expect(exitCode).toBe(2); + expect(wrote).toBe(false); + expect(output.stderr).toContain("eventLog.retention"); + } + }); + + it("still writes a genuinely valid key", async () => { + const output = outputCapture(realCliEnvironmentPorts()); + let written: Record | undefined; + const exitCode = await runCli( + // why: Comfortably under the default `lease.maxTtlMs` (4h), so this doesn't also trip the + // TTL pair rule -- this test is only about the key itself validating clean. + ["config", "set", "lease.defaultTtlMs", "2400000"], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async (contents) => { + written = contents; + }, + }), + ); + expect(exitCode).toBe(0); + expect(written).toEqual({ lease: { defaultTtlMs: 2400000 } }); + }); + + it.each(["teamId", "bundleId", "provisioningProfile", "signingIdentity"])( + "simlock config set capabilities.agent-device.ios.%s stores a non-empty string and refuses an empty string or a number with the usage error", + async (key) => { + const path = `capabilities.agent-device.ios.${key}`; + const set = async (value: string) => { + const output = outputCapture(realCliEnvironmentPorts()); + let written: Record | undefined; + const exitCode = await runCli( + ["config", "set", path, value], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async (contents) => { + written = contents; + }, + }), + ); + return { exitCode, output, written }; + }; + + const stored = await set("ABCDE12345"); + expect(stored.exitCode).toBe(0); + expect(stored.output.stdout).toBe( + `Updated ${path} in /simlock/config.json; takes effect on daemon restart.\n`, + ); + expect(stored.written).toEqual({ + capabilities: { "agent-device": { ios: { [key]: "ABCDE12345" } } }, + }); + for (const value of ["", "42"]) { + const refused = await set(value); + expect(refused.exitCode).toBe(2); + expect(JSON.parse(refused.output.stderr)).toEqual({ + error: { + code: "USAGE", + message: `Invalid config after setting ${path}: Invalid config value for "${path}": expected a non-empty string`, + }, + }); + expect(refused.written).toBeUndefined(); + } + }, + ); + + it("does not let a stray file at the scratch validation path change the outcome", async () => { + // why: Guards against B9's second bug: the validation scratch path must never be able to read a + // real file, whether one happens to already exist there or not. + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/"); + await filesystem.writeFileAtomic( + "/config-set-validation.json", + `${JSON.stringify({ totally: { bogus: true } })}\n`, + ); + const output = outputCapture(realCliEnvironmentPorts(filesystem)); + let wrote = false; + const exitCode = await runCli( + ["config", "set", "downloads.policy", "never"], + output.environmentWith({ + readConfigFile: async () => ({}), + writeConfigFile: async () => { + wrote = true; + }, + }), + ); + expect(exitCode).toBe(0); + expect(wrote).toBe(true); + }); +}); + +describe("CLI: config set and the retired eventLog.rotateBytes", () => { + const configSet = async (file: Record) => { + const output = outputCapture(realCliEnvironmentPorts()); + let written: Record | undefined; + const environment = output.environmentWith({ + readConfigFile: async () => structuredClone(file), + writeConfigFile: async (contents) => { + written = contents; + }, + }); + const exitCode = await runCli(["config", "set", "log.level", "debug"], environment); + return { configPath: environment.configPath, exitCode, output, written }; + }; + + it("simlock config set on a file holding eventLog.rotateBytes writes it without that key and prints the Removed line once; on a file without it, no such line", async () => { + const holding = await configSet({ + eventLog: { retention: 86_400_000, rotateBytes: 1_048_576 }, + log: { level: "info" }, + }); + const without = await configSet({ + eventLog: { retention: 86_400_000 }, + log: { level: "info" }, + }); + + expect(holding.exitCode).toBe(0); + expect(holding.written).toEqual({ + eventLog: { retention: 86_400_000 }, + log: { level: "debug" }, + }); + expect(holding.output.stderr).toBe( + `Removed eventLog.rotateBytes from ${holding.configPath}: it is no longer used.\n`, + ); + expect(holding.output.stdout).toBe( + `Updated log.level in ${holding.configPath}; takes effect on daemon restart.\n`, + ); + expect(without.exitCode).toBe(0); + expect(without.written).toEqual({ + eventLog: { retention: 86_400_000 }, + log: { level: "debug" }, + }); + expect(without.output.stderr).toBe(""); + }); +}); + +describe("CLI: config set and the file mode of config.json (#252)", () => { + it("writes the config.json holding gateway.token owner-only, whether it creates the file or replaces an owner-only one", async () => { + // why: Two homes, one per way `config set` comes to write the join token. `MemoryFilesystem` + // gives a file written without a `mode` Node's default (0644), so the assertion pins what + // the CLI asks for, independent of the umask of whoever runs the suite. + const fresh = new MemoryFilesystem(); + const freshExit = await runCli( + ["config", "set", "gateway", '{"url":"wss://gw.example:4700","token":"slk_fresh"}'], + outputCapture(realCliEnvironmentPorts(fresh)).environmentWith(), + ); + + const tightened = new MemoryFilesystem(); + await tightened.mkdirp("/simlock"); + await tightened.writeFileAtomic( + "/simlock/config.json", + `${JSON.stringify({ gateway: { url: "wss://gw.example:4700" } })}\n`, + { mode: 0o600 }, + ); + const tightenedExit = await runCli( + ["config", "set", "gateway.token", "slk_replaced"], + outputCapture(realCliEnvironmentPorts(tightened)).environmentWith(), + ); + + expect([freshExit, tightenedExit]).toEqual([0, 0]); + expect([ + JSON.parse(await fresh.readFile("/simlock/config.json")).gateway.token, + JSON.parse(await tightened.readFile("/simlock/config.json")).gateway.token, + ]).toEqual(["slk_fresh", "slk_replaced"]); + expect({ + created: (await fresh.stat("/simlock/config.json")).mode?.toString(8), + replaced: (await tightened.stat("/simlock/config.json")).mode?.toString(8), + }).toEqual({ created: "600", replaced: "600" }); + }); + + it("makes a config.json that others could read owner-only on the next config set", async () => { + // why: A file written by hand or by an older simlock: readable by group and others. `config set` + // must not carry that mode over to the file it writes in its place. + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic( + "/simlock/config.json", + `${JSON.stringify({ gateway: { url: "wss://gw.example:4700", token: "slk_old" } })}\n`, + { mode: 0o644 }, + ); + + const exit = await runCli( + ["config", "set", "gateway.label", "build-mac"], + outputCapture(realCliEnvironmentPorts(filesystem)).environmentWith(), + ); + + expect(exit).toBe(0); + expect(JSON.parse(await filesystem.readFile("/simlock/config.json")).gateway).toEqual({ + url: "wss://gw.example:4700", + token: "slk_old", + label: "build-mac", + }); + expect((await filesystem.stat("/simlock/config.json")).mode?.toString(8)).toBe("600"); + }); +}); + +describe("CLI: --json shape is the contract, as-is (ADR 0003 §11)", () => { + it("status --json carries camelCase fields, not expires_at_ms, estimated_boot_ms, queue_position or device_unhealthy", async () => { + const output = outputCapture(); + await runCli( + ["status", "--json"], + output.environmentWith({ connectAdmin: async () => fakeClient() }), + ); + const text = output.stdout; + expect(text).not.toMatch(/expires_at_ms|estimated_boot_ms|queue_position|device_unhealthy/); + const parsed = JSON.parse(text) as StatusGetOutput; + expect(parsed.queueDepth).toBe(0); + }); + + it("lease output is the contract's LeaseGrant plus the resolved role", async () => { + const output = outputCapture(); + await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith({ connectAdmin: async () => fakeClient({ role: "admin" }) }), + ); + const parsed = JSON.parse(output.stdout) as LeaseGrant & { role: string }; + expect(parsed.lease.id).toBe("lse_1"); + expect(parsed.role).toBe("admin"); + expect(output.stdout).not.toMatch(/expires_at_ms|estimated_boot_ms/); + }); +}); + +describe("CLI: --yes / confirm guards (safety rule 5)", () => { + it("release --all without --yes and a confirm that declines refuses with exit 2 and releases nothing", async () => { + const output = outputCapture(); + let released = false; + await expect( + runCli( + ["release", "--all"], + output.environmentWith({ + confirm: async () => false, + connectAdmin: async () => + fakeClient({ + releaseAllLeases: () => { + released = true; + return Promise.resolve({ leaseIds: [] }); + }, + }), + }), + ), + ).resolves.toBe(2); + expect(released).toBe(false); + }); + + it("release --all --yes bypasses the interactive confirm", async () => { + const output = outputCapture(); + let released = false; + await expect( + runCli( + ["release", "--all", "--yes"], + output.environmentWith({ + confirm: async () => false, + connectAdmin: async () => + fakeClient({ + releaseAllLeases: () => { + released = true; + return Promise.resolve({ leaseIds: ["lse_1"] }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(released).toBe(true); + }); + + it("nuke without confirmation or --yes refuses before connecting", async () => { + const output = outputCapture(); + let connected = false; + await expect( + runCli( + ["nuke"], + output.environmentWith({ + confirm: async () => false, + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ), + ).resolves.toBe(2); + expect(connected).toBe(false); + }); +}); diff --git a/src/cli/index-environment-helpers.test.ts b/src/cli/index-environment-helpers.test.ts new file mode 100644 index 000000000..f91b9dfa8 --- /dev/null +++ b/src/cli/index-environment-helpers.test.ts @@ -0,0 +1,166 @@ +import { Readable } from "node:stream"; +import { describe, expect, it, vi } from "vitest"; +import { + IpcError, + MemoryFilesystem, + MemoryIpcTransport, + resolveDaemonSocketPath, + type IpcConnector, +} from "../ports/index.js"; +import { runCli } from "./index.js"; +import { buildCliEnvironment } from "./environment.js"; +import { readPipedStdin } from "./stdio.js"; +import { parseDuration } from "./duration.js"; +import { fakeClient, outputCapture, realCliEnvironmentPorts } from "./test-index.js"; + +// why: a part of a decomposed module: these tests drive the environment builder, the duration +// parser and the stdin reader directly, the parts that own those behaviours (ADR 0063 section 2). +describe("CLI: the daemon endpoint check", () => { + it("buildCliEnvironment reports the daemon endpoint present only while its socket file exists", async () => { + const filesystem = new MemoryFilesystem(); + const environment = buildCliEnvironment(realCliEnvironmentPorts(filesystem), {}); + const socketPath = resolveDaemonSocketPath("/simlock", "linux"); + + expect(await environment.daemonEndpointPresent()).toBe(false); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(socketPath, ""); + expect(await environment.daemonEndpointPresent()).toBe(true); + await filesystem.rm(socketPath); + expect(await environment.daemonEndpointPresent()).toBe(false); + }); + + it("on Windows reports the named pipe present while something answers it, and not after it closes", async () => { + const ipc = new MemoryIpcTransport(); + const environment = buildCliEnvironment( + { ...realCliEnvironmentPorts(), ipc, hostPlatform: "win32" }, + {}, + ); + + expect(await environment.daemonEndpointPresent()).toBe(false); + const listener = await ipc.listen( + resolveDaemonSocketPath("/simlock", "win32"), + () => undefined, + ); + expect(await environment.daemonEndpointPresent()).toBe(true); + await listener.close(); + expect(await environment.daemonEndpointPresent()).toBe(false); + }); + + it("on Windows counts a pipe that does not answer in time as present, since only a refused or missing one proves it gone", async () => { + const connector: IpcConnector = { + connect: async () => { + throw new IpcError("connection-timeout", "timed out", undefined); + }, + }; + const environment = buildCliEnvironment( + { ...realCliEnvironmentPorts(), ipc: connector, hostPlatform: "win32" }, + {}, + ); + + expect(await environment.daemonEndpointPresent()).toBe(true); + }); +}); + +describe("CLI: requester id from the agent session", () => { + const sessionEnv = { CLAUDE_CODE_SESSION_ID: "abc" }; + + it("buildCliEnvironment falls back to the pid when no agent id or session id is set", () => { + expect(buildCliEnvironment(realCliEnvironmentPorts(), {}).requesterId).toBe( + String(process.pid), + ); + }); + + it("simlock lease --agent-id overrides a session-derived id", async () => { + const requested: string[] = []; + const client = fakeClient({ + requestLease: (input, _options) => { + requested.push(input.requesterId ?? ""); + return fakeClient().requestLease(input); + }, + }); + const output = outputCapture(); + const environment = { + ...buildCliEnvironment(realCliEnvironmentPorts(), sessionEnv), + connectAdmin: async () => client, + stderr: { write: (value: string) => (output.stderr += value) }, + stdout: { write: (value: string) => (output.stdout += value) }, + }; + + await expect( + runCli(["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], environment), + ).resolves.toBe(0); + await expect( + runCli( + [ + "lease", + "--platform", + "ios", + "--device", + "iPhone 17 Pro", + "--detach", + "--agent-id", + "explicit-agent", + ], + environment, + ), + ).resolves.toBe(0); + + expect(requested).toEqual(["claude-code:abc", "explicit-agent"]); + }); +}); + +describe("CLI: pure helpers", () => { + it("parseDuration parses units and rejects garbage", () => { + expect(parseDuration("500")).toBe(500); + expect(parseDuration("500ms")).toBe(500); + expect(parseDuration("2s")).toBe(2_000); + expect(parseDuration("3m")).toBe(180_000); + expect(parseDuration("1h")).toBe(3_600_000); + expect(() => parseDuration("banana")).toThrow("Invalid duration"); + }); + + it("parseDuration accepts the d unit and rejects an unknown one", () => { + expect(parseDuration("2d")).toBe(2 * 24 * 60 * 60 * 1000); + expect(() => parseDuration("2w")).toThrow("Invalid duration"); + }); +}); + +describe("readPipedStdin", () => { + function fakeStdin(chunks: readonly string[], isTTY: boolean): typeof process.stdin { + const stream = Readable.from(chunks); + return Object.assign(stream, { isTTY }) as unknown as typeof process.stdin; + } + + it("reads a terminal stdin to its end, joining its chunks", async () => { + const stdin = fakeStdin(["typed ", "text\n"], true); + const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); + + try { + await expect(readPipedStdin()).resolves.toBe("typed text\n"); + } finally { + stdinGetter.mockRestore(); + } + }); + + it("reads a piped stdin to EOF, concatenating every chunk in order", async () => { + const stdin = fakeStdin(["piped ", "pay", "load\n"], false); + const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); + + try { + await expect(readPipedStdin()).resolves.toBe("piped payload\n"); + } finally { + stdinGetter.mockRestore(); + } + }); + + it("returns an empty string for a piped stdin that carries nothing", async () => { + const stdin = fakeStdin([], false); + const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); + + try { + await expect(readPipedStdin()).resolves.toBe(""); + } finally { + stdinGetter.mockRestore(); + } + }); +}); diff --git a/src/cli/index-events-mcp-smoke.test.ts b/src/cli/index-events-mcp-smoke.test.ts new file mode 100644 index 000000000..e7fd0a3b9 --- /dev/null +++ b/src/cli/index-events-mcp-smoke.test.ts @@ -0,0 +1,758 @@ +import { EventEmitter } from "node:events"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { testComponentWiring, testStateOf, FakeDriver, loadTestRegistry } from "../core/testing.js"; +import { EventBus } from "../bus/index.js"; +import { ringEventHistory, seedHistory } from "../history/testing.js"; +import { CleanupReaper } from "../core/index.js"; +import { + CryptoTokenSecrets, + FakeClock, + FakeDaemonLauncher, + MemoryFilesystem, + FakeKeySetFetcher, + NodeFilesystem, + NodeIpcTransport, + NoopLogger, + FakeSystemStats, +} from "../ports/index.js"; +import { createAuth, tokensOwner } from "../auth/index.js"; +import { openMemoryStateDatabase } from "../auth/testing.js"; +import { DaemonEndpointHost } from "../daemon/connection-host.js"; +import { DaemonServer } from "../daemon/server.js"; +import { SimlockError } from "../contract/index.js"; +import type { EventPush, LeaseGrant, StatusGetOutput } from "../simlock-client/client.js"; +import { connectSimlockAdmin } from "../admin/index.js"; +import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; +import { createTestEngine } from "../leasing/testing.js"; +import { + fakeClient, + gibibyte, + outputCapture, + realCliEnvironmentPorts, + runningDaemons, + sequence, + settle, + temporaryDirectories, + testConfig, +} from "./test-index.js"; + +afterEach(async () => { + await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); + await Promise.all( + temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); +}); + +describe("CLI: mcp command (ADR 0003 §11 -- lazy module load)", () => { + it.each([ + [["--gateway", "mcp"], 0, "gateway"], + [["mcp", "--gateway"], 0, "gateway"], + [["mcp"], 0, "none"], + [["--gateway=yes", "mcp"], 2, "--gateway takes no value"], + [["--local", "mcp"], 0, "local"], + [["mcp", "--local"], 0, "local"], + [["--local=yes", "mcp"], 2, "--local takes no value"], + [["--local", "--gateway", "mcp"], 2, "--gateway and --local cannot be used together"], + [["mcp", "--gateway", "--local"], 2, "--gateway and --local cannot be used together"], + ] as const)( + "--gateway and --local are taken from anywhere in the argv and reach the command as context.daemonFlag; = and both together are usage errors (%j)", + async (argv, exitCode, expected) => { + const output = outputCapture(); + const seen: string[] = []; + + const code = await runCli( + argv, + output.environmentWith({ + runMcpStdio: async (options) => { + seen.push(options.daemonFlag ?? "none"); + }, + }), + ); + + expect(code).toBe(exitCode); + if (exitCode === 0) { + expect(seen).toEqual([expected]); + } else { + expect(JSON.parse(output.stderr)).toEqual({ error: { code: "USAGE", message: expected } }); + expect(seen).toEqual([]); + } + }, + ); + + it("does not load the MCP stdio module unless the mcp command runs", async () => { + const output = outputCapture(); + let loaded = false; + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient(), + loadMcpStdio: async () => { + loaded = true; + return async () => {}; + }, + }), + ); + expect(loaded).toBe(false); + }); + + it("loads and runs the stdio runner when the mcp command is dispatched", async () => { + const output = outputCapture(); + let ran = false; + const exitCode = await runCli( + ["mcp"], + output.environmentWith({ + loadMcpStdio: async () => async () => { + ran = true; + }, + }), + ); + expect(exitCode).toBe(0); + expect(ran).toBe(true); + }); + + it("reports a startup failure as a structured error instead of throwing raw", async () => { + const output = outputCapture(); + const exitCode = await runCli( + ["mcp"], + output.environmentWith({ + loadMcpStdio: async () => { + throw new Error("cannot bind mcp stdio transport"); + }, + }), + ); + expect(exitCode).toBe(1); + expect(JSON.parse(output.stderr)).toEqual({ + error: { code: "INTERNAL", message: "cannot bind mcp stdio transport" }, + }); + }); +}); + +describe("CLI: events history", () => { + const hour = 3_600_000; + + function envelope(seq: number, timestamp: number, id = `evt_${seq}`) { + return { id, seq, timestamp, event: "lease.granted", payload: { seq }, module: "leases" }; + } + + function printed(stdout: string): unknown[] { + if (stdout.trim() === "") return []; + return stdout + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line) as unknown); + } + + async function historyDirectory( + prepare: (directory: string) => Promise, + ): Promise { + const directory = await mkdtemp(join(tmpdir(), "simlock-cli-history-")); + temporaryDirectories.push(directory); + await prepare(directory); + return { + ...realCliEnvironmentPorts(new NodeFilesystem()), + clock: new FakeClock(2 * hour), + daemonHome: { role: "worker", dataDirectory: directory, gatewayPresent: false }, + }; + } + + const stopped = (seq: number, timestamp: number) => ({ + event: "daemon.stopping", + id: `evt_${seq}`, + module: "daemon", + payload: { reason: `r${seq}` }, + seq, + timestamp, + }); + + it("events --since with nothing listening reads history.db and launches no daemon", async () => { + const ports = await historyDirectory((directory) => seedHistory(directory, [500, hour + 500])); + const launcher = new FakeDaemonLauncher(() => { + throw new Error("a daemon was launched"); + }); + const output = outputCapture({ ...ports, launcher }); + + const exitCode = await runCli(["events", "--since", "1h"], output.environmentWith()); + + expect(launcher.launches).toBe(0); + expect(exitCode).toBe(0); + expect(printed(output.stdout)).toEqual([stopped(2, hour + 500)]); + }); + + it("events --since with nothing listening and no history.db prints nothing, exit 0, even beside an old events.jsonl", async () => { + const bare = outputCapture(await historyDirectory(async () => {})); + const withOldFile = outputCapture( + await historyDirectory((directory) => + writeFile(join(directory, "events.jsonl"), `${JSON.stringify(envelope(1, hour + 500))}\n`), + ), + ); + + const codes = [ + await runCli(["events", "--since", "1h"], bare.environmentWith()), + await runCli(["events", "--since", "1h"], withOldFile.environmentWith()), + ]; + + expect(codes).toEqual([0, 0]); + expect([bare.stdout, withOldFile.stdout]).toEqual(["", ""]); + }); + + it("events --since with nothing listening and an unreadable history.db prints one JSON error and exits 1", async () => { + const output = outputCapture( + await historyDirectory((directory) => + writeFile(join(directory, "history.db"), "this is not a sqlite database ".repeat(20)), + ), + ); + + const exitCode = await runCli(["events", "--since", "1h"], output.environmentWith()); + + expect(exitCode).toBe(1); + expect(output.stdout).toBe(""); + expect(output.stderr.trimEnd().split("\n")).toHaveLength(1); + const printedError = JSON.parse(output.stderr) as { error: { code: string; message: string } }; + expect(printedError.error.code).toBe("INTERNAL"); + expect(printedError.error.message).toMatch(/not a database/); + }); + + it("events --since reports a daemon that refuses the handshake rather than reading the file", async () => { + const output = outputCapture(); + let fileRead = false; + const exitCode = await runCli( + ["events", "--since", "1h"], + output.environmentWith({ + connectExistingAdmin: async () => { + throw new SimlockError("PROTOCOL_VERSION_UNSUPPORTED", "protocol", "no overlap", { + client: { min: 5, max: 5 }, + daemon: { min: 4, max: 4 }, + daemonVersion: "0.9.0", + }); + }, + readEventHistory: async () => { + fileRead = true; + return []; + }, + }), + ); + + expect(exitCode).toBe(1); + expect(fileRead).toBe(false); + expect(output.stderr).toContain("PROTOCOL_VERSION_UNSUPPORTED"); + }); + + it("events --since with a daemon listening replays through the daemon", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + let fileRead = false; + const exitCode = await runCli( + ["events", "--since", "1h"], + output.environmentWith({ + clock: new FakeClock(2 * hour), + connectExistingAdmin: async () => + fakeClient({ + replayEvents: async (input) => { + asked.push(input); + return [envelope(7, hour + 1)]; + }, + }), + readEventHistory: async () => { + fileRead = true; + return []; + }, + }), + ); + + expect(exitCode).toBe(0); + expect(asked).toEqual([{ sinceTs: hour }]); + expect(printed(output.stdout)).toEqual([envelope(7, hour + 1)]); + expect(fileRead).toBe(false); + }); + + it("simlock events --follow prints an event pushed during the replay once", async () => { + const output = outputCapture(); + const signals = new EventEmitter(); + let listener: ((push: EventPush) => void) | undefined; + const push = (seq: number): EventPush => ({ + subscriptionId: "sub_1", + event: envelope(seq, seq), + }); + const runPromise = runCli( + ["events", "--follow"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + subscribeEvents: async (onEvent) => { + listener = onEvent; + return async () => {}; + }, + replayEvents: async () => { + // why: Emitted after the subscribe and before the replay answers: the replay holds + // seq 2, and seq 3 lands just after its snapshot. + listener?.(push(2)); + listener?.(push(3)); + // why: Same seq and timestamp as the replayed event 2, another id: another event. + listener?.({ subscriptionId: "sub_1", event: envelope(2, 2, "evt_other") }); + return [envelope(1, 1), envelope(2, 2)]; + }, + }), + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + listener?.(push(4)); + signals.emit("SIGINT"); + + expect(await runPromise).toBe(0); + expect(printed(output.stdout)).toEqual([ + envelope(1, 1), + envelope(2, 2), + push(3), + { subscriptionId: "sub_1", event: envelope(2, 2, "evt_other") }, + push(4), + ]); + }); + + it("events --since --follow prints the history once and then the live events", async () => { + const output = outputCapture(); + const signals = new EventEmitter(); + const asked: unknown[] = []; + let listener: ((push: EventPush) => void) | undefined; + const push = (seq: number): EventPush => ({ + subscriptionId: "sub_1", + event: envelope(seq, seq), + }); + const runPromise = runCli( + ["events", "--since", "1h", "--follow"], + output.environmentWith({ + clock: new FakeClock(2 * hour), + connectAdmin: async () => + fakeClient({ + subscribeEvents: async (onEvent) => { + listener = onEvent; + return async () => {}; + }, + replayEvents: async (input) => { + asked.push(input); + listener?.(push(2)); + return [envelope(1, 1), envelope(2, 2)]; + }, + }), + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + listener?.(push(3)); + signals.emit("SIGINT"); + + expect(await runPromise).toBe(0); + expect(asked).toEqual([{ sinceTs: hour }]); + expect(printed(output.stdout)).toEqual([envelope(1, 1), envelope(2, 2), push(3)]); + }); + + it("events --follow without --since asks for no sinceTs", async () => { + const output = outputCapture(); + const signals = new EventEmitter(); + const asked: unknown[] = []; + const runPromise = runCli( + ["events", "--follow"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + replayEvents: async (input) => { + asked.push(input); + return []; + }, + }), + signals: signals as unknown as CliEnvironment["signals"], + }), + ); + await settle(); + signals.emit("SIGINT"); + + expect(await runPromise).toBe(0); + expect(asked).toEqual([{}]); + }); +}); + +describe("CLI: daemon logs (ADR 0003 §11 -- must work when the daemon is dead)", () => { + it("reads the log file without connecting to the daemon at all", async () => { + const output = outputCapture(); + let connected = false; + const exitCode = await runCli( + ["daemon", "logs"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + connectExistingAdmin: async () => { + connected = true; + return fakeClient(); + }, + readLogFile: async () => "line one\nline two\n", + }), + ); + expect(exitCode).toBe(0); + expect(connected).toBe(false); + expect(output.stdout).toBe("line one\nline two\n"); + }); + + it("--json wraps the log tail under a logs key, still with no connection attempted", async () => { + const output = outputCapture(); + let connected = false; + const exitCode = await runCli( + ["daemon", "logs", "--json"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + connectExistingAdmin: async () => { + connected = true; + return fakeClient(); + }, + readLogFile: async () => "only line\n", + }), + ); + expect(exitCode).toBe(0); + expect(connected).toBe(false); + expect(JSON.parse(output.stdout)).toEqual({ logs: "only line" }); + }); +}); + +describe("CLI: daemon logs --follow", () => { + async function follow(filesystem: MemoryFilesystem) { + const clock = new FakeClock(0); + const signals = new EventEmitter(); + const ports = { + ...realCliEnvironmentPorts(filesystem), + clock, + signals: signals as unknown as NonNullable, + }; + const output = outputCapture(ports); + const exit = runCli(["daemon", "logs", "--follow"], output.environmentWith()); + // why: `followLogFile` loads its follower module on first use, so settling once is no + // longer enough for the tail read to have started; this yields to the event loop until the + // poll timer is armed, which is the point the old single settle proved the follower was + // live. The wait is a real timer: the lazy import resolves through I/O callbacks, which + // `setImmediate` turns alone would not drain. + const settle = async (): Promise => { + for (let attempt = 0; attempt < 200; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 0)); + if (clock.pendingTimerCount > 0) return; + } + throw new Error("the follower never armed its poll timer -- lazy load of the module hung"); + }; + await settle(); + /** why: unlike the first settle this cannot require a timer -- after a signal the follower + * is meant to leave none armed -- and the lazy import is resolved by now, so immediate + * turns are enough to drain the poll's reads. */ + const settleTick = async (): Promise => { + for (let attempt = 0; attempt < 20; attempt += 1) + await new Promise((resolve) => setImmediate(resolve)); + }; + return { + clock, + exit, + output, + signals, + settle, + tick: async () => { + clock.advance(250); + await settleTick(); + }, + }; + } + + const LOG = "/simlock/daemon.log"; + + it("daemon logs --follow prints the existing tail, then a line appended later", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(LOG, "old one\nold two\n"); + const run = await follow(filesystem); + + expect(run.output.stdout).toBe("old one\nold two\n"); + await filesystem.writeFileAtomic(LOG, "old one\nold two\nnew three\n"); + await run.tick(); + + expect(run.output.stdout).toBe("old one\nold two\nnew three\n"); + run.signals.emit("SIGINT"); + await expect(run.exit).resolves.toBe(0); + }); + + it.each([ + ["ESC", "\u001b", "\\u001b"], + ["C1 CSI", "\u009b", "\\u009b"], + ])( + "prints %s in a log line escaped, in the tail and in a later line", + async (_name, raw, escaped) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(LOG, `a${raw}b\n`); + const run = await follow(filesystem); + await filesystem.writeFileAtomic(LOG, `a${raw}b\nc${raw}d\n`); + await run.tick(); + + expect(run.output.stdout).toBe(`a${escaped}b\nc${escaped}d\n`); + run.signals.emit("SIGINT"); + await expect(run.exit).resolves.toBe(0); + }, + ); + + it("it prints lines written after a rotation, when the new file is already larger than the old offset at the next poll", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(LOG, "a\n"); + const run = await follow(filesystem); + + // why: Between two polls: one more line in the old file, a rotation, and a new file already + // longer than everything the follower had read of the old one. + await filesystem.writeFileAtomic(LOG, "a\nb\n"); + await filesystem.rename(LOG, `${LOG}.1`); + await filesystem.writeFileAtomic(LOG, "after rotation 1\nafter rotation 2\n"); + await run.tick(); + + expect(run.output.stdout).toBe("a\nb\nafter rotation 1\nafter rotation 2\n"); + run.signals.emit("SIGTERM"); + await expect(run.exit).resolves.toBe(0); + }); + + it("it waits while the log file is missing and prints lines once it appears", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + const run = await follow(filesystem); + await run.tick(); + expect(run.output.stdout).toBe(""); + expect(run.output.stderr).toBe(""); + + await filesystem.writeFileAtomic(LOG, "first line\n"); + await run.tick(); + + expect(run.output.stdout).toBe("first line\n"); + run.signals.emit("SIGINT"); + await expect(run.exit).resolves.toBe(0); + }); + + it("daemon logs --follow --json fails with USAGE, exit 2", async () => { + const output = outputCapture(); + + await expect( + runCli(["daemon", "logs", "--follow", "--json"], output.environmentWith()), + ).resolves.toBe(2); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("--follow") }, + }); + }); + + it("--follow on any daemon subcommand but logs fails with USAGE, exit 2", async () => { + for (const command of ["start", "stop", "status"]) { + const output = outputCapture(); + await expect(runCli(["daemon", command, "--follow"], output.environmentWith())).resolves.toBe( + 2, + ); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("--follow") }, + }); + } + }); + + it("an interrupt ends the follow with exit 0 and leaves no timer armed", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/simlock"); + await filesystem.writeFileAtomic(LOG, "line\n"); + const run = await follow(filesystem); + expect(run.clock.pendingTimerCount).toBe(1); + + run.signals.emit("SIGINT"); + + await expect(run.exit).resolves.toBe(0); + expect(run.clock.pendingTimerCount).toBe(0); + expect(run.signals.listenerCount("SIGINT")).toBe(0); + expect(run.signals.listenerCount("SIGTERM")).toBe(0); + await filesystem.writeFileAtomic(LOG, "line\nafter exit\n"); + await run.tick(); + expect(run.output.stdout).toBe("line\n"); + }); +}); + +describe("CLI smoke test (ADR 0003 §12: one per frontend)", () => { + it("lease --detach, status, release, and token create round-trip through a real daemon", async () => { + const { socketPath } = await startTestDaemon(); + const ipc = new NodeIpcTransport(); + const environment = outputCapture().environmentWith({ + connectAdmin: async (resolveCredential) => { + const credential = await resolveCredential(); + return connectSimlockAdmin({ + ipc, + endpoint: socketPath, + principal: "smoke-test", + ...(credential === undefined ? {} : { credential }), + }); + }, + }); + + const leaseOut = outputCapture(); + const leaseExit = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + leaseOut.environmentWith({ connectAdmin: environment.connectAdmin }), + ); + expect(leaseExit).toBe(0); + const grant = JSON.parse(leaseOut.stdout) as LeaseGrant; + // why: ADR 0004: one kind of lease -- what `--detach` changes is this process, not the grant. + expect(grant.lease).toMatchObject({ ttlMs: expect.any(Number) }); + + const statusOut = outputCapture(); + await runCli( + ["status", "--json"], + statusOut.environmentWith({ connectAdmin: environment.connectAdmin }), + ); + const status = JSON.parse(statusOut.stdout) as StatusGetOutput; + expect(status.leases?.map((lease) => lease.id)).toContain(grant.lease.id); + + const releaseOut = outputCapture(); + const releaseExit = await runCli( + ["release", grant.lease.id], + releaseOut.environmentWith({ connectAdmin: environment.connectAdmin }), + ); + expect(releaseExit).toBe(0); + + const tokenOut = outputCapture(); + const tokenExit = await runCli( + ["token", "create", "--role", "operator", "--label", "ci"], + tokenOut.environmentWith({ connectAdmin: environment.connectAdmin }), + ); + expect(tokenExit).toBe(0); + const created = JSON.parse(tokenOut.stdout) as { secret: string; token: { id: string } }; + expect(typeof created.secret).toBe("string"); + expect((created as unknown as Record).token).not.toHaveProperty("hash"); + }); +}); + +/** + * why: the CLI reference promises exactly one structured error line per failure, and binds `USAGE` to + * exit 2. A `SIMLOCK_HOME` whose socket path exceeds the platform limit had neither: the path + * was resolved while *building* the environment, and `runCli` takes its environment as a + * default parameter -- whose initializer runs before the function body, and therefore before + * the try/catch. The error escaped as an uncaught rejection with a raw stack trace, and took + * `simlock --help` down with it. + */ +describe("CLI: a SIMLOCK_HOME the kernel could not bind", () => { + const tooDeep = `/tmp/${"d".repeat(120)}`; + const ports = (): CliEnvironmentPorts => ({ + ...realCliEnvironmentPorts(), + daemonHome: { role: "worker", dataDirectory: tooDeep, gatewayPresent: false }, + }); + + it("still prints help, which needs no socket at all", async () => { + const output = outputCapture(ports()); + + await expect(runCli(["--help"], output.environmentWith())).resolves.toBe(0); + + expect(output.stdout).toContain("Usage: simlock"); + expect(output.stderr).toBe(""); + }); + + it("reports a command that does need one as a single USAGE line, exit 2", async () => { + const output = outputCapture(ports()); + + await expect(runCli(["status"], output.environmentWith())).resolves.toBe(2); + + const lines = output.stderr.trimEnd().split("\n"); + expect(lines).toHaveLength(1); + const reported = JSON.parse(lines[0] ?? "") as { + readonly error: { readonly code: string; readonly message: string }; + }; + expect(reported.error.code).toBe("USAGE"); + expect(reported.error.message).toContain("SIMLOCK_HOME"); + }); +}); + +async function startTestDaemon(): Promise<{ socketPath: string; daemon: DaemonServer }> { + const directory = await mkdtemp(join(tmpdir(), "simlock-cli-smoke-")); + temporaryDirectories.push(directory); + const socketPath = join(directory, "daemon.sock"); + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const filesystem = new MemoryFilesystem(); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const config = testConfig(); + const wiring = testComponentWiring({ + clock: clock, + drivers: [driver], + eventBus: eventBus, + registry: registry, + }); + const engine = createTestEngine({ + ...wiring, + clock, + config, + drivers: [driver], + eventBus, + idGenerator: sequence(), + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + }); + const reaper = new CleanupReaper({ + clock, + config, + eventBus, + executor: engine.cleanup, + filesystem, + registry, + }); + const auth = createAuth({ + clock, + config: { enabled: false, host: "127.0.0.1", port: 4700 }, + dataDirectory: "/", + database: (await openMemoryStateDatabase([tokensOwner])).database, + filesystem, + idGenerator: sequence(), + logger: new NoopLogger(), + keySetFetcher: new FakeKeySetFetcher(), + secrets: new CryptoTokenSecrets(), + }); + const daemon = new DaemonServer({ + capacity: engine, + warmPool: engine, + deviceModes: engine, + catalog: engine, + instanceId: "instance-test", + clock, + components: wiring.components, + config, + defaultRequesterId: "test-process", + eventBus, + eventHistory: ringEventHistory(eventBus), + host: new DaemonEndpointHost({ + connector: new NodeIpcTransport(), + endpoint: socketPath, + filesystem: new NodeFilesystem(), + listenerFactory: new NodeIpcTransport(), + }), + hostFacts: () => ({ arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }), + leases: engine, + queue: engine, + reaper, + registry, + resolveRole: { resolve: () => "admin" }, + tokens: auth.tokens, + version: "test", + }); + runningDaemons.push(daemon); + await daemon.start(); + return { socketPath, daemon }; +} + +/** why: Bare-minimum ports for exercising `buildCliEnvironment`'s real `validateConfig` (B9) without + * a daemon connection -- `config set` never calls `connectAdmin`, so `ipc`/`launcher` are inert + * placeholders. */ diff --git a/src/cli/index-exit-codes-passthrough.test.ts b/src/cli/index-exit-codes-passthrough.test.ts new file mode 100644 index 000000000..1f7ca66c0 --- /dev/null +++ b/src/cli/index-exit-codes-passthrough.test.ts @@ -0,0 +1,685 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { FakeParentWatch } from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import type { LeaseGrant } from "../simlock-client/client.js"; +import { runCli, type CliEnvironment } from "./index.js"; +import { EMPTY_STATUS, detachedGrant, fakeClient, outputCapture } from "./test-index.js"; + +describe("CLI: exit codes", () => { + /** + * why: ADR 0005 §19c's other half. Same command, same refusals, same exit code -- the only thing + * that changes is where the process runs and how its output gets here. `runPassthrough` is + * deliberately wired in each of these: a remote invocation that quietly spawned locally + * would pass every assertion about output and exit code while doing the wrong thing. + * + * The switch is the daemon's own `mode` (§19c): a `gateway` owns no devices, so a command + * it resolved could not run here anyway. Nothing about the CLI's transport decides it. + */ + describe("remote passthrough (ADR 0005 device.exec)", () => { + const gatewayStatus = { + ...EMPTY_STATUS, + daemon: { + health: "running" as const, + mode: "gateway" as const, + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + }; + const ownLease = { + deviceId: "dev_1", + grantedAt: 0, + id: "lse_mine", + lastRenewedAt: 0, + idChosenByRequester: false, + ownerId: "some-other-principal", + requesterId: "test-requester", + ttlDeadline: 60_000, + ttlMs: 60_000, + }; + + it("runs the command on the daemon, prints the streamed output, and exits with its code", async () => { + const output = outputCapture(); + const seen: unknown[] = []; + let ranLocally = false; + + await expect( + runCli( + ["adb", "shell", "getprop"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: (input, options) => { + seen.push(input); + options?.onOutput?.({ chunk: "ro.build", stream: "stdout" }); + options?.onOutput?.({ chunk: "a warning\n", stream: "stderr" }); + options?.onOutput?.({ chunk: ".version=15\n", stream: "stdout" }); + return Promise.resolve({ exitCode: 9 }); + }, + }), + runPassthrough: async () => { + ranLocally = true; + return 0; + }, + }), + ), + ).resolves.toBe(9); + + expect(ranLocally).toBe(false); + // why: `requesterId` rides along because this connection resolved to the admin role, where + // the operation reads it (and requires it to match the lease). See the agent-role case + // below for the other half. + expect(seen).toEqual([ + { + args: ["shell", "getprop"], + leaseId: "lse_mine", + requesterId: "test-requester", + stdin: "", + tool: "adb", + }, + ]); + // why: Chunks land on the stream they were written to, unjoined and in order -- a caller + // piping this sees what it would have seen locally. + expect(output.stdout).toBe("ro.build.version=15\n"); + // why: The agent-fallback notice this fixture's credential-less connection writes is the only + // other thing on stderr; the command's own chunk lands there verbatim. + expect(output.stderr).toContain("a warning\n"); + }); + + it("names the lease from the requester id rather than from a flag of its own", async () => { + // why: Every argument after the tool name belongs to the tool, so identity is + // the only thing left to select a lease with -- and `--agent-id`/`SIMLOCK_AGENT_ID` + // already is that identity. A requester holding no lease is a usage error naming it. + const output = outputCapture(); + + await expect( + runCli( + ["adb", "devices"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => + Promise.resolve({ + leases: [{ ...ownLease, id: "lse_theirs", requesterId: "someone-else" }], + }), + }), + }), + ), + ).resolves.toBe(2); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("test-requester") }, + }); + }); + + it("exits 10 when the daemon killed the command for outrunning exec.timeoutMs", async () => { + // why: The exit code comes from `ERROR_TABLE`'s own column (ADR 0003 §7): 10, the same code + // the other "you ran out of time" outcome (`QUEUE_TIMEOUT`) uses, rather than the + // generic 1 a command that merely failed would get. + const output = outputCapture(); + + await expect( + runCli( + ["adb", "logcat"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: () => + Promise.reject( + new SimlockError( + "EXEC_TIMEOUT", + "domain", + "`adb logcat` exceeded exec.timeoutMs (600000ms) and was killed", + {}, + ), + ), + }), + }), + ), + ).resolves.toBe(10); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "EXEC_TIMEOUT", message: expect.stringContaining("exec.timeoutMs") }, + }); + }); + + it("without --lease, for a requester holding two leases, fails with a usage error naming both leases and --lease", async () => { + const output = outputCapture(); + + await expect( + runCli( + ["simctl", "list"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => + Promise.resolve({ + leases: [ownLease, { ...ownLease, deviceId: "dev_2", id: "lse_other" }], + }), + exec: () => Promise.reject(new Error("must not run")), + }), + }), + ), + ).resolves.toBe(2); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { + code: "USAGE", + message: expect.stringMatching(/lse_mine.*lse_other.*--lease /s), + }, + }); + }); + + it("takes `--lease ` out of the arguments and runs against that lease", async () => { + // why: The flag exists only on this path, so it is stripped here rather than parsed with the + // CLI's own flags: locally every argument after the tool name belongs to the tool. + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "--lease", "lse_named", "shell", "getprop"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.reject(new Error("must not need the lease list")), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([ + { + args: ["shell", "getprop"], + leaseId: "lse_named", + requesterId: "test-requester", + stdin: "", + tool: "adb", + }, + ]); + }); + + it("sends stdin on device.exec only when --stdin was given", async () => { + // why: `readStdin` is a `CliEnvironmentPorts` seam this test injects directly -- it exercises + // the wiring from that port's result to `device.exec`'s `stdin`, not the default + // `readPipedStdin` implementation that reads a real pipe to EOF; that one is tested on + // its own below, since a fake here would not exercise it at all. + const cases = [ + { + argv: ["adb", "shell", "cat"], + readable: false, + expected: { args: ["shell", "cat"], stdin: "", tool: "adb" }, + }, + { + argv: ["simctl", "list"], + readable: false, + expected: { args: ["list"], stdin: "", tool: "simctl" }, + }, + { + argv: ["adb", "--stdin", "shell", "cat"], + readable: true, + expected: { args: ["shell", "cat"], stdin: "piped payload\n", tool: "adb" }, + }, + { + argv: ["simctl", "--stdin", "list"], + readable: true, + expected: { args: ["list"], stdin: "piped payload\n", tool: "simctl" }, + }, + ]; + + for (const { argv, readable, expected } of cases) { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + argv, + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + readStdin: readable + ? async () => "piped payload\n" + : () => Promise.reject(new Error("stdin must not be read without --stdin")), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ ...expected, leaseId: "lse_mine", requesterId: "test-requester" }]); + } + }); + + it("sends no requesterId when the connection fell back to the agent role", async () => { + // why: An agent session is gated on the lease it owns and its `requesterId` is ignored + // entirely, so sending one would be noise that reads like authorization. + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "devices"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + role: "agent", + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ args: ["devices"], leaseId: "lse_mine", stdin: "", tool: "adb" }]); + }); + + it("spawns locally against a worker, accepting and ignoring --lease", async () => { + // why: The other half of the same switch, and the reason `--lease` is accepted on both: one + // command line has to work against either kind of daemon. A worker takes + // the flag, drops it, and passes the rest to the tool untouched. + const output = outputCapture(); + const ran: unknown[] = []; + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "--lease", "lse_ignored", "shell", "getprop"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + exec: () => Promise.reject(new Error("must not exec remotely")), + resolvePassthrough: (input) => { + seen.push(input); + return Promise.resolve({ args: ["-P", "5038"], command: "adb", env: {} }); + }, + }), + runPassthrough: async (command) => { + ran.push(command); + return 0; + }, + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ args: ["shell", "getprop"], tool: "adb" }]); + expect(ran).toHaveLength(1); + }); + + it("relabels an unknown passthrough tool as USAGE, like a refusal", async () => { + // why: ADR 0005 §19c: the CLI keeps no copy of the refusal list and relabels whatever the + // daemon answers. `UNKNOWN_PASSTHROUGH_TOOL` is the other half of that answer, and it + // has to read the same as a refused verb does -- exit 2, one `USAGE` line. + const output = outputCapture(); + + await expect( + runCli( + ["adb", "devices"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: () => + Promise.reject( + new SimlockError( + "UNKNOWN_PASSTHROUGH_TOOL", + "domain", + "No driver provides a adb passthrough", + { tool: "adb" }, + ), + ), + }), + }), + ), + ).resolves.toBe(2); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("No driver provides") }, + }); + }); + + it("takes --lease as the last argument, and refuses an empty one", async () => { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "--lease", "lse_only"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([ + { args: [], leaseId: "lse_only", requesterId: "test-requester", stdin: "", tool: "adb" }, + ]); + + const missingValue = outputCapture(); + await expect( + runCli( + ["adb", "--lease"], + missingValue.environmentWith({ + connectAdmin: async () => + fakeClient({ getStatus: () => Promise.resolve(gatewayStatus) }), + }), + ), + ).resolves.toBe(2); + + const emptyValue = outputCapture(); + await expect( + runCli( + ["adb", "--lease=", "devices"], + emptyValue.environmentWith({ + connectAdmin: async () => + fakeClient({ getStatus: () => Promise.resolve(gatewayStatus) }), + }), + ), + ).resolves.toBe(2); + }); + + it("fails a local passthrough with the status.get error when the daemon cannot answer", async () => { + // why: The switch reads `mode` off `status.get`, so an unreachable daemon fails there rather + // than at the passthrough -- and the CLI reports that call's own error rather than + // dressing it up as a passthrough failure. + const output = outputCapture(); + let ranLocally = false; + + await expect( + runCli( + ["adb", "devices"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => + Promise.reject( + new SimlockError( + "DAEMON_CONNECTION_LOST", + "transport", + "Daemon connection closed", + {}, + ), + ), + }), + runPassthrough: async () => { + ranLocally = true; + return 0; + }, + }), + ), + ).resolves.toBe(1); + expect(ranLocally).toBe(false); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "DAEMON_CONNECTION_LOST", message: "Daemon connection closed" }, + }); + }); + + it("uses an agent connection's own lease list without filtering it by requester", async () => { + // why: `lease.list` already answers per role: an agent sees only what it owns. Filtering that + // by `requesterId` as well would drop a lease it demonstrably owns whenever the two + // differ -- an `--agent-id` used on the lease but not on this invocation. + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "devices"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + role: "agent", + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => + Promise.resolve({ + leases: [{ ...ownLease, id: "lse_owned", requesterId: "another-agent-id" }], + }), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ args: ["devices"], leaseId: "lse_owned", stdin: "", tool: "adb" }]); + }); + + it("takes --stdin before or after --lease, and passes it to the tool after the tool's first argument", async () => { + const cases = [ + { + argv: ["adb", "--stdin", "--lease", "lse_named", "shell", "cat"], + expected: { args: ["shell", "cat"], leaseId: "lse_named", stdin: "sent" }, + }, + { + argv: ["adb", "--lease", "lse_named", "--stdin", "shell", "cat"], + expected: { args: ["shell", "cat"], leaseId: "lse_named", stdin: "sent" }, + }, + { + argv: ["adb", "shell", "--stdin"], + expected: { args: ["shell", "--stdin"], leaseId: "lse_mine", stdin: "" }, + }, + ]; + + for (const { argv, expected } of cases) { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + argv, + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.resolve({ leases: [ownLease] }), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + readStdin: async () => "sent", + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ ...expected, requesterId: "test-requester", tool: "adb" }]); + } + }); + + it("spawns locally against a worker, accepting and ignoring --stdin", async () => { + const cases = [ + { argv: ["adb", "--stdin", "shell", "cat"], args: ["shell", "cat"] }, + { argv: ["adb", "--stdin", "--lease", "lse_x", "shell", "cat"], args: ["shell", "cat"] }, + { argv: ["adb", "--lease", "lse_x", "--stdin", "shell", "cat"], args: ["shell", "cat"] }, + { argv: ["adb", "shell", "--stdin"], args: ["shell", "--stdin"] }, + ]; + + for (const { argv, args } of cases) { + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + argv, + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + exec: () => Promise.reject(new Error("must not exec remotely")), + resolvePassthrough: (input) => { + seen.push(input); + return Promise.resolve({ args: [], command: "adb", env: {} }); + }, + }), + readStdin: () => Promise.reject(new Error("a worker never reads stdin")), + runPassthrough: async () => 0, + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([{ args, tool: "adb" }]); + } + }); + + it("stops scanning for --lease at the tool's own first argument", async () => { + // why: Everything from the tool's subcommand onwards is the tool's, so a `--lease` there is + // an argument to *it* -- typed text, a filter, an operand -- and travels through. + const output = outputCapture(); + const seen: unknown[] = []; + + await expect( + runCli( + ["adb", "--lease", "lse_mine", "shell", "input", "text", "--lease"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve(gatewayStatus), + listLeases: () => Promise.reject(new Error("must not need the lease list")), + exec: (input) => { + seen.push(input); + return Promise.resolve({ exitCode: 0 }); + }, + }), + }), + ), + ).resolves.toBe(0); + expect(seen).toEqual([ + { + args: ["shell", "input", "text", "--lease"], + leaseId: "lse_mine", + requesterId: "test-requester", + stdin: "", + tool: "adb", + }, + ]); + }); + }); + + it("lists both tool passthroughs in root help", async () => { + const output = outputCapture(); + + await expect(runCli([], output.environmentWith())).resolves.toBe(0); + expect(output.stdout).toContain("simctl "); + expect(output.stdout).toContain("adb "); + }); + + it("releases and exits when the watched parent process dies, via the same path as a signal", async () => { + const output = outputCapture(); + const parentWatch = new FakeParentWatch(); + const heldGrant: LeaseGrant = { + ...detachedGrant, + lease: { ...detachedGrant.lease, id: "lse_parent_death" }, + }; + const released: unknown[] = []; + + const run = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + requestLease: (_input, _options) => Promise.resolve(heldGrant), + releaseLease: (input) => { + released.push(input); + return Promise.resolve({ leaseId: input.leaseId }); + }, + }), + parentPid: 4321, + parentWatch, + signals: new EventEmitter() as unknown as CliEnvironment["signals"], + }), + ); + + await vi.waitFor(() => expect(output.stdout).not.toBe("")); + parentWatch.exit(4321); + + await expect(run).resolves.toBe(0); + expect(released).toEqual([{ leaseId: "lse_parent_death" }]); + }); + + it("--bind-pid overrides which pid the CLI watches for parent death", async () => { + const output = outputCapture(); + const parentWatch = new FakeParentWatch(); + const heldGrant: LeaseGrant = { + ...detachedGrant, + lease: { ...detachedGrant.lease, id: "lse_bind_pid" }, + }; + const released: unknown[] = []; + + const run = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "9876"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + requestLease: (_input, _options) => Promise.resolve(heldGrant), + releaseLease: (input) => { + released.push(input); + return Promise.resolve({ leaseId: input.leaseId }); + }, + }), + parentPid: 4321, + parentWatch, + signals: new EventEmitter() as unknown as CliEnvironment["signals"], + }), + ); + + await vi.waitFor(() => expect(output.stdout).not.toBe("")); + // why: The captured parent is not the one being watched any more, so its death is ignored. + parentWatch.exit(4321); + expect(released).toEqual([]); + parentWatch.exit(9876); + + await expect(run).resolves.toBe(0); + expect(released).toEqual([{ leaseId: "lse_bind_pid" }]); + }); + + it("rejects a non-numeric --bind-pid as a structured USAGE error", async () => { + const output = outputCapture(); + + await expect( + runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "nope"], + output.environmentWith(), + ), + ).resolves.toBe(2); + expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "USAGE" } }); + }); + + // why: the CLI reference promises a missing required argument ends with the `simlock --help` + // pointer, and guards reach it two ways -- an explicit `withHelpHint` call or + // `requiredPositional` -- either of which can be dropped silently, so a sample of both + // paths (a command guard and the router's unknown command) is pinned here. + it("points a missing required argument at simlock --help", async () => { + const output = outputCapture(); + await expect(runCli(["lease"], output.environmentWith())).resolves.toBe(2); + expect(JSON.parse(output.stderr).error).toMatchObject({ + code: "USAGE", + message: "lease requires --platform (run `simlock --help` for usage)", + }); + }); + + it("points an unknown command at simlock --help", async () => { + const output = outputCapture(); + await expect(runCli(["nope"], output.environmentWith())).resolves.toBe(2); + expect(JSON.parse(output.stderr).error).toMatchObject({ + code: "USAGE", + message: "Unknown command: nope (run `simlock --help` for usage)", + }); + }); +}); diff --git a/src/cli/index-lease-requests.test.ts b/src/cli/index-lease-requests.test.ts new file mode 100644 index 000000000..177d3eb2a --- /dev/null +++ b/src/cli/index-lease-requests.test.ts @@ -0,0 +1,988 @@ +import { describe, expect, it } from "vitest"; +import { + FakeClock, + FakeDaemonLauncher, + MemoryFilesystem, + MemoryIpcTransport, + FakeSystemStats, +} from "../ports/index.js"; +import { waitingRequestSchema } from "../contract/index.js"; +import type { DeviceRecoveredPush, DeviceUnhealthyPush } from "../simlock-client/client.js"; +import { runCli, type CliEnvironment } from "./index.js"; +import { buildCliEnvironment } from "./environment.js"; +import { + fakeClient, + gibibyte, + outputCapture, + realCliEnvironmentPorts, + settle, +} from "./test-index.js"; + +// why: a part of a decomposed module: the lease tests build the environment with the part that +// owns it, buildCliEnvironment, where a test needs the ports wired its own way (ADR 0063 section 2). +describe("CLI: auto-launch's daemon that exits while starting (#760)", () => { + it("status fails at once with the daemon's start error when the launched daemon exits with a fatal line", async () => { + const clock = new FakeClock(0); + const fatal = JSON.stringify({ + message: "Daemon failed to start", + fields: { message: "Found /simlock/state.json, remove it" }, + }); + const launcher = new FakeDaemonLauncher(undefined, { + exit: { code: 1, signal: null }, + output: `older line\n${fatal}\n`, + }); + const output = outputCapture({ + filesystem: new MemoryFilesystem(), + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: new MemoryIpcTransport(), + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli(["status"], output.environmentWith()).then((code) => { + settled = true; + return code; + }); + for (let tick = 0; tick < 100 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(settled, "status had not settled by 5 s").toBe(true); + expect(clock.now()).toBeLessThan(20_000); + expect(await exitCode).toBe(1); + expect(JSON.parse(output.stderr)).toEqual({ + error: { + code: "DAEMON_STARTUP_FAILED", + message: "The daemon could not start: Found /simlock/state.json, remove it", + }, + }); + }); + + it("status names the exit code when the launched daemon's output cannot be read", async () => { + const clock = new FakeClock(0); + const launcher = { + launch: async () => ({ + exited: Promise.resolve({ code: 3, signal: null }), + readOutput: async (): Promise => { + throw new Error("daemon.log was replaced"); + }, + }), + }; + const output = outputCapture({ + filesystem: new MemoryFilesystem(), + clock, + systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), + ipc: new MemoryIpcTransport(), + launcher, + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }); + let settled = false; + const exitCode = runCli(["status"], output.environmentWith()).then((code) => { + settled = true; + return code; + }); + for (let tick = 0; tick < 100 && !settled; tick += 1) { + clock.advance(50); + await settle(); + await settle(); + } + expect(settled, "status had not settled by 5 s").toBe(true); + expect(await exitCode).toBe(1); + expect(JSON.parse(output.stderr).error.message).toBe( + "The daemon exited with code 3 while starting; see `simlock daemon logs`", + ); + }); +}); + +describe("CLI: token operations go through the daemon", () => { + it("token create/list/revoke all go through the client, not a local TokenStore", async () => { + const calls: string[] = []; + const client = fakeClient({ + createToken: (input) => { + calls.push("create"); + return Promise.resolve({ + secret: "sec_1", + token: { id: "tok_1", role: input.role, createdAt: 0 }, + }); + }, + listTokens: () => { + calls.push("list"); + return Promise.resolve({ tokens: [{ id: "tok_1", role: "operator", createdAt: 0 }] }); + }, + revokeToken: () => { + calls.push("revoke"); + return Promise.resolve({ revoked: true }); + }, + }); + const environment = outputCapture().environmentWith({ connectAdmin: async () => client }); + await runCli(["token", "create", "--role", "operator"], environment); + await runCli(["token", "list"], environment); + await runCli(["token", "revoke", "tok_1"], environment); + expect(calls).toEqual(["create", "list", "revoke"]); + }); +}); + +describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", () => { + it("writes {push:...} stderr lines for progress, device-unhealthy, and device-recovered as contract values", async () => { + const output = outputCapture(); + let unhealthyListener: ((push: DeviceUnhealthyPush) => void) | undefined; + let recoveredListener: ((push: DeviceRecoveredPush) => void) | undefined; + const client = fakeClient({ + onDeviceUnhealthy: (listener) => { + unhealthyListener = listener; + return () => {}; + }, + onDeviceRecovered: (listener) => { + recoveredListener = listener; + return () => {}; + }, + requestLease: (_input, options) => { + options?.onProgress?.({ stage: "provisioning", etaMs: 1_500 }); + unhealthyListener?.({ leaseId: "lse_1", deviceId: "dev_1" }); + recoveredListener?.({ leaseId: "lse_1", deviceId: "dev_1", attempts: 2 }); + return Promise.resolve({ + device: { + physical: false, + id: "dev_1", + driverDeviceId: "dev_1", + spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, + mode: "full", + }, + environment: {}, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }); + }, + }); + await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith({ connectAdmin: async () => client }), + ); + const lines = output.stderr + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(lines).toContainEqual({ push: "progress", stage: "provisioning", etaMs: 1_500 }); + expect(lines).toContainEqual({ + push: "device-unhealthy", + leaseId: "lse_1", + deviceId: "dev_1", + }); + expect(lines).toContainEqual({ + push: "device-recovered", + leaseId: "lse_1", + deviceId: "dev_1", + attempts: 2, + }); + }); + + it("maps --mode and --no-wait onto the contract's mode/noWait input fields", async () => { + const output = outputCapture(); + let capturedInput: Record | undefined; + const client = fakeClient({ + requestLease: (input) => { + capturedInput = input as unknown as Record; + return Promise.resolve({ + device: { + physical: false, + id: "dev_1", + driverDeviceId: "dev_1", + spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, + mode: "full", + }, + environment: {}, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }); + }, + }); + await runCli( + [ + "lease", + "--platform", + "ios", + "--device", + "iPhone 17 Pro", + "--detach", + "--mode", + "slim", + "--no-wait", + ], + output.environmentWith({ connectAdmin: async () => client }), + ); + expect(capturedInput?.mode).toBe("slim"); + expect(capturedInput?.noWait).toBe(true); + }); + + // why: the real environment builder reads `SIMLOCK_BASELINE`, so the rows below prove the variable, not a hand-set field. + const fromEnv = (value: string): Partial => { + const { baselineFromEnv } = buildCliEnvironment(realCliEnvironmentPorts(), { + SIMLOCK_BASELINE: value, + }); + return baselineFromEnv === undefined ? {} : { baselineFromEnv }; + }; + + it.each([ + { + flag: "--image-tag", + field: "imageTag", + value: "google_apis_playstore", + lease: ["lease", "--platform", "android", "--device", "Pixel 8", "--detach"], + flagArgs: ["--image-tag", "google_apis_playstore"], + envOverride: () => ({}), + }, + { + flag: "--except", + field: "except", + value: ["store", "other"], + lease: ["lease", "--platform", "ios", "--mode", "slim", "--detach"], + flagArgs: ["--except", "store,other"], + envOverride: () => ({}), + }, + { + flag: "--baseline", + field: "baseline", + value: "checkout", + lease: ["lease", "--platform", "android", "--detach"], + flagArgs: ["--baseline", "checkout"], + envOverride: () => ({}), + }, + { + flag: "SIMLOCK_BASELINE", + field: "baseline", + value: "checkout", + lease: ["lease", "--platform", "android", "--detach"], + flagArgs: [], + envOverride: () => fromEnv("checkout"), + }, + { + flag: "--baseline over SIMLOCK_BASELINE", + field: "baseline", + value: "other", + lease: ["lease", "--platform", "android", "--detach"], + flagArgs: ["--baseline", "other"], + envOverride: () => fromEnv("checkout"), + }, + { + flag: "an empty SIMLOCK_BASELINE", + field: "baseline", + value: undefined, + lease: ["lease", "--platform", "android", "--detach"], + flagArgs: [], + envOverride: () => fromEnv(""), + }, + { + flag: "--physical", + field: "physical", + value: true, + lease: ["lease", "--platform", "ios", "--detach"], + flagArgs: ["--physical"], + envOverride: () => ({}), + }, + ])( + "maps $flag onto the contract's $field input field, and sends none without it", + async ({ field, value, lease, flagArgs, envOverride }) => { + const inputs: Record[] = []; + const client = fakeClient({ + requestLease: (input) => { + inputs.push(input as unknown as Record); + return Promise.resolve({ + device: { + class: "phone", + driverDeviceId: "00008110-001A2B3C4D5E", + id: "dev_1", + physical: true, + spec: { + class: "phone", + model: "iPhone 15 Pro", + osVersion: "18.4", + physical: true, + platform: "ios", + }, + }, + environment: {}, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }); + }, + }); + + await runCli( + [...lease, ...flagArgs], + outputCapture().environmentWith({ connectAdmin: async () => client, ...envOverride() }), + ); + await runCli(lease, outputCapture().environmentWith({ connectAdmin: async () => client })); + + expect(inputs[0]?.[field]).toEqual(value); + expect(inputs[1]).not.toHaveProperty(field); + }, + ); + + it("fails the removed full flag as an unknown option, before connecting", async () => { + const output = outputCapture(); + let connected = false; + const exitCode = await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--full"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient({}); + }, + }), + ); + expect(exitCode).toBe(2); + expect(connected).toBe(false); + expect(output.stderr).toContain("--full"); + }); + + it("omits mode and reports noWait: false when neither flag is given", async () => { + const output = outputCapture(); + let capturedInput: Record | undefined; + const client = fakeClient({ + requestLease: (input) => { + capturedInput = input as unknown as Record; + return Promise.resolve({ + device: { + physical: false, + id: "dev_1", + driverDeviceId: "dev_1", + spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, + mode: "full", + }, + environment: {}, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }); + }, + }); + await runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], + output.environmentWith({ connectAdmin: async () => client }), + ); + expect(capturedInput).not.toHaveProperty("mode"); + expect(capturedInput?.noWait).toBe(false); + }); +}); + +describe("CLI: holder renew and release (ADR 0004 §2)", () => { + it("stops renewing when the lease is lost, and still does not re-release it", async () => { + const clock = new FakeClock(0); + const output = outputCapture(); + let leaseLostListener: + | ((push: { leaseId: string; deviceId: string; reason: string }) => void) + | undefined; + let renewed = 0; + let released = false; + const client = fakeClient({ + onLeaseLost: (listener) => { + leaseLostListener = listener; + return () => {}; + }, + renewLease: (input) => { + renewed += 1; + return Promise.resolve({ + deviceId: "dev_1", + grantedAt: 0, + id: input.leaseId, + ownerId: "test-requester", + requesterId: "test-requester", + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: clock.now() + 60_000, + }); + }, + releaseLease: () => { + released = true; + return Promise.resolve({ leaseId: "lse_1" }); + }, + }); + const runPromise = runCli( + ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], + output.environmentWith({ clock, connectAdmin: async () => client }), + ); + await settle(); + leaseLostListener?.({ deviceId: "dev_1", leaseId: "lse_1", reason: "ttl-backstop" }); + + expect(await runPromise).toBe(14); + expect(released).toBe(false); + expect(clock.pendingTimerCount).toBe(0); + clock.advance(600_000); + expect(renewed).toBe(0); + }); +}); + +describe("simlock list --requests", () => { + it("prints a starting request without a place in the queue, one on a worker with that worker, and a future timestamp as waiting 0s", async () => { + const output = outputCapture(); + const asked: unknown[] = []; + const client = fakeClient({ + list: async (input) => { + asked.push(input); + return [ + { + createdAt: -12_000, + id: "req_1", + requesterId: "agent-b", + spec: { imageTag: "google_apis", model: "Pixel 8", platform: "android" }, + stage: "starting", + }, + { + createdAt: -3_400, + id: "req_2", + queuePosition: 2, + requesterId: "local-agent", + spec: { model: "iPhone 16", osVersion: "18.4", platform: "ios" }, + stage: "queued", + workerId: "wrk_1", + }, + { + // why: Stamped by a worker whose clock runs ahead of this one. + createdAt: 5_000, + id: "req_3", + queuePosition: 1, + requesterId: "skewed-agent", + spec: { model: "iPhone 16", platform: "ios" }, + stage: "queued", + workerId: "wrk_2", + }, + { + createdAt: -1_000, + id: "req_4", + queuePosition: 3, + requesterId: "baseline-agent", + spec: { baseline: "checkout", platform: "ios" }, + stage: "queued", + }, + ]; + }, + }); + + await expect( + runCli(["list", "--requests"], output.environmentWith({ connectAdmin: async () => client })), + ).resolves.toBe(0); + + expect(asked).toEqual([{ kind: "requests" }]); + expect(output.stdout).toBe( + "Request req_1: agent-b, android Pixel 8 image tag google_apis, starting, waiting 12s\n" + + "Request req_2: local-agent on wrk_1, ios iPhone 16 18.4, queued at 2, waiting 3s\n" + + "Request req_3: skewed-agent on wrk_2, ios iPhone 16, queued at 1, waiting 0s\n" + + "Request req_4: baseline-agent, ios baseline checkout, queued at 3, waiting 1s\n", + ); + }); + + describe("the reason a queued request waits", () => { + const queued = ( + reason: string | undefined, + platform: "ios" | "android" = "ios", + ): Record => ({ + createdAt: 0, + id: "req_1", + queuePosition: 1, + requesterId: "agent", + spec: { model: "Device", platform }, + stage: "queued", + ...(reason === undefined ? {} : { reason }), + }); + + async function printed(entry: Record): Promise { + const output = outputCapture(); + const client = fakeClient({ list: async () => [entry] as never }); + await runCli( + ["list", "--requests"], + output.environmentWith({ connectAdmin: async () => client }), + ); + return output.stdout.trimEnd(); + } + + it.each([ + ["behind-earlier-request", "ios", "behind an earlier request"], + ["device-limit", "ios", "full: device limit"], + ["ram-budget", "ios", "full: RAM budget"], + ["memory-pressure", "ios", "full: host memory pressure"], + ["global-running-limit", "ios", "full: running limit"], + ["platform-running-limit", "ios", "full: running limit for iOS"], + ["platform-running-limit", "android", "full: running limit for Android"], + ["waiting-for-device", "ios", "a matching device is starting"], + ["no-free-physical", "ios", "every matching device is leased or absent"], + ] as const)( + "ends the line of a queued request with %s on %s as ' — %s'", + async (reason, platform, text) => { + expect(await printed(queued(reason, platform))).toBe( + `Request req_1: agent, ${platform} Device, queued at 1, waiting 0s — ${text}`, + ); + }, + ); + + it("ends the line of a queued request with no reason as it ended before", async () => { + expect(await printed(queued(undefined))).toBe( + "Request req_1: agent, ios Device, queued at 1, waiting 0s", + ); + }); + + it("has a text for every reason the contract's enum lists", async () => { + const field = (waitingRequestSchema.shape as Record)["reason"]; + expect(field).toBeDefined(); + const listed = (field as { unwrap(): { options: readonly string[] } }).unwrap().options; + + expect(listed.length).toBeGreaterThan(0); + for (const reason of listed) { + const line = await printed(queued(reason)); + expect(line).toMatch(/ — [a-z]/); + expect(line).not.toContain(reason); + } + }); + }); + + describe("the reason each worker cannot take a request on a gateway", () => { + const gatewayEntry = ( + workers: readonly Record[], + platform: "ios" | "android" = "ios", + ): Record => ({ + createdAt: 0, + id: "req_1", + queuePosition: 1, + requesterId: "ci", + spec: { model: "Device", platform }, + stage: "queued", + workers, + }); + + async function printedGateway( + entries: readonly Record[], + known: readonly { id: string; label?: string }[] = [], + ): Promise<{ stdout: string; listed: number }> { + const output = outputCapture(); + let listed = 0; + const client = fakeClient({ + list: async () => entries as never, + listWorkers: () => { + listed += 1; + return Promise.resolve({ + workers: known.map((worker) => ({ + catalog: [], + connection: "connected" as const, + devices: [], + drained: false, + lastSeenAt: 1, + leases: [], + ...worker, + })), + } as never); + }, + }); + await runCli( + ["list", "--requests"], + output.environmentWith({ connectAdmin: async () => client }), + ); + return { listed, stdout: output.stdout }; + } + + it("prints one indented line per worker under the request, named by its label, else its id", async () => { + const { stdout } = await printedGateway( + [ + gatewayEntry([ + { reason: "drained", workerId: "wrk_a" }, + { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, + { reason: "runtime-missing", workerId: "wrk_c" }, + { reason: "busy", workerId: "wrk_d" }, + ]), + ], + [{ id: "wrk_a", label: "mac-mini-1" }, { id: "wrk_b" }, { id: "wrk_c", label: "lab" }], + ); + + expect(stdout).toBe( + "Request req_1: ci, ios Device, queued at 1, waiting 0s\n" + + " mac-mini-1: drained\n" + + " wrk_b: full: running limit for iOS\n" + + " lab: oS not installed\n" + + " wrk_d: busy with its own queue\n", + ); + }); + + it.each([ + ["disconnected", "ios", undefined, "not connected"], + ["incompatible", "ios", undefined, "protocol mismatch"], + ["not-ready", "ios", undefined, "not ready yet"], + ["unhealthy", "ios", undefined, "not running"], + ["updating", "ios", undefined, "updating"], + ["drained", "ios", undefined, "drained"], + ["busy", "ios", undefined, "busy with its own queue"], + ["no-driver", "ios", undefined, "no iOS support"], + ["no-driver", "android", undefined, "no Android support"], + ["unknown-model", "ios", undefined, "model not offered"], + ["runtime-missing", "ios", undefined, "oS not installed"], + ["capability-missing", "ios", undefined, "missing capability"], + ["full", "ios", undefined, "full"], + ["full", "ios", "device-limit", "full: device limit"], + ["full", "ios", "ram-budget", "full: RAM budget"], + ["full", "ios", "memory-pressure", "full: host memory pressure"], + ["full", "ios", "global-running-limit", "full: running limit"], + ["full", "ios", "platform-running-limit", "full: running limit for iOS"], + ["full", "android", "platform-running-limit", "full: running limit for Android"], + ["no-free-physical", "ios", undefined, "every matching device is leased or absent"], + ["not-chosen", "ios", undefined, "not chosen this time"], + ["no-baseline", "ios", undefined, "does not hold the baseline"], + ["baseline-stale", "ios", undefined, "baseline is stale"], + ["baseline-mismatch", "ios", undefined, "baseline is for another device"], + ] as const)( + "prints %s on %s with capacity reason %s as ' wrk_a: %s'", + async (reason, platform, capacityReason, text) => { + const { stdout } = await printedGateway([ + gatewayEntry( + [ + { + ...(capacityReason === undefined ? {} : { capacityReason }), + reason, + workerId: "wrk_a", + }, + ], + platform, + ), + ]); + + expect(stdout.split("\n")[1]).toBe(` wrk_a: ${text}`); + }, + ); + + it("asks worker.list once for names, and not at all when no entry has workers", async () => { + const gateway = await printedGateway([ + gatewayEntry([{ reason: "drained", workerId: "wrk_a" }]), + { ...gatewayEntry([{ reason: "busy", workerId: "wrk_a" }]), id: "req_2" }, + ]); + const plain = await printedGateway([{ ...gatewayEntry([]), workers: undefined }]); + + expect(gateway.listed).toBe(1); + expect(plain.listed).toBe(0); + }); + + it("prints a worker's own entry with its reason after ' — ' and no indented lines", async () => { + const { stdout } = await printedGateway([ + { + ...gatewayEntry([]), + reason: "ram-budget", + workers: undefined, + workerId: "wrk_1", + }, + ]); + + expect(stdout).toBe( + "Request req_1: ci on wrk_1, ios Device, queued at 1, waiting 0s — full: RAM budget\n", + ); + }); + + it("has a text for every gateway reason the contract's enum lists", async () => { + const field = (waitingRequestSchema.shape as Record)["workers"]; + expect(field).toBeDefined(); + const element = ( + field as { unwrap(): { element: { shape: { reason: { options: readonly string[] } } } } } + ).unwrap().element; + const listed = element.shape.reason.options; + + expect(listed.length).toBeGreaterThan(0); + for (const reason of listed) { + const { stdout } = await printedGateway([gatewayEntry([{ reason, workerId: "wrk_a" }])]); + const line = stdout.split("\n")[1] ?? ""; + expect(line).toMatch(/^ {2}wrk_a: [a-zA-Z]/); + expect(line).not.toContain("-"); + } + }); + }); + + it.each([["--devices"], ["--leases"], ["--rules"]])( + "exits 2 with USAGE for --requests beside %s, without asking the daemon", + async (other) => { + const output = outputCapture(); + let listed = 0; + const client = fakeClient({ + list: () => { + listed += 1; + return Promise.resolve([]); + }, + }); + + await expect( + runCli( + ["list", "--requests", other], + output.environmentWith({ connectAdmin: async () => client }), + ), + ).resolves.toBe(2); + expect(listed).toBe(0); + expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ + error: { code: "USAGE" }, + }); + }, + ); +}); + +describe("simlock lease: a request names a model, a class, or nothing", () => { + const grant = { + device: { + physical: false, + driverDeviceId: "dev_1", + id: "dev_1", + mode: "full" as const, + spec: { physical: false, model: "iPhone 17", osVersion: "26.5", platform: "ios" as const }, + }, + environment: {}, + lease: { + deviceId: "dev_1", + grantedAt: 0, + id: "lse_1", + lastRenewedAt: 0, + idChosenByRequester: false, + ownerId: "test-requester", + requesterId: "test-requester", + ttlDeadline: 60_000, + ttlMs: 60_000, + }, + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 0, + }, + }; + + async function sentFor(args: readonly string[]): Promise | undefined> { + let sent: Record | undefined; + const client = fakeClient({ + requestLease: (input) => { + sent = input as unknown as Record; + return Promise.resolve(grant); + }, + }); + const output = outputCapture(); + await runCli( + ["lease", ...args, "--detach"], + output.environmentWith({ connectAdmin: async () => client }), + ); + return sent; + } + + it.each([ + { + name: "neither model nor class for a lease with only a platform", + args: ["--platform", "ios"], + present: { platform: "ios" }, + absent: ["model", "class"], + }, + { + name: "the class for --class, and no model", + args: ["--platform", "ios", "--class", "phone"], + present: { class: "phone", platform: "ios" }, + absent: ["model"], + }, + { + name: "both fields for --class beside --device, leaving the refusal to the daemon", + args: ["--platform", "ios", "--class", "phone", "--device", "iPhone 16"], + present: { class: "phone", model: "iPhone 16", platform: "ios" }, + absent: [], + }, + ])("sends $name", async ({ args, present, absent }) => { + const sent = await sentFor(args); + + expect(sent).toMatchObject(present); + for (const key of absent) expect(sent).not.toHaveProperty(key); + }); + + it("names --class in the lease usage text, with --device optional", async () => { + const output = outputCapture(); + + await runCli(["lease", "--help"], output.environmentWith({})); + + expect(output.stdout).toBe( + "Usage: simlock lease --platform [--device | --class ]\n" + + " [--os ] [--mode ] [--image-tag ] [--agent-id ]\n" + + " [--except ] [--baseline ]\n" + + " [--timeout ] [--lease-id ] [--physical]\n" + + " [--no-wait] [--detach] [--ttl ] [--allow-download]\n" + + " [--meta ]... [--export-env] [--bind-pid ]\n", + ); + }); + + it.each([ + ["--device", "lease --device must not be empty"], + ["--class", "lease --class must not be empty"], + ])("exits 2 with USAGE for an empty %s, before connecting", async (flag, message) => { + const output = outputCapture(); + let connected = false; + + const exitCode = await runCli( + ["lease", "--platform", "ios", flag, ""], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient({}); + }, + }), + ); + + expect(exitCode).toBe(2); + expect(connected).toBe(false); + expect(output.stderr).toContain(message); + }); + + it("prints the class of a waiting request that names no model, phone when it names none", async () => { + const output = outputCapture(); + const client = fakeClient({ + list: async () => [ + { + createdAt: 0, + id: "req_1", + queuePosition: 1, + requesterId: "a", + spec: { class: "watch", platform: "ios" }, + stage: "queued", + }, + { + createdAt: 0, + id: "req_2", + queuePosition: 2, + requesterId: "b", + spec: { platform: "android" }, + stage: "queued", + }, + ], + }); + + await runCli( + ["list", "--requests"], + output.environmentWith({ connectAdmin: async () => client }), + ); + + expect(output.stdout).toBe( + "Request req_1: a, ios class watch, queued at 1, waiting 0s\n" + + "Request req_2: b, android class phone, queued at 2, waiting 0s\n", + ); + }); + + it("prints physical in a waiting physical request's line, and nothing for a simulator request", async () => { + const output = outputCapture(); + const client = fakeClient({ + list: async () => [ + { + createdAt: 0, + id: "req_1", + queuePosition: 1, + requesterId: "a", + spec: { model: "iPhone 15 Pro", physical: true, platform: "ios" }, + stage: "queued", + }, + { + createdAt: 0, + id: "req_2", + queuePosition: 1, + requesterId: "b", + spec: { model: "iPhone 16", platform: "ios" }, + stage: "queued", + }, + ], + }); + + await runCli( + ["list", "--requests"], + output.environmentWith({ connectAdmin: async () => client }), + ); + + expect(output.stdout).toBe( + "Request req_1: a, ios physical iPhone 15 Pro, queued at 1, waiting 0s\n" + + "Request req_2: b, ios iPhone 16, queued at 1, waiting 0s\n", + ); + }); + + it("prints a waiting request's OS range as typed", async () => { + const output = outputCapture(); + const client = fakeClient({ + list: async () => [ + { + createdAt: 0, + id: "req_1", + queuePosition: 1, + requesterId: "a", + spec: { osVersion: ">=18 <26", platform: "ios" }, + stage: "queued", + }, + ], + }); + + await runCli( + ["list", "--requests"], + output.environmentWith({ connectAdmin: async () => client }), + ); + + expect(output.stdout).toBe( + "Request req_1: a, ios class phone >=18 <26, queued at 1, waiting 0s\n", + ); + }); +}); diff --git a/src/cli/index-status-daemon-lines.test.ts b/src/cli/index-status-daemon-lines.test.ts new file mode 100644 index 000000000..2d707c08d --- /dev/null +++ b/src/cli/index-status-daemon-lines.test.ts @@ -0,0 +1,752 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import type { StatusGetOutput } from "../simlock-client/client.js"; +import { runCli } from "./index.js"; +import { EMPTY_STATUS, PACKAGE_VERSION, fakeClient, outputCapture } from "./test-index.js"; + +describe("CLI: status renders the fleet a gateway reports (ADR 0005 §20)", () => { + describe("against a starting daemon", () => { + const STARTING: StatusGetOutput = { + daemon: { + health: "starting", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, + }; + + it("simlock status prints the starting line and no device, lease or capacity line", async () => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(STARTING) }), + }), + ); + + expect(output.stdout).toBe( + "Daemon: starting (worker)\n" + + "Simlock: 0.0.0-test\n" + + `CLI: ${PACKAGE_VERSION}\n` + + "Host: macOS 15.5 arm64\n" + + "Devices, leases and capacity appear once startup finishes.\n", + ); + }); + + it.each(["capacity", "devices", "leases", "queueDepth"] as const)( + "simlock status prints the starting line for an answer with no %s, though the rest is there", + async (missing) => { + const output = outputCapture(); + const { [missing]: _left, ...answer } = EMPTY_STATUS; + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ getStatus: () => Promise.resolve(answer as StatusGetOutput) }), + }), + ); + + expect(output.stdout).toContain( + "Devices, leases and capacity appear once startup finishes.\n", + ); + expect(output.stdout).not.toContain("Queue depth"); + }, + ); + + it("simlock status --json prints the answer with those fields absent", async () => { + const output = outputCapture(); + await runCli( + ["status", "--json"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(STARTING) }), + }), + ); + + expect(JSON.parse(output.stdout)).toEqual({ ...STARTING, cliVersion: PACKAGE_VERSION }); + }); + }); + + it("prints the host line", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => + Promise.resolve({ + ...EMPTY_STATUS, + host: { + arch: "arm64", + os: "macOS", + osVersion: "15.5", + tools: [ + { build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }, + ], + }, + }), + }), + }); + + await runCli(["status"], environment); + + expect(output.stdout).toContain("Host: macOS 15.5 arm64; xcode 16.4 (16F6)\n"); + }); + + describe("the Load line", () => { + const GIB = 2 ** 30; + const MIB = 2 ** 20; + const LOAD = { + ageMs: 4_000, + cpu: { busyPercent: 34.2, cores: 10, limited: false }, + disk: { freeBytes: 120_688_581_018, totalBytes: 530_857_957_786 }, + memory: { + availableBytes: 10_522_669_875, + limited: false, + pressure: "normal" as const, + swapUsedBytes: 1_288_490_189, + totalBytes: 32 * GIB, + }, + }; + const FULL_LINE = + "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago\n"; + const STARTING = { + daemon: { + health: "starting" as const, + mode: "worker" as const, + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, + }; + + const print = async (status: StatusGetOutput): Promise => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + return output.stdout; + }; + const loadLine = async (load: StatusGetOutput["load"]): Promise => { + const stdout = await print({ ...EMPTY_STATUS, load }); + return stdout.split("\n").find((line) => line.startsWith("Load:")) ?? ""; + }; + + it("simlock status prints the Load line after the Host line, on a starting daemon and a running one", async () => { + expect(await print({ ...STARTING, load: LOAD })).toBe( + "Daemon: starting (worker)\n" + + "Simlock: 0.0.0-test\n" + + `CLI: ${PACKAGE_VERSION}\n` + + "Host: macOS 15.5 arm64\n" + + FULL_LINE + + "Devices, leases and capacity appear once startup finishes.\n", + ); + + expect(await print({ ...EMPTY_STATUS, load: LOAD })).toContain( + `Host: macOS 15.5 arm64\n${FULL_LINE}`, + ); + }); + + it("simlock status prints CPU unavailable, memory unavailable or disk unavailable for an absent part, and no swap or pressure part when memory is absent", async () => { + const { cpu: _cpu, ...withoutCpu } = LOAD; + const { memory: _memory, ...withoutMemory } = LOAD; + const { disk: _disk, ...withoutDisk } = LOAD; + + expect(await loadLine(withoutCpu)).toBe( + "Load: CPU unavailable · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago", + ); + expect(await loadLine(withoutMemory)).toBe( + "Load: CPU 34% busy (10 cores) · memory unavailable · disk 112.4 GiB of 494.4 GiB free · read 4s ago", + ); + expect(await loadLine(withoutDisk)).toBe( + "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk unavailable · read 4s ago", + ); + }); + + it("simlock status prints no pressure part when the reading has none, and limit on a limited CPU or memory", async () => { + const { pressure: _pressure, ...noPressure } = LOAD.memory; + + expect(await loadLine({ ...LOAD, memory: noPressure })).toBe( + "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · disk 112.4 GiB of 494.4 GiB free · read 4s ago", + ); + expect( + await loadLine({ + ...LOAD, + cpu: { busyPercent: 34.2, cores: 2, limited: true }, + memory: { ...LOAD.memory, availableBytes: 3 * GIB, limited: true, totalBytes: 8 * GIB }, + }), + ).toBe( + "Load: CPU 34% busy (2 cores, limit) · memory 3.0 GiB of 8.0 GiB available (limit) · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago", + ); + }); + + it.each([ + ["swap 0 B", { memory: { ...LOAD.memory, swapUsedBytes: 0 } }, "swap 0 B used"], + [ + "swap 300 MiB", + { memory: { ...LOAD.memory, swapUsedBytes: 300 * MIB } }, + "swap 300 MiB used", + ], + [ + "swap 1023.6 MiB", + { memory: { ...LOAD.memory, swapUsedBytes: 1_073_322_394 } }, + "swap 1.0 GiB used", + ], + ["busy 34.5", { cpu: { ...LOAD.cpu, busyPercent: 34.5 } }, "CPU 35% busy"], + ["cores 1", { cpu: { ...LOAD.cpu, cores: 1 } }, "(1 core)"], + ["cores 0.5", { cpu: { ...LOAD.cpu, cores: 0.5 } }, "(0.5 cores)"], + ["cores 10", { cpu: { ...LOAD.cpu, cores: 10 } }, "(10 cores)"], + ["cores 0.01", { cpu: { ...LOAD.cpu, cores: 0.01 } }, "(0.01 cores)"], + ["cores 0.001", { cpu: { ...LOAD.cpu, cores: 0.001 } }, "(<0.01 cores)"], + ["ageMs 4001", { ageMs: 4_001 }, "read 5s ago"], + ["ageMs 0", { ageMs: 0 }, "read 0s ago"], + ])("simlock status formats %s as in the inputs table", async (_row, change, fragment) => { + expect(await loadLine(LOAD)).toContain("CPU 34% busy (10 cores)"); + + expect(await loadLine({ ...LOAD, ...change })).toContain(fragment); + }); + + it("simlock status prints no Load line when the answer has no load block", async () => { + expect(await print({ ...EMPTY_STATUS, load: LOAD })).toContain("Load: "); + + expect(await print(EMPTY_STATUS)).not.toContain("Load:"); + expect(await print(STARTING)).not.toContain("Load:"); + }); + + it("simlock status prints the Memory pressure line with the step, when it began and why, and none when there is no step", async () => { + const line = async (memoryPressure: unknown) => + ( + await print({ + ...EMPTY_STATUS, + capacity: { ...EMPTY_STATUS.capacity, memoryPressure }, + load: LOAD, + } as StatusGetOutput) + ) + .split("\n") + .find((row) => row.startsWith("Memory pressure:")); + const since = new Date(2026, 10, 6, 9, 12).getTime(); + const warn = { availablePercent: 34, level: "warn", trigger: "level" }; + const floor = (availablePercent: number, extra = {}) => + line({ + cause: { availablePercent, floorPercent: 10, trigger: "floor", ...extra }, + since, + step: "new-devices-paused", + }); + + expect(await line({ cause: warn, since, step: "warming-paused" })).toBe( + "Memory pressure: warming paused since 09:12 (pressure warn)", + ); + const floored = "Memory pressure: new devices paused since 09:12 (8% available, floor 10%)"; + expect(await floor(8)).toBe(floored); + expect(await floor(8, { level: "warn" })).toBe(floored); + expect(await floor(9)).toBe(floored.replace("8%", "9%")); + expect(await line({ step: "none" })).toBeUndefined(); + expect(await line(undefined)).toBeUndefined(); + + const stdout = await print({ + ...EMPTY_STATUS, + capacity: { + ...EMPTY_STATUS.capacity, + memoryPressure: { cause: warn, since, step: "warming-paused" }, + }, + load: LOAD, + } as StatusGetOutput); + expect(stdout).toContain( + `${FULL_LINE}Memory pressure: warming paused since 09:12 (pressure warn)\nRunning global`, + ); + }); + }); + + it("simlock status prints the console address when HTTP is enabled", async () => { + const print = async (status: StatusGetOutput) => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + return output.stdout; + }; + const daemon = { ...EMPTY_STATUS.daemon, consoleUrl: "http://127.0.0.1:4700/" }; + + expect(await print({ ...EMPTY_STATUS, daemon })).toContain( + `Daemon: running (worker)\nSimlock: 0.0.0-test\nCLI: ${PACKAGE_VERSION}\nConsole: http://127.0.0.1:4700/\nHost: `, + ); + expect(await print(EMPTY_STATUS)).not.toContain("Console:"); + }); + + it("simlock daemon start prints the console address when HTTP is enabled", async () => { + const start = async (status: StatusGetOutput) => { + const output = outputCapture(); + await runCli( + ["daemon", "start"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + return output.stdout; + }; + const daemon = { ...EMPTY_STATUS.daemon, consoleUrl: "http://127.0.0.1:4700/" }; + + expect(await start({ ...EMPTY_STATUS, daemon })).toBe( + "Daemon running\nConsole: http://127.0.0.1:4700/\n", + ); + expect(await start(EMPTY_STATUS)).toBe("Daemon running\n"); + }); + + describe("the history lines of simlock status", () => { + const NOT_WRITTEN = "History: the event history is not being written; see daemon.log."; + const SET_ASIDE_AT = Date.UTC(2026, 9, 16, 9, 12, 30); + const pad = (value: number): string => String(value).padStart(2, "0"); + const local = new Date(SET_ASIDE_AT); + const setAside = `History: the event history was damaged and set aside at ${local.getFullYear()}-${pad(local.getMonth() + 1)}-${pad(local.getDate())} ${pad(local.getHours())}:${pad(local.getMinutes())}; a fresh one was started.`; + + const statusLines = async (history: StatusGetOutput["history"]): Promise => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => + Promise.resolve({ + ...EMPTY_STATUS, + ...(history === undefined ? {} : { history }), + } as StatusGetOutput), + }), + }), + ); + return output.stdout.trimEnd().split("\n"); + }; + + it("prints the not-written line when writing is false, the set-aside line with the local time when setAsideAt is set, both when both hold, and neither otherwise", async () => { + const healthy = await statusLines({ opened: true, writing: true }); + const notWritten = await statusLines({ opened: false, writing: false }); + const aside = await statusLines({ opened: true, setAsideAt: SET_ASIDE_AT, writing: true }); + const both = await statusLines({ opened: true, setAsideAt: SET_ASIDE_AT, writing: false }); + + expect(healthy.filter((line) => line.startsWith("History:"))).toEqual([]); + expect(notWritten.filter((line) => line.startsWith("History:"))).toEqual([NOT_WRITTEN]); + expect(aside.filter((line) => line.startsWith("History:"))).toEqual([setAside]); + expect(both.filter((line) => line.startsWith("History:")).sort()).toEqual( + [NOT_WRITTEN, setAside].sort(), + ); + }); + + it("prints the rebuilt line with the local time when rebuiltAt is set and the could-not-rebuild line when rebuildFailed is set, and neither otherwise", async () => { + const rebuiltLine = `History: figures were rebuilt from the event history at ${local.getFullYear()}-${pad(local.getMonth() + 1)}-${pad(local.getDate())} ${pad(local.getHours())}:${pad(local.getMinutes())}.`; + const failedLine = "History: figures could not be rebuilt; see daemon.log."; + const plain = await statusLines({ opened: true, writing: true }); + const rebuilt = await statusLines({ opened: true, rebuiltAt: SET_ASIDE_AT, writing: true }); + const failed = await statusLines({ opened: true, rebuildFailed: true, writing: true }); + + expect(plain.filter((line) => line.startsWith("History:"))).toEqual([]); + expect(rebuilt.filter((line) => line.startsWith("History:"))).toEqual([rebuiltLine]); + expect(failed.filter((line) => line.startsWith("History:"))).toEqual([failedLine]); + }); + }); + + it("prints an install line while one runs and none after", async () => { + const install = { + component: "26.4", + platform: "ios" as const, + since: 1_000, + state: "downloading" as const, + waiters: 1, + }; + const print = async (installs: StatusGetOutput["installs"]) => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + clock: new FakeClock(43_000), + connectAdmin: async () => + fakeClient({ getStatus: () => Promise.resolve({ ...EMPTY_STATUS, installs }) }), + }), + ); + return output.stdout; + }; + + expect(await print([install])).toContain("Install ios 26.4: downloading for 42s, 1 waiter\n"); + expect(await print([])).not.toContain("Install "); + }); + + it("prints an install whose start is ahead of this machine's clock as 0s old, not a negative age", async () => { + const output = outputCapture(); + const install = { + component: "26.4", + platform: "ios" as const, + since: 5_000, + state: "waiting" as const, + waiters: 1, + }; + await runCli( + ["status"], + output.environmentWith({ + clock: new FakeClock(1_000), + connectAdmin: async () => + fakeClient({ + getStatus: () => Promise.resolve({ ...EMPTY_STATUS, installs: [install] }), + }), + }), + ); + + expect(output.stdout).toContain("Install ios 26.4: waiting for 0s, 1 waiter\n"); + }); + + it("prints a gateway's installs once each, naming the worker, not again under the worker", async () => { + const output = outputCapture(); + const install = { + component: "35", + platform: "android" as const, + since: 0, + state: "waiting" as const, + waiters: 3, + }; + const status: StatusGetOutput = { + ...EMPTY_STATUS, + daemon: { + health: "running", + mode: "gateway", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + installs: [{ ...install, workerId: "wrk_1" }], + workers: [ + { + catalog: [], + connection: "connected", + devices: [], + drained: false, + id: "wrk_1", + installs: [install], + lastSeenAt: 1, + leases: [], + }, + ], + }; + const environment = output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }); + + await runCli(["status"], environment); + + expect(output.stdout.match(/Install /g)).toHaveLength(1); + expect(output.stdout).toContain("Install android 35 on wrk_1: waiting for 0s, 3 waiters\n"); + }); + + it("prints each device's mode and whether it serves a request naming no mode", async () => { + const output = outputCapture(); + const spec = { + physical: false, + model: "iPhone 17", + osVersion: "26.0", + platform: "ios" as const, + }; + const status: StatusGetOutput = { + ...EMPTY_STATUS, + devices: [ + { + physical: false, + id: "dev_slim", + mode: "slim", + servesDefaultMode: false, + spec, + state: "ready", + }, + { + physical: false, + id: "dev_full", + mode: "full", + servesDefaultMode: true, + spec, + state: "leased", + }, + { + physical: false, + id: "dev_copy", + mode: "full", + servesDefaultMode: true, + spec: { ...spec, baseline: { name: "checkout", version: 2 } }, + state: "leased", + }, + ], + }; + const environment = output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }); + + await runCli(["status"], environment); + + expect(output.stdout).toContain("Device dev_slim: ready, mode slim, serves default mode: no\n"); + expect(output.stdout).toContain( + "Device dev_full: leased, mode full, serves default mode: yes\n", + ); + expect(output.stdout).toContain( + "Device dev_copy: leased, mode full, serves default mode: yes, baseline checkout v2\n", + ); + }); + + it("prints the RAM budget, marked over limit when over, and no RAM line when the daemon reports none", async () => { + type RamBudget = NonNullable["ramBudget"]>; + const statusWith = (ramBudget?: RamBudget) => { + const status: StatusGetOutput = { + ...EMPTY_STATUS, + capacity: { ...EMPTY_STATUS.capacity, ...(ramBudget === undefined ? {} : { ramBudget }) }, + }; + return status; + }; + const print = async (status: StatusGetOutput) => { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + return output.stdout; + }; + const gib = 1024 ** 3; + + expect( + await print(statusWith({ limitBytes: 12 * gib, overLimit: false, usedBytes: 4.5 * gib })), + ).toContain("RAM budget: 4.50 GiB/12.00 GiB used\n"); + expect( + await print(statusWith({ limitBytes: 12 * gib, overLimit: true, usedBytes: 13 * gib })), + ).toContain("RAM budget: 13.00 GiB/12.00 GiB used (over limit)\n"); + expect(await print(statusWith())).not.toContain("RAM"); + }); + + it("prints, per platform, whether the worker is at its RAM budget, after any over-limit mark", async () => { + const status: StatusGetOutput = { + ...EMPTY_STATUS, + capacity: { + ...EMPTY_STATUS.capacity, + android: { ...EMPTY_STATUS.capacity.android, atRamBudget: false }, + ios: { ...EMPTY_STATUS.capacity.ios, atRamBudget: true, overLimit: true }, + }, + }; + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + const lines = output.stdout.split("\n"); + expect(lines).toContain( + "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0 (over limit) (at RAM budget)", + ); + expect(lines).toContain("Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0"); + }); + + it("simlock status prints a physical device as physical and its class, with today's markers, and a simulator's line as before", async () => { + const status: StatusGetOutput = { + ...EMPTY_STATUS, + devices: [ + { + class: "phone", + id: "dev_p1", + physical: true, + servesDefaultMode: false, + spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, + state: "ready", + }, + { + class: "tablet", + id: "dev_p2", + physical: true, + servesDefaultMode: false, + spec: { model: "iPad Pro", osVersion: "18.4", physical: true, platform: "ios" }, + state: "quarantined", + quarantineAttempts: 1, + workerId: "wrk_1", + }, + { + id: "dev_v1", + mode: "full", + physical: false, + servesDefaultMode: true, + spec: { model: "iPhone 16", osVersion: "18.4", physical: false, platform: "ios" }, + state: "ready", + }, + ], + }; + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + const lines = output.stdout.split("\n"); + expect(lines).toContain("Device dev_p1: ready, physical phone"); + expect(lines).toContain("Device dev_p2 on wrk_1: quarantined, physical tablet (purge retry 1)"); + expect(lines).toContain("Device dev_v1: ready, mode full, serves default mode: yes"); + }); + + it("status ends a quarantined device's marker with ; and the cause for each kind, and prints today's marker when there is no cause", async () => { + const quarantined = ( + id: string, + quarantineCause?: Record, + ): Record => ({ + id, + mode: "full", + physical: false, + quarantineAttempts: 1, + quarantineNextRetryAt: 1_760_619_150_000, + servesDefaultMode: true, + spec: { model: "iPhone 16", osVersion: "18.4", physical: false, platform: "ios" }, + state: "quarantined", + ...(quarantineCause === undefined ? {} : { quarantineCause }), + }); + const status = { + ...EMPTY_STATUS, + devices: [ + quarantined("dev_purge", { + kind: "purge-failed", + message: "DriverCrashError: simctl erase exited 164, then stalled", + }), + quarantined("dev_shutdown", { + kind: "shutdown-failed", + message: "Error: shutdown exploded", + }), + quarantined("dev_delete", { kind: "delete-failed", message: "Error: delete exploded" }), + quarantined("dev_prov", { kind: "stalled", stalledState: "provisioning" }), + quarantined("dev_recl", { kind: "stalled", stalledState: "reclaiming" }), + quarantined("dev_none"), + ], + } as unknown as StatusGetOutput; + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + const lines = output.stdout.split("\n"); + const head = + "quarantined, mode full, serves default mode: yes (purge retry 1, next retry at 1760619150000"; + expect(lines).toContain( + `Device dev_purge: ${head}; purge failed: DriverCrashError: simctl erase exited 164, then stalled)`, + ); + expect(lines).toContain( + `Device dev_shutdown: ${head}; shutdown failed: Error: shutdown exploded)`, + ); + expect(lines).toContain(`Device dev_delete: ${head}; delete failed: Error: delete exploded)`); + expect(lines).toContain(`Device dev_prov: ${head}; stalled while provisioning)`); + expect(lines).toContain(`Device dev_recl: ${head}; stalled while reclaiming)`); + expect(lines).toContain(`Device dev_none: ${head})`); + }); + + it("status prints absent () as the state of an absent device", async () => { + const physical = (id: string, state: string, absentReason?: string) => ({ + class: "phone", + id, + physical: true, + servesDefaultMode: false, + spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, + state, + ...(absentReason === undefined ? {} : { absentReason }), + }); + const status = { + ...EMPTY_STATUS, + devices: [ + physical("dev_ready", "ready"), + physical("dev_locked", "absent", "locked"), + physical("dev_untrusted", "absent", "untrusted"), + physical("dev_gone", "absent", "not-present"), + ], + } as unknown as StatusGetOutput; + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + const lines = output.stdout.split("\n"); + expect(lines).toContain("Device dev_ready: ready, physical phone"); + expect(lines).toContain("Device dev_locked: absent (locked), physical phone"); + expect(lines).toContain("Device dev_untrusted: absent (untrusted), physical phone"); + expect(lines).toContain("Device dev_gone: absent (not-present), physical phone"); + }); + + it("status prints a bare state for an absent device with no reason and for a device that is not absent", async () => { + const physical = (id: string, state: string, absentReason?: string) => ({ + class: "phone", + id, + physical: true, + servesDefaultMode: false, + spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, + state, + ...(absentReason === undefined ? {} : { absentReason }), + }); + const status = { + ...EMPTY_STATUS, + devices: [physical("dev_bare", "absent"), physical("dev_stray", "ready", "locked")], + } as unknown as StatusGetOutput; + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + const lines = output.stdout.split("\n"); + expect(lines).toContain("Device dev_bare: absent, physical phone"); + expect(lines).toContain("Device dev_stray: ready, physical phone"); + }); + + it("simlock status marks a stalled device", async () => { + const device = (id: string, extra: object) => ({ + physical: false, + id, + mode: "full" as const, + servesDefaultMode: true, + spec: { physical: false, model: "iPhone 16", osVersion: "18.4", platform: "ios" as const }, + state: "provisioning" as const, + ...extra, + }); + const status: StatusGetOutput = { + ...EMPTY_STATUS, + devices: [ + device("dev_stuck", { stalled: true, transitionAgeMs: 120_000 }), + device("dev_fresh", { transitionAgeMs: 2_000 }), + ], + }; + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + expect(output.stdout).toContain( + "Device dev_stuck: provisioning, mode full, serves default mode: yes (mid-transition 120000ms, stalled)\n", + ); + expect(output.stdout).toContain( + "Device dev_fresh: provisioning, mode full, serves default mode: yes (mid-transition 2000ms)\n", + ); + }); +}); diff --git a/src/cli/index-status-warm-pool.test.ts b/src/cli/index-status-warm-pool.test.ts new file mode 100644 index 000000000..5fb921069 --- /dev/null +++ b/src/cli/index-status-warm-pool.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import type { SimlockAdminClient, StatusGetOutput } from "../simlock-client/client.js"; +import { runCli } from "./index.js"; +import { + EMPTY_STATUS, + PACKAGE_VERSION, + fakeClient, + outputCapture, + usageAnswer, +} from "./test-index.js"; + +describe("CLI: status renders the fleet a gateway reports (ADR 0005 §20)", () => { + it("names the mode, each worker, and which worker a device and a lease live on", async () => { + const output = outputCapture(); + const status = { + capacity: { + android: { + atRamBudget: false, + limit: 0, + maxRunning: 0, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }, + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 1, warm: 0 }, + ios: { + atRamBudget: false, + limit: 2, + maxRunning: 2, + overLimit: false, + reserved: 0, + running: 1, + used: 1, + warm: 0, + }, + }, + daemon: { + health: "running" as const, + mode: "gateway" as const, + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + host: { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] }, + devices: [ + { + physical: false, + id: "dev_1", + spec: { + physical: false, + model: "iPhone 17", + osVersion: "26.0", + platform: "ios" as const, + }, + mode: "full" as const, + servesDefaultMode: true, + state: "leased" as const, + workerId: "wrk_1", + }, + ], + leases: [ + { + deviceId: "dev_1", + grantedAt: 1, + id: "lease_1", + lastRenewedAt: 1, + idChosenByRequester: false, + ownerId: "agent-1", + requesterId: "agent-1", + ttlDeadline: 2, + ttlMs: 1, + workerId: "wrk_1", + }, + ], + queueDepth: 0, + workers: [ + { + capacity: undefined, + catalog: [], + connection: "connected" as const, + devices: [], + drained: true, + id: "wrk_1", + lastSeenAt: 1, + leases: [], + }, + ], + }; + const environment = output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }); + + await runCli(["status"], environment); + + expect(output.stdout).toContain("Daemon: running (gateway)"); + // why: A gateway runs no drivers: its host line names the machine and nothing after it. + expect(output.stdout).toContain("Host: Linux 6.8.0 x64\n"); + expect(output.stdout).toContain("wrk_1: connected, drained"); + expect(output.stdout).toContain( + "Device dev_1 on wrk_1: leased, mode full, serves default mode: yes", + ); + expect(output.stdout).toContain("Lease lease_1: agent-1 on wrk_1"); + }); + + describe("the warm pool block", () => { + const warmPool = { + enabled: true, + reserveRunning: { android: 0, ios: 1 }, + targets: [ + { + booting: 1, + count: 2, + mode: "full" as const, + model: "iPhone 17", + origin: "configured" as const, + osVersion: "26.0", + platform: "ios" as const, + ready: 1, + }, + { + booting: 0, + count: 1, + mode: "full" as const, + model: "iPhone 17", + origin: "learned" as const, + osVersion: "27.0", + platform: "ios" as const, + ready: 0, + short: "runtime-missing" as const, + }, + { + booting: 0, + count: 1, + imageTag: "google_apis_playstore", + mode: "full" as const, + model: "Pixel 8", + origin: "learned" as const, + osVersion: "35", + platform: "android" as const, + ready: 1, + }, + { + baseline: "checkout", + booting: 0, + count: 1, + mode: "full" as const, + model: "iPhone 17", + origin: "learned" as const, + osVersion: "26.0", + platform: "ios" as const, + ready: 1, + }, + ], + }; + const statusWith = (block: StatusGetOutput["warmPool"]) => + fakeClient({ getStatus: () => Promise.resolve({ ...EMPTY_STATUS, warmPool: block }) }); + + it("simlock status prints the origin word on every warm pool line, learned lines included, the tag in the name, and a learned line's short:", async () => { + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ connectAdmin: async () => statusWith(warmPool) }), + ); + + expect(output.stdout).toContain( + "warm pool: enabled, reserve ios 1 android 0\n" + + " iPhone 17 / 26.0 / full configured wanted 2 ready 1 booting 1\n" + + " iPhone 17 / 27.0 / full learned wanted 1 ready 0 booting 0 short: runtime-missing\n" + + " Pixel 8 / 35 / full / google_apis_playstore learned wanted 1 ready 1 booting 0\n" + + " iPhone 17 / 26.0 / full / baseline checkout learned wanted 1 ready 1 booting 0\n", + ); + }); + + it("simlock status leaves the runtime out of a target that names none", async () => { + const output = outputCapture(); + const unversioned = { + enabled: true, + reserveRunning: { android: 0, ios: 0 }, + targets: [ + { + booting: 0, + count: 1, + mode: "slim" as const, + model: "iPhone 17", + origin: "configured" as const, + platform: "ios" as const, + ready: 0, + }, + ], + }; + + await runCli( + ["status"], + output.environmentWith({ connectAdmin: async () => statusWith(unversioned) }), + ); + + expect(output.stdout).toContain( + "warm pool: enabled, reserve ios 0 android 0\n" + + " iPhone 17 / slim configured wanted 1 ready 0 booting 0\n", + ); + }); + + it("simlock status prints a pool that is off as disabled, with every target short disabled", async () => { + const output = outputCapture(); + const off = { + enabled: false, + reserveRunning: { android: 0, ios: 0 }, + targets: warmPool.targets.slice(0, 1).map((target) => ({ + ...target, + booting: 0, + ready: 0, + short: "disabled" as const, + })), + }; + + await runCli( + ["status"], + output.environmentWith({ connectAdmin: async () => statusWith(off) }), + ); + + expect(output.stdout).toContain( + "warm pool: disabled, reserve ios 0 android 0\n" + + " iPhone 17 / 26.0 / full configured wanted 2 ready 0 booting 0 short: disabled\n", + ); + }); + + it.each([ + { + name: "worker whose answer has no block", + mode: "worker" as const, + warmPool: undefined, + }, + { + name: "gateway whose own block is empty", + mode: "gateway" as const, + warmPool: { enabled: false, reserveRunning: { android: 0, ios: 0 }, targets: [] }, + }, + ])("simlock status prints no warm pool line for a $name", async ({ mode, warmPool }) => { + const output = outputCapture(); + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => + Promise.resolve({ + ...EMPTY_STATUS, + daemon: { + health: "running" as const, + mode, + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + warmPool, + }), + }), + }), + ); + + expect(output.stdout).toBe( + `Daemon: running (${mode})\n` + + "Simlock: 0.0.0-test\n" + + `CLI: ${PACKAGE_VERSION}\n` + + "Host: macOS 15.5 arm64\n" + + "Running global: 0 + 0 reserved/2, warm 0\n" + + "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + + "Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + + "Queue depth: 0\n", + ); + }); + + it.each(["worker", "gateway"] as const)( + "simlock status of a %s with queueDepth 2 and physicalQueueDepth 1 prints Queue depth: 2", + async (mode) => { + const output = outputCapture(); + const answer = { + ...EMPTY_STATUS, + daemon: { ...EMPTY_STATUS.daemon, mode }, + physicalQueueDepth: 1, + queueDepth: 2, + } as unknown as StatusGetOutput; + + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(answer) }), + }), + ); + + expect(output.stdout).toBe( + `Daemon: running (${mode})\n` + + "Simlock: 0.0.0-test\n" + + `CLI: ${PACKAGE_VERSION}\n` + + "Host: macOS 15.5 arm64\n" + + "Running global: 0 + 0 reserved/2, warm 0\n" + + "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + + "Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + + "Queue depth: 2\n", + ); + }, + ); + + it("simlock status --json carries the block as the daemon sent it", async () => { + const output = outputCapture(); + + await runCli( + ["status", "--json"], + output.environmentWith({ connectAdmin: async () => statusWith(warmPool) }), + ); + + expect(JSON.parse(output.stdout).warmPool).toEqual(warmPool); + expect(JSON.parse(output.stdout).cliVersion).toBe(PACKAGE_VERSION); + }); + + it("simlock doctor prints one stderr line per unreachable warm pool target, keeps the JSON report on stdout, and exits 0", async () => { + const output = outputCapture(); + const findings = [ + { + kind: "warm-pool-target-unreachable" as const, + message: "iOS 27.0 is not installed", + platform: "ios" as const, + reason: "runtime-missing" as const, + remedy: "run simlock component install ios 27.0", + target: "iPhone 17 / 27.0 / full", + }, + { + kind: "warm-pool-target-unreachable" as const, + message: "the targets want 4 running devices, and the running limit leaves room for 3", + reason: "over-limit" as const, + remedy: "lower the counts of the warmPool.targets, or raise the running limit", + target: "all targets", + }, + { deviceId: "d-1", kind: "expired-live-lease" as const, leaseId: "l-1" }, + ]; + + await expect( + runCli( + ["doctor"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), + }), + ), + ).resolves.toBe(0); + + expect( + output.stderr + .split("\n") + .filter((line) => line.includes("warm-pool-target-unreachable") || line.includes("d-1")), + ).toEqual([ + "warm-pool-target-unreachable iPhone 17 / 27.0 / full: iOS 27.0 is not installed; run simlock component install ios 27.0", + "warm-pool-target-unreachable all targets: the targets want 4 running devices, and the running limit leaves room for 3; lower the counts of the warmPool.targets, or raise the running limit", + ]); + expect(output.stderr).not.toContain("undefined"); + expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); + }); + }); +}); + +describe("CLI text output escapes control characters in remote text", () => { + const NOW = Date.parse("2026-10-05T12:00:00.000Z"); + type View = Awaited>["workers"][number]; + const worker = (overrides: Partial = {}): View => ({ + connection: "connected", + drained: false, + id: "3f81a2c4", + lastSeenAt: 1_000, + version: "0.3.0", + ...overrides, + }); + const failure = (message: string) => + ({ at: 2_000, from: "0.3.0", message, stage: "installing", to: "0.4.0" }) as const; + + it.each([ + ["ESC", "\u001b", "\\u001b"], + ["BEL", "\u0007", "\\u0007"], + ["CR", "\r", "\\u000d"], + ["NUL", "\u0000", "\\u0000"], + ["C1 CSI", "\u009b", "\\u009b"], + ])( + "simlock stats, worker list, worker update, device add, update, warm forget and daemon logs print %s in remote text escaped", + async (_name, raw, escaped) => { + const text = `a${raw}b`; + const shown = `a${escaped}b`; + const run = async (argv: readonly string[], client: Partial) => { + const output = outputCapture(); + const code = await runCli( + [...argv], + output.environmentWith({ + readLogFile: async () => `ok\n${text}\n`, + clock: new FakeClock(NOW), + connectAdmin: async () => fakeClient(client), + sleep: async () => undefined, + }), + ); + return { code, stdout: output.stdout }; + }; + const updateAnswer = (version: string) => + ({ + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: { + forced: false, + from: version, + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + }, + upToDate: false, + version, + workerId: "3f81a2c4", + }) as Awaited>; + + const stats = await run(["stats"], { + usage: async (window) => + usageAnswer(window, { + requesters: [{ id: text, granted: 0, heldTotalMs: 0, rejected: 0, requests: 1 }], + }), + }); + const list = await run(["worker", "list"], { + listWorkers: async () => ({ workers: [worker({ version: text })] }), + }); + const detached = await run(["worker", "update", "3f81a2c4", "--detach"], { + listWorkers: async () => ({ workers: [worker()] }), + updateWorker: async () => updateAnswer(text), + }); + const failed = await run(["worker", "update", "3f81a2c4"], { + listWorkers: async () => ({ + workers: [worker({ lastUpdateFailure: failure(text), update: null })], + }), + updateWorker: async () => updateAnswer("0.3.0"), + }); + const enrolled = await run(["device", "add", "--platform", "android", "SER-1"], { + addDevice: async () => ({ + appCount: 0, + class: "phone", + driverDeviceId: "SER-1", + id: "dev_1", + model: text, + osVersion: text, + platform: "android", + reenrolled: false, + }), + }); + + let polled = 0; + const daemonFollowed = await run(["update"], { + updateDaemon: async () => { + polled += 1; + const answer = updateAnswer(text); + return polled === 1 ? answer : { ...answer, update: null }; + }, + }); + const daemonFailed = await run(["update"], { + updateDaemon: async () => ({ + ...updateAnswer("0.3.0"), + update: null, + lastUpdateFailure: failure(text), + }), + }); + const forgotten = await run(["warm", "forget", "android", "Pixel"], { + forgetWarmKind: async () => ({ + forgotten: [{ platform: "android", model: text, osVersion: "36", mode: "full" }], + }), + }); + const logs = await run(["daemon", "logs"], {}); + const results = { + logs, + stats, + list, + detached, + failed, + enrolled, + daemonFollowed, + daemonFailed, + forgotten, + }; + for (const [name, result] of Object.entries(results)) { + expect({ name, raw: result.stdout.includes(raw) }).toEqual({ name, raw: false }); + expect({ name, shown: result.stdout.includes(shown) }).toEqual({ name, shown: true }); + } + }, + ); +}); diff --git a/src/cli/index-update-status-stats.test.ts b/src/cli/index-update-status-stats.test.ts new file mode 100644 index 000000000..73ea8c020 --- /dev/null +++ b/src/cli/index-update-status-stats.test.ts @@ -0,0 +1,846 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { fromWireError, SimlockError } from "../contract/index.js"; +import type { SimlockAdminClient } from "../simlock-client/client.js"; +import { runCli, type CliEnvironment } from "./index.js"; +import { + PACKAGE_VERSION, + fakeClient, + outputCapture, + statusLines, + usageAnswer, +} from "./test-index.js"; + +describe("simlock update and simlock status", () => { + type UpdateAnswer = Awaited>; + type Step = UpdateAnswer | SimlockError; + const lost = () => + new SimlockError("DAEMON_CONNECTION_LOST", "transport", "The daemon connection was lost", {}); + const answer = (overrides: Partial = {}): UpdateAnswer => ({ + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + ...overrides, + }); + const state = ( + stage: "waiting" | "installing" | "restarting", + leases = 0, + ): NonNullable => { + const base = { from: "0.3.0", to: "0.4.0", forced: false, startedAt: 1_000 }; + return stage === "waiting" ? { stage, leases, ...base } : { stage, ...base }; + }; + const failure = (stage: "installing" | "restarting", message: string) => ({ + stage, + from: "0.3.0", + to: "0.4.0", + message, + at: 2_000, + }); + const started = (update = state("installing")) => + answer({ started: true, resolved: "0.4.0", update }); + const running = (version = "0.4.0", overrides: Partial = {}) => + answer({ version, ...overrides }); + + async function runUpdate( + argv: readonly string[], + script: { + readonly first: UpdateAnswer | SimlockError; + readonly polls?: readonly Step[]; + readonly after?: readonly (Step | Error)[]; + readonly onPoll?: (index: number, interrupt: () => void) => void; + readonly onReconnect?: (interrupt: () => void) => void; + }, + ) { + const output = outputCapture(); + const clock = new FakeClock(0); + const calls: unknown[] = []; + const polls = [...(script.polls ?? [])]; + const after = [...(script.after ?? [])]; + let pollCount = 0; + const signals = new EventEmitter(); + let opened = 0; + const closedClients = new Set(); + const clientOver = (steps: Step[], first?: UpdateAnswer | SimlockError) => { + const id = opened++; + return fakeClient({ + close: async () => { + closedClients.add(id); + }, + updateDaemon: async (input) => { + calls.push(input); + const isFirst = first !== undefined && calls.length === 1; + if (!isFirst) script.onPoll?.(pollCount++, () => signals.emit("SIGINT")); + const step = isFirst ? first : steps.length > 1 ? steps.shift() : steps[0]; + if (step === undefined) throw new Error("no scripted answer"); + if (step instanceof Error) throw step; + return step; + }, + }); + }; + const environment = output.environmentWith({ + clock, + connectAdmin: async () => clientOver(polls, script.first), + connectExistingAdmin: async () => { + script.onReconnect?.(() => signals.emit("SIGINT")); + const step = after.length > 1 ? after.shift() : after[0]; + if (step === undefined || step instanceof Error) throw step ?? new Error("refused"); + return clientOver([step]); + }, + sleep: async (milliseconds) => clock.advance(milliseconds), + signals: signals as unknown as CliEnvironment["signals"], + }); + const code = await runCli(argv, environment); + return { + code, + calls, + stdout: output.stdout, + stderr: output.stderr, + opened, + closed: closedClients.size, + }; + } + + it("follows an update to simlock is running, reports a failure and a rollback, says when nothing is needed, stops following on Ctrl-C only, prints --json, refuses bad input, and shows the version, the update and the last failure in simlock status", async () => { + let reconnects = 0; + // why: the follower reconnects every second and gives up when 15 minutes have passed since the last answer, so the last attempt is at 14:59 on the fake clock. + const RECONNECTS_IN_15_MINUTES = 899; + const header = "Updating simlock 0.3.0 → 0.4.0\n"; + const wait = + "Waiting for 2 leases to end. Ctrl-C stops following; simlock update --force stops waiting.\n"; + const cases: readonly { + readonly name: string; + readonly argv: readonly string[]; + readonly script: Parameters[1]; + readonly code: number; + readonly stdout: string; + readonly calls?: readonly unknown[]; + readonly reconnects?: number; + }[] = [ + { + name: "follows to running", + argv: ["update"], + script: { + first: started(state("waiting", 2)), + polls: [ + answer({ update: state("waiting", 2) }), + answer({ update: state("waiting", 1) }), + answer({ update: state("installing") }), + answer({ update: state("restarting") }), + lost(), + ], + after: [new Error("refused"), running()], + }, + code: 0, + stdout: `${header}${wait}Waiting for 1 lease to end.\nInstalling simlock@0.4.0\nRestarting the daemon\nsimlock 0.4.0 is running\n`, + calls: [{}, ...Array.from({ length: 6 }, () => ({ query: true }))], + }, + { + name: "follows an update that already runs, whatever version it names", + argv: ["update", "--version", "0.9.9"], + script: { + first: answer({ update: state("installing") }), + polls: [lost()], + after: [running()], + }, + code: 0, + stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nsimlock 0.4.0 is running\n`, + }, + { + name: "reports a failed install", + argv: ["update"], + script: { + first: started(), + polls: [answer({ lastUpdateFailure: failure("installing", "npm error code EACCES") })], + }, + code: 1, + stdout: `${header}Installing simlock@0.4.0\nUpdate failed (installing): npm error code EACCES\nsimlock 0.3.0 is still running and takes leases again.\n`, + }, + { + name: "reports a version that did not start and the rollback", + argv: ["update"], + script: { + first: started(), + polls: [answer({ update: state("restarting") }), lost()], + after: [ + answer({ + lastUpdateFailure: failure("restarting", "0.4.0 did not start: health failed"), + }), + ], + }, + code: 1, + stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nUpdate failed (restarting): 0.4.0 did not start: health failed. Rolled back to 0.3.0.\n`, + }, + { + name: "waits through a daemon that answers while it is still starting", + argv: ["update"], + script: { + first: started(), + polls: [lost()], + after: [ + answer({ health: "starting" }), + answer({ health: "starting" }), + answer({ health: "starting" }), + answer({ + lastUpdateFailure: failure("restarting", "0.4.0 did not start: health failed"), + }), + ], + }, + code: 1, + stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nUpdate failed (restarting): 0.4.0 did not start: health failed. Rolled back to 0.3.0.\n`, + }, + { + name: "gives up when the daemon does not come back in 15 minutes", + argv: ["update"], + script: { + first: started(), + polls: [answer({ update: state("restarting") }), lost()], + after: [new Error("refused")], + onReconnect: () => { + reconnects += 1; + }, + }, + code: 1, + stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nThe daemon did not come back; run simlock status.\n`, + reconnects: RECONNECTS_IN_15_MINUTES, + }, + { + name: "stops with the daemon's own error when a poll is refused for a reason other than a lost connection", + argv: ["update"], + script: { + first: started(), + polls: [new SimlockError("UPDATE_RESOLVE_FAILED", "domain", "npm is down", {})], + }, + code: 1, + stdout: `${header}Installing simlock@0.4.0\n`, + }, + { + name: "prints its usage for --help and asks the daemon nothing", + argv: ["update", "--help"], + script: { first: answer() }, + code: 0, + stdout: "Usage: simlock update [--version ] [--force] [--json]\n", + calls: [], + }, + { + name: "the latest version is the running one", + argv: ["update"], + script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, + code: 0, + stdout: "simlock 0.3.0 is already the latest\n", + calls: [{}], + }, + { + name: "the named version is the running one", + argv: ["update", "--version", "0.3.0"], + script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, + code: 0, + stdout: "simlock 0.3.0 is already running\n", + calls: [{ version: "0.3.0" }], + }, + { + name: "the running version is newer than the latest", + argv: ["update"], + script: { first: answer({ upToDate: true, resolved: "0.4.0", version: "0.5.0-rc.1" }) }, + code: 0, + stdout: + "simlock 0.5.0-rc.1 is newer than the latest (0.4.0). Use --version to install another version.\n", + }, + { + name: "sends force and the version", + argv: ["update", "--force", "--version", "0.4.0-rc.1"], + script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, + code: 0, + stdout: "simlock 0.3.0 is already running\n", + calls: [{ force: true, version: "0.4.0-rc.1" }], + }, + { + name: "--json prints the final answer only", + argv: ["update", "--json"], + script: { first: started(state("waiting", 1)), polls: [lost()], after: [running()] }, + code: 0, + stdout: `${JSON.stringify(running())}\n`, + }, + { + name: "--json with nothing to do prints the answer at once", + argv: ["update", "--json"], + script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, + code: 0, + stdout: `${JSON.stringify(answer({ upToDate: true, resolved: "0.3.0" }))}\n`, + calls: [{}], + }, + { + name: "Ctrl-C stops following and sends nothing more", + argv: ["update"], + script: { + first: started(state("waiting", 2)), + polls: [answer({ update: state("waiting", 2) })], + onPoll: (_index, interrupt) => interrupt(), + }, + code: 0, + stdout: `${header}${wait}Stopped following. The update goes on; run simlock status to see it.\n`, + calls: [{}, { query: true }], + }, + { + name: "Ctrl-C while the daemon restarts stops following", + argv: ["update"], + script: { + first: started(), + polls: [lost()], + after: [new Error("refused")], + onReconnect: (interrupt) => interrupt(), + }, + code: 0, + stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nStopped following. The update goes on; run simlock status to see it.\n`, + calls: [{}, { query: true }], + }, + ]; + for (const { name, argv, script, code, stdout, calls, reconnects: expected } of cases) { + reconnects = 0; + const result = await runUpdate(argv, script); + if (expected !== undefined) expect(reconnects, name).toBe(expected); + expect({ name, code: result.code, stdout: result.stdout }, name).toEqual({ + name, + code, + stdout, + }); + if (calls !== undefined) expect(result.calls, name).toEqual(calls); + expect(result.closed, `${name}: every client opened is closed`).toBe(result.opened); + } + + const refusals: readonly [readonly string[], number, string][] = [ + [["update", "--version", "latest"], 2, "--version takes a version, e.g. 0.4.0"], + [["update", "--version", "../x"], 2, "--version takes a version, e.g. 0.4.0"], + [["update", "--version", `1.0.0-${"a".repeat(59)}`], 2, "--version takes a version"], + ]; + for (const [argv, code, message] of refusals) { + const result = await runUpdate(argv, { first: answer() }); + expect({ argv, code, stderr: result.stderr.includes(message), calls: result.calls }).toEqual({ + argv, + code, + stderr: true, + calls: [], + }); + } + + const refused = [ + ["UPDATE_RESOLVE_FAILED", 1, "could not resolve simlock@9.9.9: No match found"], + ["UPDATE_UNSUPPORTED_INSTALL", 2, "simlock was not installed with npm i -g"], + ] as const; + for (const [codeName, exit, message] of refused) { + const result = await runUpdate(["update"], { + first: new SimlockError(codeName, "domain", message, {}), + }); + expect({ code: result.code, stdout: result.stdout }).toEqual({ code: exit, stdout: "" }); + expect(JSON.parse(result.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: codeName, message }, + }); + } + + const next = "Host: macOS 15.5 arm64"; + expect(await statusLines({ version: "0.3.0" })).toEqual([ + "Daemon: running (worker)", + "Simlock: 0.3.0", + `CLI: ${PACKAGE_VERSION}`, + next, + expect.stringMatching(/^Running global/), + ]); + for (const [update, line] of [ + [state("waiting", 2), "Updating to 0.4.0: waiting for 2 leases"], + [state("waiting", 1), "Updating to 0.4.0: waiting for 1 lease"], + [state("installing"), "Updating to 0.4.0: installing"], + [state("restarting"), "Updating to 0.4.0: restarting"], + ] as const) { + expect((await statusLines({ version: "0.3.0", update })).slice(0, 4)).toEqual([ + "Daemon: running (worker)", + "Simlock: 0.3.0", + `CLI: ${PACKAGE_VERSION}`, + line, + ]); + } + expect( + ( + await statusLines({ + version: "0.3.0", + lastUpdateFailure: failure("installing", "npm error code EACCES"), + }) + ).slice(0, 4), + ).toEqual([ + "Daemon: running (worker)", + "Simlock: 0.3.0", + `CLI: ${PACKAGE_VERSION}`, + "Last update failed (installing): npm error code EACCES", + ]); + }); + + it("simlock status says when the daemon has seen a newer Simlock version on npm", async () => { + for (const [version, latestVersion, line] of [ + ["0.3.0", "0.4.0", "Simlock: 0.3.0 (0.4.0 available)"], + ["0.9.0", "0.10.0", "Simlock: 0.9.0 (0.10.0 available)"], + ["0.4.0-rc.2", "0.4.0", "Simlock: 0.4.0-rc.2 (0.4.0 available)"], + ["0.4.0", "0.4.0", "Simlock: 0.4.0"], + ["0.5.0-rc.1", "0.4.0", "Simlock: 0.5.0-rc.1"], + ["0.3.0", null, "Simlock: 0.3.0"], + ] as const) { + expect((await statusLines({ version, latestVersion }))[1]).toBe(line); + } + }); +}); + +describe("CLI: stats", () => { + const NOW = Date.parse("2026-10-05T12:00:00.000Z"); + const HOUR = 3_600_000; + + function statsRun(argv: readonly string[], usage: SimlockAdminClient["usage"]) { + const output = outputCapture(); + const asked: { from: number; to: number; includeLedger?: boolean | undefined }[] = []; + const run = runCli( + ["stats", ...argv], + output.environmentWith({ + adminTokenFromEnv: "admin-token", + clock: new FakeClock(NOW), + connectAdmin: async (resolveCredential) => { + await resolveCredential(); + return fakeClient({ + usage: async (window) => { + asked.push(window); + return usage(window); + }, + }); + }, + }), + ); + return { asked, output, run }; + } + + it("simlock stats words each refusal as the Inputs table says, exits 2, and asks nothing of the daemon", async () => { + for (const argv of [["--since", "1h"], ["--from", "2026-10-05T10:00:00Z"], ["--json"]]) { + const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); + await expect(run, argv.join(" ")).resolves.toBe(0); + expect(output.stderr, argv.join(" ")).not.toContain("USAGE"); + } + for (const [argv, message] of [ + [ + ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], + "stats takes --since or --from, not both", + ], + [["--to", "2026-10-04T10:00:00Z"], "--to needs --from"], + [ + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], + "--from must be earlier than --to, or than now when --to is left out", + ], + [ + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T10:00:00Z"], + "--from must be earlier than --to, or than now when --to is left out", + ], + [ + ["--from", "2026-10-05T13:00:00Z"], + "--from must be earlier than --to, or than now when --to is left out", + ], + [["--since", "soon"], "Invalid duration: soon"], + [["--from", "2026-10-15"], "Invalid time: 2026-10-15 (use ISO 8601 with Z or an offset)"], + [ + ["--from", "2026-10-04T10:00:00Z", "--to", "10/15/2026"], + "Invalid time: 10/15/2026 (use ISO 8601 with Z or an offset)", + ], + ] as const) { + const { asked, output, run } = statsRun(argv, async (window) => usageAnswer(window)); + + await expect(run, argv.join(" ")).resolves.toBe(2); + expect(output.stderr, argv.join(" ")).toContain('"code":"USAGE"'); + expect(output.stderr, argv.join(" ")).toContain(`"message":"${message}"`); + expect(asked, argv.join(" ")).toEqual([]); + expect(output.stdout).toBe(""); + } + }); + + it("simlock stats --json prints the operation's output unchanged", async () => { + const answer = usageAnswer( + { from: NOW - HOUR, to: NOW }, + { + partial: true, + requesters: [ + { granted: 1, heldTotalMs: 5, id: "tok_a", label: "ci", rejected: 0, requests: 1 }, + ], + }, + ); + const { output, run } = statsRun(["--since", "1h", "--json"], async () => answer); + + await expect(run).resolves.toBe(0); + + expect(JSON.parse(output.stdout)).toEqual(answer); + expect(output.stdout.endsWith("\n")).toBe(true); + }); + + it("asks for the last 24 hours by default, and for --since up to now", async () => { + const byDefault = statsRun([], async (window) => usageAnswer(window)); + const since = statsRun(["--since", "90m"], async (window) => usageAnswer(window)); + + await byDefault.run; + await since.run; + + expect(byDefault.asked).toEqual([{ from: NOW - 24 * HOUR, to: NOW }]); + expect(since.asked).toEqual([{ from: NOW - 90 * 60_000, to: NOW }]); + }); + + it("asks for the ISO window --from and --to name, and for --from up to now", async () => { + const both = statsRun( + ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T12:00:00Z"], + async (window) => usageAnswer(window), + ); + const open = statsRun(["--from", "2026-10-05T10:00:00Z"], async (window) => + usageAnswer(window), + ); + + await both.run; + await open.run; + + expect(both.asked).toEqual([ + { from: Date.parse("2026-10-04T10:00:00Z"), to: Date.parse("2026-10-04T12:00:00Z") }, + ]); + expect(open.asked).toEqual([{ from: Date.parse("2026-10-05T10:00:00Z"), to: NOW }]); + }); + + it("prints the figures as a table, with the partial note above them", async () => { + const { output, run } = statsRun(["--since", "30d"], async (window) => + usageAnswer(window, { coversFrom: NOW - 60_000, partial: true }), + ); + + await expect(run).resolves.toBe(0); + + const lines = output.stdout.split("\n"); + expect(lines[0]).toBe("Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z"); + expect(lines[1]).toContain("Figures cover from 2026-10-05T11:59:00.000Z"); + expect(lines).toContain("Totals"); + }); + + it.each([ + ["", ["--since", "30d"]], + [" with --csv", ["--since", "30d", "--csv"]], + ])( + "prints the HISTORY_NOT_KEPT message on stderr, nothing on stdout, and exits with its code when the window ends before the history%s", + async (_name, args) => { + const { output, run } = statsRun(args, async () => { + throw fromWireError( + "HISTORY_NOT_KEPT", + "The history does not reach back to the end of that window; its figures start at 2026-10-05T11:59:00.000Z.", + { oldestTs: NOW - 60_000 }, + ); + }); + + await expect(run).resolves.toBe(12); + + expect(output.stdout).toBe(""); + expect(output.stderr).toContain('"code":"HISTORY_NOT_KEPT"'); + expect(output.stderr).toContain("its figures start at 2026-10-05T11:59:00.000Z"); + }, + ); + + const LEDGER_HEADER = + "requestId,requester,label,platform,model,workerId,source,requestedAt,grantedAt,rejectedAt,endedAt,waitMs,heldMs,turnaroundMs,outcome,probe,reason,endReason"; + const TRUNCATION_NOTICE = + "The ledger holds the newest 100000 requests of the window; ask for a shorter window for the rest."; + + it("simlock stats --csv prints the header row and one row per ledger entry with times as ISO 8601, durations as whole milliseconds (400, not 400.0), probe as true or false, and asks usage.get with includeLedger: true", async () => { + const requests = [ + { + endReason: "explicit", + endedAt: Date.parse("2026-10-05T11:09:30.000Z"), + grantedAt: Date.parse("2026-10-05T11:01:02.400Z"), + heldMs: 507_600.4, + label: "ci-runner", + model: "iPhone 16", + outcome: "granted" as const, + platform: "ios", + probe: false, + requestId: "req_8f2", + requestedAt: Date.parse("2026-10-05T11:01:02.000Z"), + requester: "tok_ci", + source: "warm", + turnaroundMs: 508_000, + waitMs: 400, + }, + { + outcome: "open" as const, + platform: "android", + probe: true, + requestId: "req_9", + requestedAt: Date.parse("2026-10-05T11:02:00.000Z"), + requester: "gw:gw-1:tok_a", + }, + ]; + const { asked, output, run } = statsRun(["--since", "1h", "--csv"], async (window) => + usageAnswer(window, { requests }), + ); + + await expect(run).resolves.toBe(0); + + expect(asked[0]?.includeLedger).toBe(true); + expect(output.stdout.split("\r\n")).toEqual([ + LEDGER_HEADER, + "req_8f2,tok_ci,ci-runner,ios,iPhone 16,,warm,2026-10-05T11:01:02.000Z,2026-10-05T11:01:02.400Z,,2026-10-05T11:09:30.000Z,400,507600,508000,granted,false,,explicit", + "req_9,gw:gw-1:tok_a,,android,,,,2026-10-05T11:02:00.000Z,,,,,,,open,true,,", + "", + ]); + }); + + it("simlock stats --csv writes the truncation notice on stderr and exits 0 when ledgerTruncated, and writes no notice otherwise", async () => { + const truncated = statsRun(["--csv"], async (window) => + usageAnswer(window, { ledgerTruncated: true, requests: [] }), + ); + const whole = statsRun(["--csv"], async (window) => usageAnswer(window, { requests: [] })); + + await expect(truncated.run).resolves.toBe(0); + await expect(whole.run).resolves.toBe(0); + + expect(truncated.output.stderr).toBe(`${JSON.stringify({ notice: TRUNCATION_NOTICE })}\n`); + expect(truncated.output.stdout.startsWith(LEDGER_HEADER)).toBe(true); + expect(whole.output.stderr).toBe(""); + expect(whole.output.stdout.startsWith(LEDGER_HEADER)).toBe(true); + }); + + it("simlock stats refuses --csv with --json as a usage error, exit 2", async () => { + const accepted = statsRun(["--csv"], async (window) => usageAnswer(window, { requests: [] })); + const both = statsRun(["--csv", "--json"], async (window) => usageAnswer(window)); + + await expect(accepted.run).resolves.toBe(0); + await expect(both.run).resolves.toBe(2); + + expect(both.output.stderr).toContain('"code":"USAGE"'); + expect(both.output.stderr).toContain('"message":"stats takes --csv or --json, not both"'); + expect(both.asked).toEqual([]); + expect(both.output.stdout).toBe(""); + }); + + it("closes the connection after the answer, and after a failure", async () => { + let closed = 0; + const output = outputCapture(); + const connect = (usage: SimlockAdminClient["usage"]) => async () => + fakeClient({ + close: async () => { + closed += 1; + }, + usage, + }); + + await runCli( + ["stats"], + output.environmentWith({ connectAdmin: connect(async (window) => usageAnswer(window)) }), + ); + await runCli( + ["stats"], + output.environmentWith({ + connectAdmin: connect(async () => { + throw new Error("boom"); + }), + }), + ); + + expect(closed).toBe(2); + }); + + it("prints its usage on --help without connecting", async () => { + const output = outputCapture(); + let connected = false; + + const exitCode = await runCli( + ["stats", "--help"], + output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }), + ); + + expect(exitCode).toBe(0); + expect(connected).toBe(false); + expect(output.stdout).toBe( + "Usage: simlock stats [--since | --from [--to ]] [--json | --csv]\n", + ); + }); +}); + +describe("simlock rotate", () => { + const heldLease = { + deviceId: "dev_1", + grantedAt: 0, + id: "lse_mine", + lastRenewedAt: 0, + idChosenByRequester: false, + ownerId: "some-other-principal", + requesterId: "test-requester", + ttlDeadline: 60_000, + ttlMs: 60_000, + }; + + const two = [heldLease, { ...heldLease, deviceId: "dev_2", id: "lse_other" }]; + + async function rotateWith( + argv: readonly string[], + leases: readonly (typeof heldLease)[], + role: "admin" | "agent" = "admin", + daemonHome?: { role: "gateway"; dataDirectory: string; gatewayPresent: true }, + ) { + const output = outputCapture(); + const rotations: unknown[] = []; + let closes = 0; + const exitCode = await runCli( + argv, + output.environmentWith({ + ...(daemonHome === undefined ? {} : { daemonHome }), + connectAdmin: async () => + fakeClient({ + role, + close: () => { + closes += 1; + return Promise.resolve(); + }, + listLeases: () => Promise.resolve({ leases: [...leases] }), + rotate: (leaseId, orientation, options) => { + rotations.push({ leaseId, orientation, options }); + return Promise.resolve({ leaseId: `${leaseId}_answered`, orientation }); + }, + }), + }), + ); + return { closes: () => closes, exitCode, output, rotations }; + } + + it("a user runs simlock rotate [--lease ], on a worker or with --gateway, and sees Rotated lease to , or a usage error", async () => { + const only = await rotateWith(["rotate", "landscape-left"], [heldLease]); + expect(only.exitCode).toBe(0); + expect(only.rotations).toEqual([ + { + leaseId: "lse_mine", + orientation: "landscape_left", + options: { requesterId: "test-requester" }, + }, + ]); + expect(only.output.stdout).toBe("Rotated lease lse_mine_answered to landscape_left\n"); + expect(only.closes()).toBe(1); + + const asAgent = await rotateWith(["rotate", "landscape-left"], [heldLease], "agent"); + expect(asAgent.exitCode).toBe(0); + expect(asAgent.rotations).toEqual([ + { leaseId: "lse_mine", orientation: "landscape_left", options: undefined }, + ]); + + const gateway = { + role: "gateway", + dataDirectory: "/simlock/gateway", + gatewayPresent: true, + } as const; + for (const argv of [ + ["--gateway", "rotate", "landscape-left"], + ["rotate", "landscape-left", "--gateway"], + ]) { + const throughGateway = await rotateWith(argv, [heldLease], "agent", gateway); + expect(throughGateway.exitCode, argv.join(" ")).toBe(0); + expect(throughGateway.rotations, argv.join(" ")).toEqual([ + { leaseId: "lse_mine", orientation: "landscape_left", options: undefined }, + ]); + expect(throughGateway.output.stdout, argv.join(" ")).toBe( + "Rotated lease lse_mine_answered to landscape_left\n", + ); + } + + for (const argv of [ + ["rotate", "--lease", "lse_other", "left"], + ["rotate", "left", "--lease", "lse_other"], + ["rotate", "left", "--lease=lse_other"], + ]) { + const named = await rotateWith(argv, two); + expect(named.exitCode).toBe(0); + expect(named.rotations).toEqual([ + { + leaseId: "lse_other", + orientation: "landscape_left", + options: { requesterId: "test-requester" }, + }, + ]); + } + + for (const [spelling, orientation] of [ + ["LEFT", "landscape_left"], + [" left ", "landscape_left"], + ["upside-down", "portrait_upside_down"], + ]) { + const spelled = await rotateWith(["rotate", spelling as string], [heldLease]); + expect(spelled.exitCode).toBe(0); + expect(spelled.output.stdout).toBe(`Rotated lease lse_mine_answered to ${orientation}\n`); + } + + const refusals: [readonly string[], readonly (typeof heldLease)[], RegExp][] = [ + [["rotate", "sideways"], [heldLease], /^Unknown orientation sideways; use portrait,/], + [["rotate", "auto"], [heldLease], /^Unknown orientation auto;/], + [["rotate", "landscapeLeft"], [heldLease], /^Unknown orientation landscapeLeft;/], + [["rotate"], [heldLease], /^Expected orientation/], + [["rotate", "left"], [], /test-requester/], + [["rotate", "left"], two, /lse_mine.*lse_other.*--lease /s], + ]; + for (const [argv, leases, message] of refusals) { + const refused = await rotateWith(argv, leases); + expect(refused.exitCode).toBe(2); + expect(refused.rotations).toEqual([]); + expect(JSON.parse(refused.output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { code: "USAGE", message: expect.stringMatching(message) }, + }); + } + + const unsupported = outputCapture(); + let unsupportedCloses = 0; + await expect( + runCli( + ["rotate", "left", "--lease", "lse_android"], + unsupported.environmentWith({ + connectAdmin: async () => + fakeClient({ + close: () => { + unsupportedCloses += 1; + return Promise.resolve(); + }, + rotate: () => + Promise.reject( + new SimlockError( + "ROTATION_UNSUPPORTED", + "domain", + "Lease lse_android is on an Android emulator; only an iOS simulator can be rotated", + { + class: "emulator", + leaseId: "lse_android", + physical: false, + platform: "android", + }, + ), + ), + }), + }), + ), + ).resolves.toBe(12); + expect(JSON.parse(unsupported.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ + error: { + code: "ROTATION_UNSUPPORTED", + message: + "Lease lse_android is on an Android emulator; only an iOS simulator can be rotated", + }, + }); + expect(unsupported.stdout).toBe(""); + expect(unsupportedCloses).toBe(1); + + const help = await rotateWith(["rotate", "--help"], [heldLease]); + expect(help.exitCode).toBe(0); + expect(help.output.stdout).toBe( + "Usage: simlock rotate [--lease ]\n", + ); + expect(help.rotations).toEqual([]); + const listing = outputCapture(); + await runCli(["--help"], listing.environmentWith()); + expect(listing.stdout).toContain(" rotate [--lease ]"); + }); +}); diff --git a/src/cli/index-worker.test.ts b/src/cli/index-worker.test.ts new file mode 100644 index 000000000..ba395a7dc --- /dev/null +++ b/src/cli/index-worker.test.ts @@ -0,0 +1,858 @@ +import { describe, expect, it } from "vitest"; +import { FakeClock } from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import type { SimlockAdminClient, StatusGetOutput } from "../simlock-client/client.js"; +import { type WorkerView } from "../admin/index.js"; +import { runCli } from "./index.js"; +import { EMPTY_STATUS, fakeClient, outputCapture } from "./test-index.js"; + +describe("CLI: worker commands (ADR 0005 §8/§23)", () => { + const connectedWorker = { + capacity: { + android: { + atRamBudget: false, + limit: 2, + maxRunning: 2, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }, + global: { maxRunning: 4, overLimit: false, reserved: 0, running: 1, warm: 1 }, + ios: { + atRamBudget: false, + limit: 2, + maxRunning: 2, + overLimit: false, + reserved: 0, + running: 1, + used: 1, + warm: 1, + }, + }, + catalog: [], + connection: "connected" as const, + devices: [], + drained: false, + id: "wrk_1", + label: "mac-mini-1", + lastSeenAt: 1_000, + leases: [ + { + deviceId: "dev_1", + grantedAt: 1, + id: "lease_1", + lastRenewedAt: 1, + idChosenByRequester: false, + ownerId: "agent-1", + requesterId: "agent-1", + ttlDeadline: 2, + ttlMs: 1, + }, + ], + }; + + it("renders each worker's state, capacity and lease count", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers: [connectedWorker] }) }), + }); + + await expect(runCli(["worker", "list"], environment)).resolves.toBe(0); + + expect(output.stdout).toContain("wrk_1 (mac-mini-1): connected -- ios 1/2"); + // why: No host facts on this view yet, so the line ends at the lease count. + expect(output.stdout).toContain("1 lease(s)\n"); + }); + + it("prints each worker's operating system, architecture, and tool versions", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + listWorkers: () => + Promise.resolve({ + workers: [ + { + ...connectedWorker, + host: { + arch: "arm64", + os: "macOS", + osVersion: "15.5", + tools: [ + { build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }, + { name: "emulator", platform: "android" as const, version: "35.4.9" }, + ], + }, + }, + ], + }), + }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout).toContain( + "1 lease(s) -- macOS 15.5 arm64; xcode 16.4 (16F6), emulator 35.4.9", + ); + }); + + describe("a worker's Load line", () => { + const GIB = 2 ** 30; + const PARTS = { + cpu: { busyPercent: 34.2, cores: 10, limited: false }, + disk: { freeBytes: 120_688_581_018, totalBytes: 530_857_957_786 }, + memory: { + availableBytes: 10_522_669_875, + limited: false, + pressure: "normal" as const, + swapUsedBytes: 1_288_490_189, + totalBytes: 32 * GIB, + }, + }; + const READING = + "CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free"; + const NOW = 61_000; + const install = { + component: "26.4", + platform: "ios" as const, + since: 1_000, + state: "downloading" as const, + waiters: 2, + }; + const printWorkers = async (workers: unknown[]) => { + const output = outputCapture(); + await runCli( + ["worker", "list"], + output.environmentWith({ + clock: new FakeClock(NOW), + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers } as never) }), + }), + ); + return output.stdout; + }; + + it("simlock worker list prints a connected or starting worker's Load line after its own line and before its install lines", async () => { + const stdout = await printWorkers([ + { + ...connectedWorker, + installs: [install], + load: { ...PARTS, readAt: NOW - 12_000 }, + }, + { + connection: "connected", + drained: false, + health: "starting", + id: "wrk_2", + lastSeenAt: 1, + load: { memory: PARTS.memory, readAt: NOW - 3_000 }, + }, + ]); + + const lines = stdout.trimEnd().split("\n"); + expect(lines[0]).toMatch(/^wrk_1 \(mac-mini-1\): connected -- /); + expect(lines[1]).toBe(` Load: ${READING} · read 12s ago`); + expect(lines[2]).toMatch(/^ {2}Install ios 26\.4: downloading for 60s, 2 waiters$/); + expect(lines[3]).toMatch(/^wrk_2: connected, starting/); + expect(lines[4]).toBe( + " Load: CPU unavailable · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk unavailable · read 3s ago", + ); + }); + + it("simlock worker list prints `no reading while disconnected` for a disconnected worker, and no Load line for an incompatible one or one that sent none", async () => { + const stdout = await printWorkers([ + { ...connectedWorker, connection: "disconnected", drained: true, id: "wrk_a" }, + { + connection: "incompatible", + drained: false, + id: "wrk_b", + lastSeenAt: 1, + protocol: { gateway: { max: 35, min: 35 }, worker: { max: 4, min: 4 } }, + }, + { ...connectedWorker, id: "wrk_c" }, + { ...connectedWorker, id: "wrk_d", load: { ...PARTS, readAt: NOW - 1_000 } }, + ]); + + const lines = stdout.trimEnd().split("\n"); + const loadAfter = (id: string) => { + const index = lines.findIndex( + (line) => line.startsWith(`${id}:`) || line.startsWith(`${id} (`), + ); + const next = lines[index + 1]; + return next?.startsWith(" Load:") === true ? next : undefined; + }; + expect(loadAfter("wrk_d")).toBe(` Load: ${READING} · read 1s ago`); + expect(loadAfter("wrk_a")).toBe(" Load: no reading while disconnected"); + expect(loadAfter("wrk_b")).toBeUndefined(); + expect(loadAfter("wrk_c")).toBeUndefined(); + }); + + it("simlock worker list prints `read 0s ago` for a readAt ahead of its clock", async () => { + const stdout = await printWorkers([ + { ...connectedWorker, load: { ...PARTS, readAt: NOW + 9_000 } }, + ]); + + expect(stdout).toContain(` Load: ${READING} · read 0s ago\n`); + }); + + it("simlock worker list rounds a reading's age up to whole seconds", async () => { + const stdout = await printWorkers([ + { ...connectedWorker, load: { ...PARTS, readAt: NOW - 1_001 } }, + ]); + + expect(stdout).toContain(` Load: ${READING} · read 2s ago\n`); + }); + + it("simlock status against a gateway prints the gateway's own Load line, and each worker's Load line aged from the status time", async () => { + const output = outputCapture(); + const status: StatusGetOutput = { + ...EMPTY_STATUS, + daemon: { + health: "running", + mode: "gateway", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + load: { ageMs: 4_000, ...PARTS }, + workers: [ + { + catalog: [], + connection: "connected", + devices: [], + drained: false, + id: "wrk_1", + lastSeenAt: 1, + leases: [], + load: { ...PARTS, readAt: NOW - 12_000 }, + }, + ], + }; + + await runCli( + ["status"], + output.environmentWith({ + clock: new FakeClock(NOW), + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), + }), + ); + + expect(output.stdout).toContain(`\nLoad: ${READING} · read 4s ago\n`); + expect(output.stdout).toContain(`\n Load: ${READING} · read 12s ago\n`); + }); + }); + + it("prints a starting worker as starting, with its leases unknown rather than none", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + listWorkers: () => + Promise.resolve({ + workers: [ + { + connection: "connected" as const, + drained: true, + health: "starting" as const, + id: "wrk_1", + lastSeenAt: 1, + }, + ], + }), + }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout).toBe( + "wrk_1: connected, starting, drained -- capacity unknown, leases unknown\n", + ); + }); + + it("prints each worker's installs in progress under it", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + clock: new FakeClock(61_000), + connectAdmin: async () => + fakeClient({ + listWorkers: () => + Promise.resolve({ + workers: [ + { + ...connectedWorker, + installs: [ + { + component: "26.4", + platform: "ios" as const, + since: 1_000, + state: "downloading" as const, + waiters: 2, + }, + ], + }, + { + ...connectedWorker, + id: "wrk_2", + installs: [ + { + component: "35", + platform: "android" as const, + since: 31_000, + state: "waiting" as const, + waiters: 1, + }, + ], + }, + { ...connectedWorker, id: "wrk_3", installs: [] }, + ], + }), + }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout).toContain( + "1 lease(s)\n Install ios 26.4: downloading for 60s, 2 waiters\nwrk_2 (mac-mini-1): connected", + ); + expect(output.stdout).toContain( + "1 lease(s)\n Install android 35: waiting for 30s, 1 waiter\nwrk_3 (mac-mini-1): connected", + ); + expect(output.stdout.match(/Install /g)).toHaveLength(2); + }); + + it("says so plainly when no worker has ever connected", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ connectAdmin: async () => fakeClient() }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout).toContain("No workers have connected"); + }); + + it("names the protocol skew on an incompatible worker", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ + listWorkers: () => + Promise.resolve({ + workers: [ + { + ...connectedWorker, + connection: "incompatible" as const, + leases: [], + protocol: { gateway: { max: 5, min: 5 }, worker: { max: 4, min: 4 } }, + }, + ], + }), + }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout).toContain("incompatible"); + expect(output.stdout).toContain("protocol 4-4 vs gateway 5-5"); + }); + + it("worker list shows a worker's version, the latest only when newer, and its update stage", async () => { + const waiting = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const cases: readonly { + expected: string; + latestVersion: string | null; + update: WorkerView["update"]; + version: string | undefined; + }[] = [ + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases", + latestVersion: "0.4.0", + update: waiting, + version: "0.3.0", + }, + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: installing", + latestVersion: "0.4.0", + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + }, + version: "0.3.0", + }, + { + expected: " -- updating to 0.4.0: waiting for 1 lease", + latestVersion: null, + update: { ...waiting, leases: 1 }, + version: undefined, + }, + { expected: " -- simlock 0.4.0", latestVersion: "0.4.0", update: null, version: "0.4.0" }, + { expected: " -- simlock 0.5.0", latestVersion: "0.4.0", update: null, version: "0.5.0" }, + { expected: " -- simlock 0.4.0", latestVersion: null, update: null, version: "0.4.0" }, + ]; + for (const { expected, latestVersion, update, version } of cases) { + const output = outputCapture(); + const worker = { + ...connectedWorker, + latestVersion, + update, + ...(version === undefined ? {} : { version }), + }; + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers: [worker] }) }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout.trimEnd().slice(-expected.length)).toBe(expected); + if (!expected.includes("available")) expect(output.stdout).not.toContain("available"); + if (update === null) expect(output.stdout).not.toContain("updating"); + } + }); + + it("emits the raw contract value under --json", async () => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers: [connectedWorker] }) }), + }); + + await runCli(["worker", "list", "--json"], environment); + + expect(JSON.parse(output.stdout)).toMatchObject({ workers: [{ id: "wrk_1" }] }); + }); + + it("drains, undrains and removes by id", async () => { + const calls: string[] = []; + const client = fakeClient({ + drainWorker: (input) => { + calls.push(`drain:${input.workerId}`); + return Promise.resolve({ drained: true as const, workerId: input.workerId }); + }, + removeWorker: (input) => { + calls.push(`remove:${input.workerId}`); + return Promise.resolve({ removed: true, workerId: input.workerId }); + }, + undrainWorker: (input) => { + calls.push(`undrain:${input.workerId}`); + return Promise.resolve({ drained: false as const, workerId: input.workerId }); + }, + }); + const environment = outputCapture().environmentWith({ connectAdmin: async () => client }); + + await expect(runCli(["worker", "drain", "wrk_1"], environment)).resolves.toBe(0); + await expect(runCli(["worker", "undrain", "wrk_1"], environment)).resolves.toBe(0); + await expect(runCli(["worker", "remove", "wrk_2"], environment)).resolves.toBe(0); + + expect(calls).toEqual(["drain:wrk_1", "undrain:wrk_1", "remove:wrk_2"]); + }); + + it("requires a worker id, and rejects an unknown subcommand", async () => { + const environment = outputCapture().environmentWith({ + connectAdmin: async () => fakeClient(), + }); + + await expect(runCli(["worker", "drain"], environment)).resolves.toBe(2); + await expect(runCli(["worker", "explode"], environment)).resolves.toBe(2); + }); + + it("prints usage without connecting to anything", async () => { + const output = outputCapture(); + let connected = false; + const environment = output.environmentWith({ + connectAdmin: async () => { + connected = true; + return fakeClient(); + }, + }); + + await expect(runCli(["worker"], environment)).resolves.toBe(0); + + expect(output.stdout).toContain("simlock worker list"); + expect(connected).toBe(false); + }); + + it("mints a worker join token (ADR 0005 §8)", async () => { + const roles: string[] = []; + const client = fakeClient({ + createToken: (input) => { + roles.push(input.role); + return Promise.resolve({ + secret: "sec_join", + token: { createdAt: 0, id: "tok_1", role: input.role }, + }); + }, + }); + const output = outputCapture(); + const environment = output.environmentWith({ connectAdmin: async () => client }); + + await expect(runCli(["token", "create", "--role", "worker"], environment)).resolves.toBe(0); + + expect(roles).toEqual(["worker"]); + expect(JSON.parse(output.stdout)).toMatchObject({ token: { role: "worker" } }); + }); + + it("still rejects a role that is not one of the three", async () => { + const environment = outputCapture().environmentWith({ + connectAdmin: async () => fakeClient(), + }); + + await expect(runCli(["token", "create", "--role", "root"], environment)).resolves.toBe(2); + }); +}); + +describe("CLI: a worker's capabilities", () => { + const platform = { + atRamBudget: false, + limit: 2, + maxRunning: 2, + overLimit: false, + reserved: 0, + running: 0, + used: 0, + warm: 0, + }; + const worker = (overrides: Record = {}) => ({ + capacity: { + android: platform, + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: platform, + }, + catalog: [], + connection: "connected" as const, + devices: [], + drained: false, + id: "wrk_1", + label: "mac-mini", + lastSeenAt: 1_000, + leases: [], + ...overrides, + }); + const listed = async (workers: unknown[]): Promise => { + const output = outputCapture(); + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers } as never) }), + }); + await runCli(["worker", "list"], environment); + return output.stdout; + }; + + it("worker list prints ` -- agent-device available`, ` -- agent-device off` and ` -- agent-device unavailable: ` on the worker's line", async () => { + const shown = await listed([ + worker({ capabilities: [{ name: "agent-device", state: "available" }] }), + worker({ capabilities: [{ name: "agent-device", state: "off" }], id: "wrk_2" }), + worker({ + capabilities: [ + { + name: "agent-device", + reason: "found 0.20.1, need 0.21.28 or later", + state: "unavailable", + }, + ], + id: "wrk_3", + }), + worker({ + capabilities: [{ name: "agent-device", reason: "starting", state: "unavailable" }], + id: "wrk_4", + }), + ]); + + expect(shown).toContain("wrk_1 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s)"); + expect(shown).toMatch(/^wrk_1 .* -- agent-device available$/m); + expect(shown).toMatch(/^wrk_2 .* -- agent-device off$/m); + expect(shown).toMatch( + /^wrk_3 .* -- agent-device unavailable: found 0\.20\.1, need 0\.21\.28 or later$/m, + ); + expect(shown).toMatch(/^wrk_4 .* -- agent-device unavailable: starting$/m); + }); + + it("simlock worker list shows each worker's memory step as the end of its first line, and nothing at none, with no field, or for a disconnected worker", async () => { + const withStep = ( + id: string, + step: "none" | "warming-paused" | "new-devices-paused", + extra = {}, + ) => + worker({ + capabilities: [{ name: "agent-device", state: "available" }], + capacity: { ...worker().capacity, memoryPressure: { step } }, + id, + ...extra, + }); + const shown = await listed([ + withStep("wrk_1", "warming-paused"), + withStep("wrk_2", "new-devices-paused"), + withStep("wrk_3", "none"), + worker({ id: "wrk_4" }), + withStep("wrk_5", "new-devices-paused", { connection: "disconnected" }), + ]); + + const lines = shown.trimEnd().split("\n"); + const first = (id: string) => lines.find((line) => line.startsWith(`${id} `)); + expect(first("wrk_1")).toBe( + "wrk_1 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available -- memory: warming paused", + ); + expect(first("wrk_2")).toBe( + "wrk_2 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available -- memory: new devices paused", + ); + expect(first("wrk_3")).toBe( + "wrk_3 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available", + ); + expect(first("wrk_4")).toBe("wrk_4 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s)"); + expect(first("wrk_5")).toBe( + "wrk_5 (mac-mini): disconnected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available", + ); + }); + + it("worker list prints two entries in the order the worker sent them", async () => { + const shown = await listed([ + worker({ + capabilities: [ + { name: "zeta", state: "off" }, + { name: "agent-device", state: "available" }, + ], + }), + ]); + + expect(shown).toMatch(/^wrk_1 .* -- zeta off -- agent-device available$/m); + }); + + it("worker list prints nothing for a worker whose list is absent", async () => { + const shown = await listed([worker()]); + + expect(shown).toMatch(/^wrk_1 .* 0 lease\(s\)$/m); + }); + + it("worker list puts the suffix on the worker's line, before its install lines", async () => { + const shown = await listed([ + worker({ + capabilities: [{ name: "agent-device", state: "available" }], + installs: [ + { component: "26.4", platform: "ios", since: 1_000, state: "downloading", waiters: 2 }, + ], + }), + ]); + + expect(shown).toContain( + "-- agent-device available\n Install ios 26.4: downloading for 0s, 2 waiters", + ); + }); + + it("status against a gateway prints the same suffix in its workers block", async () => { + const output = outputCapture(); + const status = { + capacity: { + android: platform, + global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, + ios: platform, + }, + daemon: { + health: "running" as const, + mode: "gateway" as const, + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + devices: [], + host: { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] }, + leases: [], + queueDepth: 0, + workers: [ + worker({ + capabilities: [{ name: "agent-device", reason: "starting", state: "unavailable" }], + }), + ], + }; + const environment = output.environmentWith({ + connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status as never) }), + }); + + await runCli(["status"], environment); + + expect(output.stdout).toMatch(/^wrk_1 .* -- agent-device unavailable: starting$/m); + }); +}); + +describe("simlock worker update", () => { + type UpdateAnswer = Awaited>; + type View = Awaited>["workers"][number]; + const view = (overrides: Partial = {}): View => ({ + connection: "connected", + drained: false, + id: "3f81a2c4", + label: "mac-studio-2", + lastSeenAt: 1_000, + version: "0.3.0", + ...overrides, + }); + const run = { forced: false, from: "0.3.0", startedAt: 1_000, to: "0.4.0" } as const; + const answer = (overrides: Partial = {}): UpdateAnswer => ({ + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: { ...run, stage: "installing" }, + upToDate: false, + version: "0.3.0", + workerId: "3f81a2c4", + ...overrides, + }); + + it("follows the update to is running, reports a failure with exit 1, says why the worker is up to date, detaches, and refuses on a worker daemon", async () => { + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm exited 1", + stage: "installing", + to: "0.4.0", + } as const; + const cases: readonly { + readonly name: string; + readonly argv: readonly string[]; + readonly answer: UpdateAnswer | SimlockError; + readonly lists?: readonly View[]; + readonly code: number; + readonly stdout?: string; + readonly stderr?: string; + readonly polls?: number; + }[] = [ + { + name: "follows to running", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0", "--force"], + answer: answer(), + lists: [ + view({ update: { ...run, leases: 2, stage: "waiting" } }), + view({ update: { ...run, stage: "installing" } }), + view({ connection: "disconnected", update: { ...run, stage: "restarting" } }), + view({ update: { ...run, stage: "restarting" }, version: "0.4.0" }), + view({ update: null, version: "0.4.0" }), + ], + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is running 0.4.0\n", + polls: 5, + }, + { + name: "fails with the stage and message", + argv: ["worker", "update", "3f81a2c4"], + answer: answer(), + lists: [view({ lastUpdateFailure: failure, update: null })], + code: 1, + stdout: "Update failed (installing): npm exited 1\n", + polls: 1, + }, + { + name: "already on the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)\n", + polls: 1, + }, + { + name: "already running the requested version", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already running 0.4.0\n", + polls: 1, + }, + { + name: "newer than the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ + resolved: "0.3.0", + started: false, + update: null, + upToDate: true, + version: "0.4.0", + }), + code: 0, + stdout: + "mac-studio-2 (3f81a2c4) is newer than the latest (0.3.0). Use --version to install another version.\n", + polls: 1, + }, + { + name: "detach", + argv: ["worker", "update", "3f81a2c4", "--detach"], + answer: answer(), + code: 0, + stdout: + "Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.\n", + polls: 1, + }, + { + name: "worker daemon", + argv: ["worker", "update", "3f81a2c4"], + answer: new SimlockError( + "UNSUPPORTED_IN_WORKER_MODE", + "domain", + "worker.update is a gateway operation", + { operation: "worker.update" }, + ), + code: 2, + stderr: "worker.update is a gateway operation", + polls: 0, + }, + ]; + for (const spec of cases) { + const output = outputCapture(); + const clock = new FakeClock(0); + const sent: unknown[] = []; + const lists = [...(spec.lists ?? [view()])]; + let polls = 0; + const environment = output.environmentWith({ + clock, + connectAdmin: async () => + fakeClient({ + listWorkers: async () => { + polls += 1; + const next = lists.length > 1 ? lists.shift() : lists[0]; + return { workers: next === undefined ? [] : [next] }; + }, + updateWorker: async (input) => { + sent.push(input); + if (spec.answer instanceof Error) throw spec.answer; + return spec.answer; + }, + }), + // why: a follow that never reaches an outcome fails on this line, not by timing out. + sleep: async (milliseconds) => { + if (polls > (spec.lists ?? []).length + 2) + throw new Error(`${spec.name}: the follow never ended`); + clock.advance(milliseconds); + }, + }); + + const code = await runCli(spec.argv, environment); + + expect({ code, name: spec.name, stdout: output.stdout }).toEqual({ + code: spec.code, + name: spec.name, + stdout: spec.stdout ?? "", + }); + if (spec.stderr !== undefined) expect(output.stderr).toContain(spec.stderr); + expect({ name: spec.name, polls }).toEqual({ name: spec.name, polls: spec.polls ?? 0 }); + expect(sent).toEqual([ + { + workerId: "3f81a2c4", + ...(spec.argv.includes("--version") ? { version: "0.4.0" } : {}), + ...(spec.argv.includes("--force") ? { force: true } : {}), + }, + ]); + } + }); +}); diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index c3efab45a..6cbe58a0e 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -1,140 +1,19 @@ -import { testConfig as configWithDefaults } from "../core/testing.js"; import { EventEmitter } from "node:events"; -import { readFileSync } from "node:fs"; -import { appendFile, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { createServer, type Socket } from "node:net"; -import { Readable } from "node:stream"; - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { testComponentWiring, testStateOf } from "../core/testing.js"; - -import { EventBus } from "../bus/index.js"; -import { ringEventHistory, seedHistory } from "../history/testing.js"; -import { CleanupReaper, STARTUP_READ_LIMIT_MS, type Config } from "../core/index.js"; -import { FakeDriver, loadTestRegistry } from "../core/testing.js"; -import { - CryptoTokenSecrets, - FakeClock, - FakeDaemonLauncher, - FakeParentWatch, - FakeServiceManager, - IpcError, - launcherFor, - LaunchdServiceManager, - ScriptedProcessRunner, - MemoryFilesystem, - MemoryIpcTransport, - resolveDaemonSocketPath, - FakeKeySetFetcher, - NodeFilesystem, - NodeIpcTransport, - NoopLogger, - serviceDefinition, - type ServiceManager, - type ServiceState, - type Filesystem, - type IdGenerator, - type IpcConnection, - type IpcConnector, -} from "../ports/index.js"; -import { - FakeHttpJsonClient, - FakeSystemStats, - FakeTcpListenerFactory, - FakeTcpStreamConnector, - FakeWebSocketClientConnector, -} from "../ports/index.js"; -import { type Auth, createAuth, tokensOwner } from "../auth/index.js"; -import { openMemoryStateDatabase } from "../auth/testing.js"; -import { DaemonEndpointHost } from "../daemon/connection-host.js"; -import { DaemonServer } from "../daemon/server.js"; -import { - PROTOCOL_VERSION_RANGE, - fromWireError, - SimlockError, - waitingRequestSchema, - type AnySimlockError, - type UsageOutput, -} from "../contract/index.js"; -import type { - CatalogGetOutput, - DeviceRecoveredPush, - DeviceUnhealthyPush, - DoctorReport, - EventPush, - LeaseGrant, - LeaseListOutput, - LeaseRecord, - ListGetOutput, - SimlockAdminClient, - SimlockConfig, - StatusGetOutput, - TokenListOutput, -} from "../simlock-client/client.js"; -import { connectSimlockAdmin, type WorkerView } from "../admin/index.js"; -import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; +import { describe, expect, it } from "vitest"; +import { MemoryFilesystem } from "../ports/index.js"; +import { SimlockError } from "../contract/index.js"; +import type { DoctorReport, LeaseGrant, SimlockAdminClient } from "../simlock-client/client.js"; import type { ToolOutcome } from "../instructions/index.js"; -import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; -import { buildCliEnvironment } from "./environment.js"; +import { runCli, type CliEnvironment } from "./index.js"; import { readLogFile } from "./log-file.js"; -import { readPipedStdin } from "./stdio.js"; -import { parseDuration } from "./duration.js"; -import { createTestEngine } from "../leasing/testing.js"; -import { promiseState } from "../test-support/promise-state.js"; - -const gibibyte = 1024 ** 3; - -/** - * why: Yields to the macrotask queue, so `runLease` gets past `requestLease` and reaches its wait - * point -- its push listeners registered and its renew timer armed -- before a test fires a - * push or advances the clock. - */ -const settle = (): Promise => new Promise((resolve) => setTimeout(resolve, 0)); -const detachedGrant: LeaseGrant = { - device: { - physical: false, - id: "device-1", - driverDeviceId: "ABCD", - spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, - mode: "full", - }, - environment: {}, - lease: { - id: "lse_env", - deviceId: "device-1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 61_000, - }, - timing: { - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, -}; -const runningDaemons: DaemonServer[] = []; -const temporaryDirectories: string[] = []; - -afterEach(async () => { - await Promise.all(runningDaemons.splice(0).map((daemon) => daemon.stop("test"))); - await Promise.all( - temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), - ); -}); - -// why: read straight from the manifest so the expected version is not computed by the code under test. -const PACKAGE_VERSION = ( - JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8")) as { - version: string; - } -).version; +import { + PACKAGE_VERSION, + detachedGrant, + fakeClient, + outputCapture, + settle, + simlockError, +} from "./test-index.js"; describe("readLogFile", () => { it("returns just the current log when there is no rotated generation", async () => { @@ -817,10858 +696,40 @@ describe("CLI: exit codes", () => { ).resolves.toBe(0); expect(seen).toEqual([{ args: ["--help"], tool: "adb" }]); }); +}); - /** - * why: ADR 0005 §19c's other half. Same command, same refusals, same exit code -- the only thing - * that changes is where the process runs and how its output gets here. `runPassthrough` is - * deliberately wired in each of these: a remote invocation that quietly spawned locally - * would pass every assertion about output and exit code while doing the wrong thing. - * - * The switch is the daemon's own `mode` (§19c): a `gateway` owns no devices, so a command - * it resolved could not run here anyway. Nothing about the CLI's transport decides it. - */ - describe("remote passthrough (ADR 0005 device.exec)", () => { - const gatewayStatus = { - ...EMPTY_STATUS, - daemon: { - health: "running" as const, - mode: "gateway" as const, - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, +describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", () => { + it("ignores a lease-lost push for a lease this connection does not hold (own-lease-id filter)", async () => { + const output = outputCapture(); + let leaseLostListener: + | ((push: { leaseId: string; deviceId: string; reason: string }) => void) + | undefined; + let released = false; + const signals = new EventEmitter(); + const client = fakeClient({ + onLeaseLost: (listener) => { + leaseLostListener = listener; + return () => {}; + }, + releaseLease: () => { + released = true; + return Promise.resolve({ leaseId: "lse_1" }); }, - }; - const ownLease = { - deviceId: "dev_1", - grantedAt: 0, - id: "lse_mine", - lastRenewedAt: 0, - idChosenByRequester: false, - ownerId: "some-other-principal", - requesterId: "test-requester", - ttlDeadline: 60_000, - ttlMs: 60_000, - }; - - it("runs the command on the daemon, prints the streamed output, and exits with its code", async () => { - const output = outputCapture(); - const seen: unknown[] = []; - let ranLocally = false; - - await expect( - runCli( - ["adb", "shell", "getprop"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: (input, options) => { - seen.push(input); - options?.onOutput?.({ chunk: "ro.build", stream: "stdout" }); - options?.onOutput?.({ chunk: "a warning\n", stream: "stderr" }); - options?.onOutput?.({ chunk: ".version=15\n", stream: "stdout" }); - return Promise.resolve({ exitCode: 9 }); - }, - }), - runPassthrough: async () => { - ranLocally = true; - return 0; - }, - }), - ), - ).resolves.toBe(9); - - expect(ranLocally).toBe(false); - // why: `requesterId` rides along because this connection resolved to the admin role, where - // the operation reads it (and requires it to match the lease). See the agent-role case - // below for the other half. - expect(seen).toEqual([ - { - args: ["shell", "getprop"], - leaseId: "lse_mine", - requesterId: "test-requester", - stdin: "", - tool: "adb", - }, - ]); - // why: Chunks land on the stream they were written to, unjoined and in order -- a caller - // piping this sees what it would have seen locally. - expect(output.stdout).toBe("ro.build.version=15\n"); - // why: The agent-fallback notice this fixture's credential-less connection writes is the only - // other thing on stderr; the command's own chunk lands there verbatim. - expect(output.stderr).toContain("a warning\n"); - }); - - it("names the lease from the requester id rather than from a flag of its own", async () => { - // why: Every argument after the tool name belongs to the tool, so identity is - // the only thing left to select a lease with -- and `--agent-id`/`SIMLOCK_AGENT_ID` - // already is that identity. A requester holding no lease is a usage error naming it. - const output = outputCapture(); - - await expect( - runCli( - ["adb", "devices"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => - Promise.resolve({ - leases: [{ ...ownLease, id: "lse_theirs", requesterId: "someone-else" }], - }), - }), - }), - ), - ).resolves.toBe(2); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("test-requester") }, - }); - }); - - it("exits 10 when the daemon killed the command for outrunning exec.timeoutMs", async () => { - // why: The exit code comes from `ERROR_TABLE`'s own column (ADR 0003 §7): 10, the same code - // the other "you ran out of time" outcome (`QUEUE_TIMEOUT`) uses, rather than the - // generic 1 a command that merely failed would get. - const output = outputCapture(); - - await expect( - runCli( - ["adb", "logcat"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: () => - Promise.reject( - new SimlockError( - "EXEC_TIMEOUT", - "domain", - "`adb logcat` exceeded exec.timeoutMs (600000ms) and was killed", - {}, - ), - ), - }), - }), - ), - ).resolves.toBe(10); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "EXEC_TIMEOUT", message: expect.stringContaining("exec.timeoutMs") }, - }); - }); - - it("without --lease, for a requester holding two leases, fails with a usage error naming both leases and --lease", async () => { - const output = outputCapture(); - - await expect( - runCli( - ["simctl", "list"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => - Promise.resolve({ - leases: [ownLease, { ...ownLease, deviceId: "dev_2", id: "lse_other" }], - }), - exec: () => Promise.reject(new Error("must not run")), - }), - }), - ), - ).resolves.toBe(2); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { - code: "USAGE", - message: expect.stringMatching(/lse_mine.*lse_other.*--lease /s), - }, - }); - }); - - it("takes `--lease ` out of the arguments and runs against that lease", async () => { - // why: The flag exists only on this path, so it is stripped here rather than parsed with the - // CLI's own flags: locally every argument after the tool name belongs to the tool. - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "--lease", "lse_named", "shell", "getprop"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.reject(new Error("must not need the lease list")), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([ - { - args: ["shell", "getprop"], - leaseId: "lse_named", - requesterId: "test-requester", - stdin: "", - tool: "adb", - }, - ]); - }); - - it("sends stdin on device.exec only when --stdin was given", async () => { - // why: `readStdin` is a `CliEnvironmentPorts` seam this test injects directly -- it exercises - // the wiring from that port's result to `device.exec`'s `stdin`, not the default - // `readPipedStdin` implementation that reads a real pipe to EOF; that one is tested on - // its own below, since a fake here would not exercise it at all. - const cases = [ - { - argv: ["adb", "shell", "cat"], - readable: false, - expected: { args: ["shell", "cat"], stdin: "", tool: "adb" }, - }, - { - argv: ["simctl", "list"], - readable: false, - expected: { args: ["list"], stdin: "", tool: "simctl" }, - }, - { - argv: ["adb", "--stdin", "shell", "cat"], - readable: true, - expected: { args: ["shell", "cat"], stdin: "piped payload\n", tool: "adb" }, - }, - { - argv: ["simctl", "--stdin", "list"], - readable: true, - expected: { args: ["list"], stdin: "piped payload\n", tool: "simctl" }, - }, - ]; - - for (const { argv, readable, expected } of cases) { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - argv, - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - readStdin: readable - ? async () => "piped payload\n" - : () => Promise.reject(new Error("stdin must not be read without --stdin")), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ ...expected, leaseId: "lse_mine", requesterId: "test-requester" }]); - } - }); - - it("sends no requesterId when the connection fell back to the agent role", async () => { - // why: An agent session is gated on the lease it owns and its `requesterId` is ignored - // entirely, so sending one would be noise that reads like authorization. - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "devices"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - role: "agent", - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ args: ["devices"], leaseId: "lse_mine", stdin: "", tool: "adb" }]); - }); - - it("spawns locally against a worker, accepting and ignoring --lease", async () => { - // why: The other half of the same switch, and the reason `--lease` is accepted on both: one - // command line has to work against either kind of daemon. A worker takes - // the flag, drops it, and passes the rest to the tool untouched. - const output = outputCapture(); - const ran: unknown[] = []; - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "--lease", "lse_ignored", "shell", "getprop"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - exec: () => Promise.reject(new Error("must not exec remotely")), - resolvePassthrough: (input) => { - seen.push(input); - return Promise.resolve({ args: ["-P", "5038"], command: "adb", env: {} }); - }, - }), - runPassthrough: async (command) => { - ran.push(command); - return 0; - }, - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ args: ["shell", "getprop"], tool: "adb" }]); - expect(ran).toHaveLength(1); - }); - - it("relabels an unknown passthrough tool as USAGE, like a refusal", async () => { - // why: ADR 0005 §19c: the CLI keeps no copy of the refusal list and relabels whatever the - // daemon answers. `UNKNOWN_PASSTHROUGH_TOOL` is the other half of that answer, and it - // has to read the same as a refused verb does -- exit 2, one `USAGE` line. - const output = outputCapture(); - - await expect( - runCli( - ["adb", "devices"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: () => - Promise.reject( - new SimlockError( - "UNKNOWN_PASSTHROUGH_TOOL", - "domain", - "No driver provides a adb passthrough", - { tool: "adb" }, - ), - ), - }), - }), - ), - ).resolves.toBe(2); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("No driver provides") }, - }); - }); - - it("takes --lease as the last argument, and refuses an empty one", async () => { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "--lease", "lse_only"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([ - { args: [], leaseId: "lse_only", requesterId: "test-requester", stdin: "", tool: "adb" }, - ]); - - const missingValue = outputCapture(); - await expect( - runCli( - ["adb", "--lease"], - missingValue.environmentWith({ - connectAdmin: async () => - fakeClient({ getStatus: () => Promise.resolve(gatewayStatus) }), - }), - ), - ).resolves.toBe(2); - - const emptyValue = outputCapture(); - await expect( - runCli( - ["adb", "--lease=", "devices"], - emptyValue.environmentWith({ - connectAdmin: async () => - fakeClient({ getStatus: () => Promise.resolve(gatewayStatus) }), - }), - ), - ).resolves.toBe(2); - }); - - it("fails a local passthrough with the status.get error when the daemon cannot answer", async () => { - // why: The switch reads `mode` off `status.get`, so an unreachable daemon fails there rather - // than at the passthrough -- and the CLI reports that call's own error rather than - // dressing it up as a passthrough failure. - const output = outputCapture(); - let ranLocally = false; - - await expect( - runCli( - ["adb", "devices"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => - Promise.reject( - new SimlockError( - "DAEMON_CONNECTION_LOST", - "transport", - "Daemon connection closed", - {}, - ), - ), - }), - runPassthrough: async () => { - ranLocally = true; - return 0; - }, - }), - ), - ).resolves.toBe(1); - expect(ranLocally).toBe(false); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "DAEMON_CONNECTION_LOST", message: "Daemon connection closed" }, - }); - }); - - it("uses an agent connection's own lease list without filtering it by requester", async () => { - // why: `lease.list` already answers per role: an agent sees only what it owns. Filtering that - // by `requesterId` as well would drop a lease it demonstrably owns whenever the two - // differ -- an `--agent-id` used on the lease but not on this invocation. - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "devices"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - role: "agent", - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => - Promise.resolve({ - leases: [{ ...ownLease, id: "lse_owned", requesterId: "another-agent-id" }], - }), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ args: ["devices"], leaseId: "lse_owned", stdin: "", tool: "adb" }]); - }); - - it("takes --stdin before or after --lease, and passes it to the tool after the tool's first argument", async () => { - const cases = [ - { - argv: ["adb", "--stdin", "--lease", "lse_named", "shell", "cat"], - expected: { args: ["shell", "cat"], leaseId: "lse_named", stdin: "sent" }, - }, - { - argv: ["adb", "--lease", "lse_named", "--stdin", "shell", "cat"], - expected: { args: ["shell", "cat"], leaseId: "lse_named", stdin: "sent" }, - }, - { - argv: ["adb", "shell", "--stdin"], - expected: { args: ["shell", "--stdin"], leaseId: "lse_mine", stdin: "" }, - }, - ]; - - for (const { argv, expected } of cases) { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - argv, - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.resolve({ leases: [ownLease] }), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - readStdin: async () => "sent", - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ ...expected, requesterId: "test-requester", tool: "adb" }]); - } - }); - - it("spawns locally against a worker, accepting and ignoring --stdin", async () => { - const cases = [ - { argv: ["adb", "--stdin", "shell", "cat"], args: ["shell", "cat"] }, - { argv: ["adb", "--stdin", "--lease", "lse_x", "shell", "cat"], args: ["shell", "cat"] }, - { argv: ["adb", "--lease", "lse_x", "--stdin", "shell", "cat"], args: ["shell", "cat"] }, - { argv: ["adb", "shell", "--stdin"], args: ["shell", "--stdin"] }, - ]; - - for (const { argv, args } of cases) { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - argv, - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - exec: () => Promise.reject(new Error("must not exec remotely")), - resolvePassthrough: (input) => { - seen.push(input); - return Promise.resolve({ args: [], command: "adb", env: {} }); - }, - }), - readStdin: () => Promise.reject(new Error("a worker never reads stdin")), - runPassthrough: async () => 0, - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([{ args, tool: "adb" }]); - } - }); - - it("stops scanning for --lease at the tool's own first argument", async () => { - // why: Everything from the tool's subcommand onwards is the tool's, so a `--lease` there is - // an argument to *it* -- typed text, a filter, an operand -- and travels through. - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["adb", "--lease", "lse_mine", "shell", "input", "text", "--lease"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve(gatewayStatus), - listLeases: () => Promise.reject(new Error("must not need the lease list")), - exec: (input) => { - seen.push(input); - return Promise.resolve({ exitCode: 0 }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(seen).toEqual([ - { - args: ["shell", "input", "text", "--lease"], - leaseId: "lse_mine", - requesterId: "test-requester", - stdin: "", - tool: "adb", - }, - ]); }); - }); - - it("lists both tool passthroughs in root help", async () => { - const output = outputCapture(); - - await expect(runCli([], output.environmentWith())).resolves.toBe(0); - expect(output.stdout).toContain("simctl "); - expect(output.stdout).toContain("adb "); - }); - - it("releases and exits when the watched parent process dies, via the same path as a signal", async () => { - const output = outputCapture(); - const parentWatch = new FakeParentWatch(); - const heldGrant: LeaseGrant = { - ...detachedGrant, - lease: { ...detachedGrant.lease, id: "lse_parent_death" }, - }; - const released: unknown[] = []; - - const run = runCli( + const runPromise = runCli( ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], output.environmentWith({ - connectAdmin: async () => - fakeClient({ - requestLease: (_input, _options) => Promise.resolve(heldGrant), - releaseLease: (input) => { - released.push(input); - return Promise.resolve({ leaseId: input.leaseId }); - }, - }), - parentPid: 4321, - parentWatch, - signals: new EventEmitter() as unknown as CliEnvironment["signals"], - }), - ); - - await vi.waitFor(() => expect(output.stdout).not.toBe("")); - parentWatch.exit(4321); - - await expect(run).resolves.toBe(0); - expect(released).toEqual([{ leaseId: "lse_parent_death" }]); - }); - - it("--bind-pid overrides which pid the CLI watches for parent death", async () => { - const output = outputCapture(); - const parentWatch = new FakeParentWatch(); - const heldGrant: LeaseGrant = { - ...detachedGrant, - lease: { ...detachedGrant.lease, id: "lse_bind_pid" }, - }; - const released: unknown[] = []; - - const run = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "9876"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - requestLease: (_input, _options) => Promise.resolve(heldGrant), - releaseLease: (input) => { - released.push(input); - return Promise.resolve({ leaseId: input.leaseId }); - }, - }), - parentPid: 4321, - parentWatch, - signals: new EventEmitter() as unknown as CliEnvironment["signals"], + connectAdmin: async () => client, + signals: signals as unknown as CliEnvironment["signals"], }), ); - - await vi.waitFor(() => expect(output.stdout).not.toBe("")); - // why: The captured parent is not the one being watched any more, so its death is ignored. - parentWatch.exit(4321); - expect(released).toEqual([]); - parentWatch.exit(9876); - - await expect(run).resolves.toBe(0); - expect(released).toEqual([{ leaseId: "lse_bind_pid" }]); - }); - - it("rejects a non-numeric --bind-pid as a structured USAGE error", async () => { - const output = outputCapture(); - - await expect( - runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "nope"], - output.environmentWith(), - ), - ).resolves.toBe(2); - expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "USAGE" } }); - }); - - // why: the CLI reference promises a missing required argument ends with the `simlock --help` - // pointer, and guards reach it two ways -- an explicit `withHelpHint` call or - // `requiredPositional` -- either of which can be dropped silently, so a sample of both - // paths (a command guard and the router's unknown command) is pinned here. - it("points a missing required argument at simlock --help", async () => { - const output = outputCapture(); - await expect(runCli(["lease"], output.environmentWith())).resolves.toBe(2); - expect(JSON.parse(output.stderr).error).toMatchObject({ - code: "USAGE", - message: "lease requires --platform (run `simlock --help` for usage)", - }); - }); - - it("points an unknown command at simlock --help", async () => { - const output = outputCapture(); - await expect(runCli(["nope"], output.environmentWith())).resolves.toBe(2); - expect(JSON.parse(output.stderr).error).toMatchObject({ - code: "USAGE", - message: "Unknown command: nope (run `simlock --help` for usage)", - }); + await settle(); + leaseLostListener?.({ leaseId: "some-other-lease", deviceId: "dev_2", reason: "ttl-backstop" }); + await settle(); + signals.emit("SIGINT"); + const exitCode = await runPromise; + expect(exitCode).toBe(0); + expect(released).toBe(true); + expect(output.stderr).not.toContain("lease-lost"); }); }); - -describe("CLI: admin credential resolution (ADR 0003 §5)", () => { - it.each([ - [ - "--token over env and file", - ["--token", "from-flag", "status"], - "from-env", - "from-file", - "from-flag", - true, - ], - [ - "SIMLOCK_ADMIN_TOKEN over the file when --token is absent", - ["status"], - "from-env", - "from-file", - "from-env", - false, - ], - [ - "the admin.token file when neither --token nor the env var is set", - ["status"], - undefined, - "from-file", - "from-file", - true, - ], - ] as const)( - "resolves the admin credential in priority order: %s", - async (_case, argv, envToken, fileToken, expected, stderrEmpty) => { - const output = outputCapture(); - let seen: string | undefined; - await runCli( - [...argv], - output.environmentWith({ - ...(envToken === undefined ? {} : { adminTokenFromEnv: envToken }), - readAdminTokenFile: async () => fileToken, - connectAdmin: async (resolveCredential) => { - seen = await resolveCredential(); - return fakeClient(); - }, - }), - ); - expect(seen).toBe(expected); - if (stderrEmpty) expect(output.stderr).toBe(""); - }, - ); - - it("retries a briefly-missing admin.token file before giving up", async () => { - const output = outputCapture(); - let reads = 0; - let seen: string | undefined; - await runCli( - ["status"], - output.environmentWith({ - readAdminTokenFile: async () => { - reads += 1; - return reads < 3 ? undefined : "from-file-after-retry"; - }, - connectAdmin: async (resolveCredential) => { - seen = await resolveCredential(); - return fakeClient(); - }, - }), - ); - expect(reads).toBe(3); - expect(seen).toBe("from-file-after-retry"); - expect(output.stderr).toBe(""); - }); - - it("connects with no credential and writes a stderr notice when every source is empty", async () => { - const output = outputCapture(); - let seen: string | undefined | "unset" = "unset"; - await runCli( - ["status"], - output.environmentWith({ - readAdminTokenFile: async () => undefined, - connectAdmin: async (resolveCredential) => { - seen = await resolveCredential(); - return fakeClient(); - }, - }), - ); - expect(seen).toBeUndefined(); - expect(output.stderr.trim().split("\n")).toHaveLength(1); - const notice: unknown = JSON.parse(output.stderr); - expect((notice as { notice: string }).notice).toMatch(/agent/i); - }); - - it("does not read the admin.token file at all when --token is given (B2 ordering)", async () => { - // why: The file source must never be consulted -- let alone before the connection exists -- when - // a higher-priority source already answered. Also guards against a regression back to - // resolving credentials before `connectAdmin` is even called. - const output = outputCapture(); - let fileReads = 0; - await runCli( - ["--token", "from-flag", "status"], - output.environmentWith({ - readAdminTokenFile: async () => { - fileReads += 1; - return "from-file"; - }, - connectAdmin: async (resolveCredential) => { - await resolveCredential(); - return fakeClient(); - }, - }), - ); - expect(fileReads).toBe(0); - }); - - it("B1: degrades to an agent session with a stderr notice when the daemon rejects the credential", async () => { - const output = outputCapture(); - let attempt = 0; - let seenOnRetry: string | undefined | "unset" = "unset"; - await runCli( - ["status"], - output.environmentWith({ - readAdminTokenFile: async () => "stale-secret", - connectAdmin: async (resolveCredential) => { - attempt += 1; - if (attempt === 1) { - const credential = await resolveCredential(); - expect(credential).toBe("stale-secret"); - throw simlockError("ADMIN_AUTHENTICATION_FAILED"); - } - seenOnRetry = await resolveCredential(); - return fakeClient(); - }, - }), - ); - expect(attempt).toBe(2); - expect(seenOnRetry).toBeUndefined(); - expect(output.stderr.trim().split("\n")).toHaveLength(1); - const notice = JSON.parse(output.stderr) as { notice: string }; - expect(notice.notice).toMatch(/admin\.token/i); - expect(notice.notice).toMatch(/agent/i); - }); - - it("B1: does not retry, and propagates the error, when the connection already had no credential", async () => { - const output = outputCapture(); - let attempts = 0; - const exitCode = await runCli( - ["status"], - output.environmentWith({ - readAdminTokenFile: async () => undefined, - connectAdmin: async (resolveCredential) => { - attempts += 1; - await resolveCredential(); - throw simlockError("ADMIN_AUTHENTICATION_FAILED"); - }, - }), - ); - expect(attempts).toBe(1); - expect(exitCode).toBe(1); - expect(JSON.parse(output.stderr).error.code).toBe("ADMIN_AUTHENTICATION_FAILED"); - }); - - it("B1: an explicit but wrong --token also degrades to agent, with a generic (not file-specific) notice", async () => { - const output = outputCapture(); - let attempt = 0; - await runCli( - ["--token", "wrong", "status"], - output.environmentWith({ - connectAdmin: async (resolveCredential) => { - attempt += 1; - const credential = await resolveCredential(); - if (attempt === 1) { - expect(credential).toBe("wrong"); - throw simlockError("ADMIN_AUTHENTICATION_FAILED"); - } - expect(credential).toBeUndefined(); - return fakeClient(); - }, - }), - ); - expect(attempt).toBe(2); - const notice = JSON.parse(output.stderr) as { notice: string }; - expect(notice.notice).not.toMatch(/admin\.token/i); - expect(notice.notice).toMatch(/agent/i); - }); - - it("B2: reaches admin on a cold start, on the very first invocation, no daemon running yet", async () => { - const filesystem = new MemoryFilesystem(); - const ipcTransport = new MemoryIpcTransport(); - const socketPath = "/simlock/daemon.sock"; - const adminTokenPath = "/simlock/admin.token"; - const launcher = new FakeDaemonLauncher(async () => { - await startInMemoryDaemon({ adminTokenPath, filesystem, ipcTransport, socketPath }); - }); - const output = outputCapture({ - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - ipc: ipcTransport, - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - const exitCode = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith(), - ); - expect(exitCode).toBe(0); - expect(launcher.launches).toBe(1); - const grant = JSON.parse(output.stdout) as LeaseGrant & { role: string }; - expect(grant.role).toBe("admin"); - // why: Provisioning progress pushes are expected noise on stderr; the agent-fallback notice is - // the thing that must be absent. - expect(output.stderr).not.toContain("notice"); - }); - - it("daemon start reports the config error and launches nothing when config.json is invalid", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic( - "/simlock/config.json", - JSON.stringify({ capacity: { config: { limits: { android: { maxDevices: 0 } } } } }), - ); - const launcher = new FakeDaemonLauncher(); - const output = outputCapture({ - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - ipc: new MemoryIpcTransport(), - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - const exitCode = await runCli(["daemon", "start"], output.environmentWith()); - expect(exitCode).toBe(2); - expect(launcher.launches).toBe(0); - expect(output.stderr).toContain("limits.android.maxDevices"); - expect(output.stderr).not.toContain("Timed out"); - }); - - it("B1: a stale admin.token degrades to an agent session instead of bricking the invocation", async () => { - const filesystem = new MemoryFilesystem(); - const ipcTransport = new MemoryIpcTransport(); - const socketPath = "/simlock/daemon.sock"; - const adminTokenPath = "/simlock/admin.token"; - // why: A daemon actually runs and persists its own secret first, then the file is overwritten - // with a value that hashes to nothing the running daemon recognizes -- exactly what a - // `kill -9`'d daemon's leftover `admin.token` looks like to the *next* daemon incarnation, - // which never touches a file it did not itself just persist. - await startInMemoryDaemon({ adminTokenPath, filesystem, ipcTransport, socketPath }); - await filesystem.writeFileAtomic(adminTokenPath, "stale-secret-from-a-killed-daemon\n"); - const launcher = new FakeDaemonLauncher(); - const output = outputCapture({ - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - ipc: ipcTransport, - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - const exitCode = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith(), - ); - expect(exitCode).toBe(0); - expect(launcher.launches).toBe(0); - const grant = JSON.parse(output.stdout) as LeaseGrant & { role: string }; - expect(grant.role).toBe("agent"); - expect(output.stderr).toContain("admin.token"); - }); -}); - -describe("CLI: auto-launch's startup deadline (#663)", () => { - /** - * why: the launcher hands back a daemon that claims its socket only `bootMs` later, the way - * `NodeDaemonLauncher` returns while the detached process it spawned is still booting; what - * the CLI can see is how long the socket stays absent, not how long `launch` itself blocks. - */ - function slowBootPorts(bootMs: number): { - readonly ports: CliEnvironmentPorts; - readonly clock: FakeClock; - } { - const filesystem = new MemoryFilesystem(); - const clock = new FakeClock(0); - const ipcTransport = new MemoryIpcTransport(); - const ports: CliEnvironmentPorts = { - filesystem, - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: ipcTransport, - launcher: new FakeDaemonLauncher(() => { - clock.setTimer(bootMs, () => { - void startInMemoryDaemon({ - adminTokenPath: "/simlock/admin.token", - filesystem, - ipcTransport, - socketPath: "/simlock/daemon.sock", - }); - }); - }), - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }; - return { clock, ports }; - } - - /** - * why: turns the clock at the auto-launch poll's own 50 ms step, with room to run past the - * startup deadline and keep pumping the boot the CLI abandoned; the clock is fake, so the - * extra ticks cost nothing. The `settled` guard makes a hang fail on a named assertion - * instead of on a test timeout. - */ - async function runWithTicks( - command: readonly string[], - ports: CliEnvironmentPorts, - clock: FakeClock, - runOnMs: number, - ): Promise<{ readonly exitCode: number; readonly output: ReturnType }> { - const output = outputCapture(ports); - let settled = false; - const exitCode = runCli(command, output.environmentWith()).then((code) => { - settled = true; - return code; - }); - for (let tick = 0; tick < 500 && (!settled || clock.now() < runOnMs); tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect( - settled, - `simlock ${command.join(" ")} still had not settled at ${String(clock.now())}ms`, - ).toBe(true); - return { exitCode: await exitCode, output }; - } - - it("waits 20 s for a daemon it auto-launched: within the deadline it reports as running, past it the command reports a failed start", async () => { - const cases = [ - { bootMs: 19_900, label: "19.9 s" }, - { bootMs: 20_100, label: "20.1 s" }, - ]; - - // why: every case runs before anything is asserted, so the case that passes cannot hide - // what the case that fails did to the daemon it launched. - const outcomes = await Promise.all( - cases.map(async (testCase) => { - const { clock, ports } = slowBootPorts(testCase.bootMs); - const { exitCode, output } = await runWithTicks( - ["daemon", "start", "--json"], - ports, - clock, - testCase.bootMs + 1_000, - ); - const after = outputCapture(ports); - const statusExit = await runCli(["daemon", "status", "--json"], after.environmentWith()); - const status = JSON.parse(after.stdout || "{}") as { daemon?: { health?: string } }; - return { - exitCode, - label: testCase.label, - timedOutStarting: output.stderr.includes("Timed out starting simlock daemon"), - stdout: output.stdout, - statusHealth: status.daemon?.health ?? after.stdout, - statusExit, - }; - }), - ); - - expect( - outcomes.map((outcome) => ({ - exitCode: outcome.exitCode, - label: outcome.label, - // why: the failed start is asserted as the timeout's wording, not the whole line or - // its error code, so what the code reports can change without moving this test. - timedOutStarting: outcome.timedOutStarting, - stdout: outcome.stdout, - })), - ).toEqual([ - { - exitCode: 0, - label: "19.9 s", - timedOutStarting: false, - stdout: '{"status":"running"}\n', - }, - { - exitCode: 1, - label: "20.1 s", - timedOutStarting: true, - stdout: "", - }, - ]); - - // why: asserted apart from the exit code so a fix that only softens the timeout message - // fails here -- `daemon status` never launches one, so what it reaches is the daemon this - // command launched and then gave up on. - for (const outcome of outcomes) { - expect(outcome.statusExit, `${outcome.label}: status stderr`).toBe(0); - expect( - ["starting", "running"].includes(outcome.statusHealth as string), - `${outcome.label}: status says ${JSON.stringify(outcome.statusHealth)}`, - ).toBe(true); - } - }); -}); - -describe("CLI: auto-launch's failed start (#672)", () => { - it("reports a daemon it launched but never reached as DAEMON_STARTUP_FAILED, exit 1", async () => { - // why: nothing ever claims the socket, which is what a daemon still booting past the - // deadline looks like to the CLI. The run goes well past any deadline the loop could hold, - // so the code asserted here is the one the CLI chose for giving up, never one a clock reached. - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher(); - const output = outputCapture({ - filesystem: new MemoryFilesystem(), - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: new MemoryIpcTransport(), - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli(["daemon", "start", "--json"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 700 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(settled, `simlock daemon start still had not settled at ${String(clock.now())}ms`).toBe( - true, - ); - expect(await exitCode).toBe(1); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: expect.stringContaining("Timed out starting simlock daemon"), - }, - }); - expect(launcher.launches).toBe(1); - }); -}); - -describe("CLI: auto-launch's daemon that exits while starting (#760)", () => { - it("status fails at once with the daemon's start error when the launched daemon exits with a fatal line", async () => { - const clock = new FakeClock(0); - const fatal = JSON.stringify({ - message: "Daemon failed to start", - fields: { message: "Found /simlock/state.json, remove it" }, - }); - const launcher = new FakeDaemonLauncher(undefined, { - exit: { code: 1, signal: null }, - output: `older line\n${fatal}\n`, - }); - const output = outputCapture({ - filesystem: new MemoryFilesystem(), - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: new MemoryIpcTransport(), - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli(["status"], output.environmentWith()).then((code) => { - settled = true; - return code; - }); - for (let tick = 0; tick < 100 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(settled, "status had not settled by 5 s").toBe(true); - expect(clock.now()).toBeLessThan(20_000); - expect(await exitCode).toBe(1); - expect(JSON.parse(output.stderr)).toEqual({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: "The daemon could not start: Found /simlock/state.json, remove it", - }, - }); - }); - - it("status names the exit code when the launched daemon's output cannot be read", async () => { - const clock = new FakeClock(0); - const launcher = { - launch: async () => ({ - exited: Promise.resolve({ code: 3, signal: null }), - readOutput: async (): Promise => { - throw new Error("daemon.log was replaced"); - }, - }), - }; - const output = outputCapture({ - filesystem: new MemoryFilesystem(), - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: new MemoryIpcTransport(), - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli(["status"], output.environmentWith()).then((code) => { - settled = true; - return code; - }); - for (let tick = 0; tick < 100 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(settled, "status had not settled by 5 s").toBe(true); - expect(await exitCode).toBe(1); - expect(JSON.parse(output.stderr).error.message).toBe( - "The daemon exited with code 3 while starting; see `simlock daemon logs`", - ); - }); -}); - -describe("CLI: auto-launch's daemon that lost the socket to another daemon (#760)", () => { - it("status goes on with the daemon that holds the socket when the launched daemon exits already running", async () => { - const filesystem = new MemoryFilesystem(); - const clock = new FakeClock(0); - // why: the winner is not listening yet at the first poll after the launch, so the loop reaches - // the launched daemon's exit before it finds the winner. - class LateWinnerTransport extends MemoryIpcTransport { - connects = 0; - override async connect(endpoint: string): ReturnType { - this.connects += 1; - if (this.connects === 2) { - throw new IpcError("endpoint-not-found", `No IPC endpoint at ${endpoint}`, undefined); - } - return super.connect(endpoint); - } - } - const ipcTransport = new LateWinnerTransport(); - const launcher = new FakeDaemonLauncher( - async () => { - await startInMemoryDaemon({ - adminTokenPath: "/simlock/admin.token", - filesystem, - ipcTransport, - socketPath: "/simlock/daemon.sock", - }); - }, - { - exit: { code: 1, signal: null }, - output: `${JSON.stringify({ - message: "Daemon failed to start", - fields: { message: "Simlock daemon is already running at /simlock/daemon.sock" }, - })}\n`, - }, - ); - const output = outputCapture({ - filesystem, - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: ipcTransport, - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli(["status"], output.environmentWith()).then((code) => { - settled = true; - return code; - }); - for (let tick = 0; tick < 100 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(settled, "status had not settled by 5 s").toBe(true); - expect(await exitCode).toBe(0); - expect(output.stderr).not.toContain("DAEMON_STARTUP_FAILED"); - expect(ipcTransport.connects).toBeGreaterThanOrEqual(3); - }); -}); - -describe("CLI: auto-launch's handshake cut short by the launched daemon (#684)", () => { - it.each([ - { closeAfter: "hello", rejectedCredential: false }, - { closeAfter: "request", rejectedCredential: false }, - { closeAfter: "hello", rejectedCredential: true }, - { closeAfter: "request", rejectedCredential: true }, - ] as const)( - "reports a daemon it launched that answers hello then dies during that start as a failed start, not a lost connection (after $closeAfter, credential rejected: $rejectedCredential)", - async ({ closeAfter, rejectedCredential }) => { - // why: this stages the daemon that loses its HTTP port race exactly as the socket shows it - // to the CLI: absent before the launch, then listening, answering `hello`, and closing on - // its own `http-start-failed` stop before the command's first real request is answered. - const dyingDaemon = new HelloThenDeathIpc(closeAfter, rejectedCredential); - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher(() => { - dyingDaemon.startListening(); - }); - const output = outputCapture({ - filesystem: new MemoryFilesystem(), - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: dyingDaemon, - launcher, - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli( - ["daemon", "start", "--json", "--token", "probe"], - output.environmentWith(), - ).then((code) => { - settled = true; - return code; - }); - // why: the tick bound is generous because #671 made every module a command loads lazy, so - // one `daemon start` now passes through several more import turns than when this repro was - // first staged; the clock is fake and the loop stops the moment the command settles, so the - // extra headroom pins nothing -- it only keeps a hung command failing on this named - // assertion instead of on a test timeout. - for (let tick = 0; tick < 100 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect( - settled, - `simlock daemon start still had not settled at ${String(clock.now())}ms`, - ).toBe(true); - - // why: the staging itself, proven from the fake's side -- the socket was absent until this - // command launched the daemon and answered `hello` before it closed the connection. Without - // these the assertion below could be - // reached by a connection that never saw a daemon at all. - expect(launcher.launches).toBe(1); - expect(dyingDaemon.helloReplies).toBe(1); - expect(dyingDaemon.deaths).toBe(1); - expect(dyingDaemon.rejectedCredentials).toBe(rejectedCredential ? 1 : 0); - - expect(await exitCode).toBe(1); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toMatchObject({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: expect.stringContaining("daemon"), - }, - }); - }, - ); - - it.each([false, true])( - "keeps a pre-existing daemon's hello-then-close as DAEMON_CONNECTION_LOST (credential rejected: %s)", - async (rejectedCredential) => { - const dyingDaemon = new HelloThenDeathIpc("hello", rejectedCredential); - dyingDaemon.startListening(); - const launcher = new FakeDaemonLauncher(); - const output = outputCapture({ - ...realCliEnvironmentPorts(), - ipc: dyingDaemon, - launcher, - }); - - expect( - await runCli(["daemon", "start", "--json", "--token", "probe"], output.environmentWith()), - ).toBe(1); - expect(launcher.launches).toBe(0); - expect(dyingDaemon.helloReplies).toBe(1); - expect(dyingDaemon.deaths).toBe(1); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toEqual({ - error: { code: "DAEMON_CONNECTION_LOST", message: "Daemon connection closed" }, - }); - }, - ); - - it("reports a daemon that answers the startup status probe with DAEMON_STOPPING as its own start error and closes its client", async () => { - const daemon = new HelloThenDeathIpc("stopping"); - const launcher = new FakeDaemonLauncher(() => daemon.startListening(), { - exit: { code: 1, signal: null }, - output: `${JSON.stringify({ message: "Daemon failed to start", fields: { message: "listen EADDRINUSE: address already in use :::7777" } })}\n`, - }); - const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, launcher }); - expect(await runCli(["daemon", "start", "--token", "probe"], output.environmentWith())).toBe(1); - expect(launcher.launches).toBe(1); - expect(daemon.helloReplies).toBe(1); - expect(daemon.clientCloses).toBe(1); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: "The daemon could not start: listen EADDRINUSE: address already in use :::7777", - }, - }); - }); - - it("daemon start's wait for health to leave starting ends past the daemon's startup-read cap with DAEMON_STARTUP_FAILED, leaving the daemon running", async () => { - const daemon = new HelloThenDeathIpc("starting"); - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher(() => daemon.startListening()); - const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, clock, launcher }); - let settled = false; - const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 3_000 && !settled; tick += 1) { - clock.advance(50); - await settle(); - } - expect(settled, `daemon start still waiting at ${String(clock.now())}ms`).toBe(true); - expect( - clock.now(), - "the wait outlasts the daemon's own startup read by 30 s", - ).toBeGreaterThanOrEqual(STARTUP_READ_LIMIT_MS + 30_000); - expect(await run).toBe(1); - expect(daemon.frameTypes, "the CLI sends the daemon no stop").not.toContain("daemon.stop"); - expect(daemon.deaths, "the daemon is not stopped by the CLI").toBe(0); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: expect.stringContaining("still starting"), - }, - }); - }); - - it("bounds the startup status probe by the original launch deadline and closes its client", async () => { - const daemon = new HelloThenDeathIpc("silent"); - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher(() => { - clock.setTimer(19_000, () => daemon.startListening()); - }); - const output = outputCapture({ ...realCliEnvironmentPorts(), ipc: daemon, clock, launcher }); - let settled = false; - const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 500 && !settled; tick += 1) { - clock.advance(50); - await settle(); - } - expect(settled, "silent startup probe must settle within the launch budget").toBe(true); - expect( - clock.now(), - "probe must use the budget left after the socket appeared", - ).toBeLessThanOrEqual(20_050); - expect(clock.now(), "probe must wait out the launch deadline").toBeGreaterThanOrEqual(20_000); - expect(await run).toBe(1); - expect(launcher.launches).toBe(1); - expect(daemon.helloReplies).toBe(1); - expect(daemon.clientCloses).toBe(1); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { - code: "DAEMON_STARTUP_FAILED", - message: expect.stringContaining("Timed out starting"), - }, - }); - }); - - it("fails a start with DAEMON_STARTUP_FAILED when the daemon stops reading after hello, without waiting for its side of the close", async () => { - const directory = await mkdtemp(join(tmpdir(), "simlock-wedged-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - const sockets: Socket[] = []; - let hellos = 0; - const server = createServer((socket) => { - sockets.push(socket); - let buffered = ""; - socket.on("data", (chunk) => { - buffered += chunk.toString(); - for (const line of buffered.split("\n").slice(0, -1)) { - const frame = JSON.parse(line) as { id: number; type: string }; - if (frame.type !== "hello") continue; - hellos += 1; - socket.write( - `${JSON.stringify({ - id: frame.id, - ok: true, - payload: { - protocolVersion: PROTOCOL_VERSION_RANGE.max, - daemonProtocolRange: PROTOCOL_VERSION_RANGE, - version: "0.0.0-test", - role: "admin", - principal: "test-requester", - }, - })}\n`, - ); - // why: a daemon wedged after `hello` neither answers nor ends its side of the socket. - socket.pause(); - } - buffered = buffered.slice(buffered.lastIndexOf("\n") + 1); - }); - }); - await new Promise((resolve) => server.listen(socketPath, resolve)); - try { - const real = new NodeIpcTransport(); - let launched = false; - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher(() => { - launched = true; - }); - let probeSent = false; - const ipc: IpcConnector = { - connect: async () => { - if (!launched) throw new IpcError("endpoint-not-found", "not yet", undefined); - const connection = await real.connect(socketPath); - const write = connection.write.bind(connection); - connection.write = async (contents) => { - await write(contents); - if ((JSON.parse(contents.trim()) as { type: string }).type === "status.get") { - probeSent = true; - } - }; - return connection; - }, - }; - const output = outputCapture({ ...realCliEnvironmentPorts(), ipc, clock, launcher }); - let settled = false; - const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 200 && hellos === 0; tick += 1) { - clock.advance(50); - await new Promise((resolve) => setTimeout(resolve, 5)); - } - expect(hellos).toBe(1); - // why: the server observing hello does not mean the client has read its reply and armed - // the probe deadline; advance the fake clock only after the client sends that probe. - await expect.poll(() => probeSent).toBe(true); - clock.advance(20_000); - await expect.poll(() => settled, { timeout: 2_000 }).toBe(true); - expect(await run).toBe(1); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { code: "DAEMON_STARTUP_FAILED" }, - }); - } finally { - for (const socket of sockets) socket.destroy(); - await new Promise((resolve) => server.close(() => resolve())); - } - }); - - it("exits 1 for a socket closed before hello and reports DAEMON_STARTUP_FAILED only after launch", async () => { - for (const scenario of [ - { launch: false, afterExit: false, code: "DAEMON_CONNECTION_LOST", connects: 1 }, - { launch: true, afterExit: false, code: "DAEMON_STARTUP_FAILED", connects: 2 }, - { launch: true, afterExit: true, code: "DAEMON_STARTUP_FAILED", connects: 3 }, - ]) { - const directory = await mkdtemp(join(tmpdir(), "simlock-closed-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - let accepted = 0; - // why: the daemon that refuses its history accepts the CLI's connection, then stops and - // closes it; the CLI is still reading `admin.token` then, so the close lands before `hello`. - const server = createServer((socket) => { - accepted += 1; - socket.destroy(); - }); - await new Promise((resolve) => server.listen(socketPath, resolve)); - try { - const real = new NodeIpcTransport(); - let launched = !scenario.launch; - const clock = new FakeClock(0); - const launcher = new FakeDaemonLauncher( - () => { - launched = true; - }, - { exit: { code: 1, signal: null } }, - ); - let connects = 0; - const ipc: IpcConnector = { - connect: async () => { - connects += 1; - if (!launched || (scenario.afterExit && connects === 2)) { - throw new IpcError("endpoint-not-found", "not yet", undefined); - } - const connection = await real.connect(socketPath); - await expect.poll(() => connection.closed).toBe(true); - return connection; - }, - }; - const output = outputCapture({ ...realCliEnvironmentPorts(), ipc, clock, launcher }); - let settled = false; - const run = runCli(["daemon", "start", "--token", "probe"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 100 && !settled; tick += 1) { - clock.advance(50); - await new Promise((resolve) => setTimeout(resolve, 5)); - } - expect(launcher.launches).toBe(scenario.launch ? 1 : 0); - expect(accepted).toBe(1); - expect(connects).toBe(scenario.connects); - expect( - settled, - `simlock daemon start still had not settled at ${String(clock.now())}ms`, - ).toBe(true); - expect(await run).toBe(1); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1)!)).toMatchObject({ - error: { code: scenario.code }, - }); - } finally { - await new Promise((resolve) => server.close(() => resolve())); - } - } - }); - - it.each([ - { operation: "pending lease", health: "running" }, - { operation: "pending lease", health: "starting" }, - { operation: "held lease", health: "running" }, - { operation: "in-flight request", health: "running" }, - ] as const)( - "keeps an auto-launched daemon's death after its startup status probe as DAEMON_CONNECTION_LOST during $operation (status: $health)", - async ({ operation, health }) => { - const ipc = new MemoryIpcTransport(); - const clock = new FakeClock(0); - const requests: string[] = []; - const replies: string[] = []; - const connections: IpcConnection[] = []; - const launcher = new FakeDaemonLauncher(async () => { - await ipc.listen("/simlock/daemon.sock", (connection) => { - connections.push(connection); - connection.onData((contents) => { - const frame = JSON.parse(contents) as { id: number; type: string }; - requests.push(frame.type); - const payload = - frame.type === "hello" - ? { - protocolVersion: PROTOCOL_VERSION_RANGE.max, - daemonProtocolRange: PROTOCOL_VERSION_RANGE, - version: "0.0.0-test", - role: "admin", - principal: "test-requester", - } - : frame.type === "lease.request" && operation !== "pending lease" - ? detachedGrant - : frame.type === "status.get" - ? { ...EMPTY_STATUS, daemon: { ...EMPTY_STATUS.daemon, health } } - : undefined; - if (payload === undefined) return; - replies.push(frame.type); - void connection.write(`${JSON.stringify({ id: frame.id, ok: true, payload })}\n`); - }); - }); - }); - const output = outputCapture({ - ...realCliEnvironmentPorts(), - ipc, - clock, - launcher, - }); - let settled = false; - const runPromise = runCli( - operation !== "in-flight request" - ? ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--token", "probe"] - : ["simctl", "list", "--token", "probe"], - output.environmentWith(), - ).then((code) => { - settled = true; - return code; - }); - const ready = (): boolean => - operation === "held lease" - ? output.stdout !== "" - : requests.includes( - operation === "pending lease" ? "lease.request" : "driver.passthrough", - ); - for (let tick = 0; tick < 100 && !ready() && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(ready()).toBe(true); - expect(settled).toBe(false); - expect(launcher.launches).toBe(1); - expect(connections).toHaveLength(1); - expect(requests).toContain( - operation === "in-flight request" ? "driver.passthrough" : "lease.request", - ); - if (operation === "pending lease") expect(replies).not.toContain("lease.request"); - await connections[0]!.close(); - expect(await runPromise).toBe(1); - expect(JSON.parse(output.stderr)).toMatchObject({ - error: { code: "DAEMON_CONNECTION_LOST" }, - }); - expect(replies).toContain("status.get"); - if (operation === "held lease") { - expect(JSON.parse(output.stdout)).toMatchObject({ lease: detachedGrant.lease }); - expect(JSON.parse(output.stderr).error.message).toContain(detachedGrant.lease.id); - expect(requests).toContain("lease.request"); - } else { - expect(output.stdout).toBe(""); - } - }, - ); - - it("preserves the daemon's refusal of the first auto-launched request", async () => { - const filesystem = new MemoryFilesystem(); - const ipc = new MemoryIpcTransport(); - const launcher = new FakeDaemonLauncher(async () => { - await startInMemoryDaemon({ - filesystem, - ipcTransport: ipc, - socketPath: "/simlock/daemon.sock", - adminTokenPath: "/simlock/admin.token", - }); - }); - const output = outputCapture({ - ...realCliEnvironmentPorts(), - filesystem, - ipc, - launcher, - }); - - expect( - await runCli(["lease", "--platform", "android", "--detach"], output.environmentWith()), - ).toBe(12); - expect(launcher.launches).toBe(1); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "NO_DRIVER" } }); - }); -}); - -async function runDaemonStatus( - testCase: { - readonly role: "worker" | "gateway"; - readonly running: boolean; - readonly state: ServiceState | undefined; - }, - json: boolean, -): Promise<{ readonly stdout: string }> { - const home = "/Users/me"; - const output = outputCapture(); - const manager = - testCase.state === undefined ? undefined : new FakeServiceManager({ state: testCase.state }); - const environment = output.environmentWith({ - daemonHome: { role: testCase.role, dataDirectory: "/simlock", gatewayPresent: false }, - connectExistingAdmin: async () => { - if (!testCase.running) throw new Error("connect ECONNREFUSED"); - return fakeClient(); - }, - service: { - manager, - platform: manager === undefined ? "linux" : "darwin", - user: () => "me", - definition: async () => - serviceDefinition({ - role: testCase.role, - root: `${home}/.simlock`, - dataDirectory: "/simlock", - home, - nodePath: "/Users/me/.nvm/versions/node/v22.13.0/bin/node", - mainPath: "/Users/me/simlock/dist/daemon/main.js", - env: {}, - }), - }, - }); - await expect( - runCli(["daemon", "status", ...(json ? ["--json"] : [])], environment), - ).resolves.toBe(0); - return output; -} - -describe("CLI: daemon status (ADR 0003 §11)", () => { - it.each([ - { name: "the socket cannot be reached at all", error: () => new Error("connect ECONNREFUSED") }, - { - name: "a transport-kind SimlockError (connection lost)", - error: () => simlockError("DAEMON_CONNECTION_LOST"), - }, - ])("reports stopped when $name", async ({ error }) => { - const output = outputCapture(); - await expect( - runCli( - ["daemon", "status", "--json"], - output.environmentWith({ - connectExistingAdmin: async () => { - throw error(); - }, - }), - ), - ).resolves.toBe(0); - expect(JSON.parse(output.stdout)).toEqual({ status: "stopped" }); - }); - - it("distinguishes a refused handshake (socket reachable) from an absent socket", async () => { - const output = outputCapture(); - await expect( - runCli( - ["daemon", "status", "--json"], - output.environmentWith({ - connectExistingAdmin: async () => { - throw simlockError("ADMIN_AUTHENTICATION_FAILED"); - }, - }), - ), - ).resolves.toBe(1); - const parsed = JSON.parse(output.stdout) as { status: string; error: { code: string } }; - expect(parsed.status).toBe("handshake-refused"); - expect(parsed.error.code).toBe("ADMIN_AUTHENTICATION_FAILED"); - }); - - it("never auto-launches the daemon for daemon status", async () => { - const output = outputCapture(); - let launched = false; - await runCli( - ["daemon", "status"], - output.environmentWith({ - connectAdmin: async () => { - launched = true; - return fakeClient(); - }, - connectExistingAdmin: async () => fakeClient(), - }), - ); - expect(launched).toBe(false); - }); - - it("daemon status prints the Service: line and the service field in --json, for a worker or a gateway, running or stopped, and none on an unsupported platform", async () => { - const home = "/Users/me"; - const node = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; - const main = "/Users/me/simlock/dist/daemon/main.js"; - const worker = "com.callstack.simlock.worker"; - const gateway = "com.callstack.simlock.gateway"; - const plist = (name: string) => `${home}/Library/LaunchAgents/${name}.plist`; - const cases: readonly { - readonly label: string; - readonly role: "worker" | "gateway"; - readonly running: boolean; - readonly state: ServiceState | undefined; - readonly line: string | undefined; - readonly service: Record | undefined; - }[] = [ - { - label: "not installed, stopped", - role: "worker", - running: false, - state: { state: "not-installed" }, - line: "Service: not installed", - service: { state: "not-installed", manager: "launchd", name: worker }, - }, - { - label: "installed, running", - role: "worker", - running: true, - state: { state: "installed", path: plist(worker) }, - line: `Service: installed (launchd, ${worker})`, - service: { state: "installed", manager: "launchd", name: worker, path: plist(worker) }, - }, - { - label: "not loaded, running", - role: "worker", - running: true, - state: { state: "not-loaded", path: plist(worker) }, - line: `Service: not loaded (launchd, ${worker}): commands start the daemon without it`, - service: { state: "not-loaded", manager: "launchd", name: worker, path: plist(worker) }, - }, - { - label: "Node path gone, stopped", - role: "worker", - running: false, - state: { state: "broken", path: plist(worker), missing: [node] }, - line: `Service: broken (launchd, ${worker}): ${node} is gone; run simlock daemon install`, - service: { - state: "broken", - manager: "launchd", - name: worker, - path: plist(worker), - missing: [node], - }, - }, - { - label: "both paths gone, stopped", - role: "worker", - running: false, - state: { state: "broken", path: plist(worker), missing: [node, main] }, - line: `Service: broken (launchd, ${worker}): ${node}, ${main} are gone; run simlock daemon install`, - service: { - state: "broken", - manager: "launchd", - name: worker, - path: plist(worker), - missing: [node, main], - }, - }, - { - label: "gateway, running", - role: "gateway", - running: true, - state: { state: "installed", path: plist(gateway) }, - line: `Service: installed (launchd, ${gateway})`, - service: { state: "installed", manager: "launchd", name: gateway, path: plist(gateway) }, - }, - { - label: "unsupported platform", - role: "worker", - running: true, - state: undefined, - line: undefined, - service: undefined, - }, - ]; - for (const testCase of cases) { - const human = await runDaemonStatus(testCase, false); - const lines = human.stdout.trimEnd().split("\n"); - expect( - [lines[0], ...lines.filter((line) => line.startsWith("Service:"))], - testCase.label, - ).toEqual([ - testCase.running ? "Daemon: running (worker)" : "Daemon stopped", - ...(testCase.line === undefined ? [] : [testCase.line]), - ]); - if (testCase.line !== undefined) - expect(lines.at(-1), `${testCase.label} last line`).toBe(testCase.line); - const json = JSON.parse((await runDaemonStatus(testCase, true)).stdout) as { - readonly service?: unknown; - }; - expect(json.service, `${testCase.label} --json`).toEqual(testCase.service); - } - }); - - it("daemon status fails with the service manager's error instead of printing Daemon stopped", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - connectExistingAdmin: async () => { - throw new Error("connect ECONNREFUSED"); - }, - service: { - manager: { - manager: "launchd", - state: async () => { - throw new Error("launchctl timed out"); - }, - } as unknown as ServiceManager, - platform: "darwin", - user: () => "me", - definition: async () => - serviceDefinition({ - role: "worker", - root: "/Users/me/.simlock", - dataDirectory: "/simlock", - home: "/Users/me", - nodePath: "/n", - mainPath: "/m.js", - env: {}, - }), - }, - }); - - await expect(runCli(["daemon", "status"], environment)).resolves.toBe(1); - - expect(output.stdout).not.toContain("Daemon stopped"); - expect(output.stderr).toContain("launchctl timed out"); - }); - - it("asks the daemon to purge orphans only once the operator has confirmed", async () => { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["doctor", "--purge-orphans"], - output.environmentWith({ - confirm: async () => true, - connectAdmin: async () => - fakeClient({ - runDoctor: (input) => { - seen.push(input); - return Promise.resolve({ findings: [], sources: [] }); - }, - }), - }), - ), - ).resolves.toBe(0); - - expect(seen).toEqual([{ fix: false, purgeOrphans: true }]); - }); - - it("prints one stderr line per prerequisite finding, keeps the JSON report on stdout, and exits 0", async () => { - const output = outputCapture(); - const findings = [ - { - kind: "prerequisite-missing" as const, - message: "The Android emulator package is not installed.", - platform: "android" as const, - prerequisite: "android-emulator", - remedy: "Run `sdkmanager --install emulator`.", - }, - { - kind: "prerequisite-missing" as const, - message: "Xcode is not installed.", - platform: "ios" as const, - prerequisite: "xcode", - remedy: "Run `xcode-select`.", - }, - { deviceId: "d-1", kind: "expired-live-lease" as const, leaseId: "l-1" }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), - }), - ), - ).resolves.toBe(0); - - // why: The harness has no admin credential, so a connection notice precedes these lines. - expect(output.stderr.split("\n").filter((line) => line.startsWith("Missing"))).toEqual([ - "Missing [android] android-emulator: The Android emulator package is not installed. " + - "Run `sdkmanager --install emulator`.", - "Missing [ios] xcode: Xcode is not installed. Run `xcode-select`.", - ]); - expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); - }); - - it("simlock doctor prints a service-broken finding to stderr, naming each missing path, and keeps the finding in the JSON report", async () => { - const node = "/Users/me/.nvm/versions/node/v22.13.0/bin/node"; - const main = "/Users/me/simlock/dist/daemon/main.js"; - for (const testCase of [ - { - missing: [node], - line: `Service broken: com.callstack.simlock.worker (launchd) names missing ${node}. Run \`simlock daemon install\` again.`, - }, - { - missing: [node, main], - line: `Service broken: com.callstack.simlock.worker (launchd) names missing ${node}, ${main}. Run \`simlock daemon install\` again.`, - }, - ]) { - const output = outputCapture(); - const findings = [ - { - kind: "service-broken" as const, - manager: "launchd" as const, - name: "com.callstack.simlock.worker", - missing: testCase.missing, - remedy: "Run `simlock daemon install` again.", - }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), - }), - ), - ).resolves.toBe(0); - - expect(output.stderr.split("\n").filter((line) => line.startsWith("Service broken"))).toEqual( - [testCase.line], - ); - expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); - } - }); - - it("simlock doctor prints a driver-advisory finding to stderr as `Warning [] : `", async () => { - const output = outputCapture(); - const findings = [ - { - code: "device-profile-hidden", - kind: "driver-advisory" as const, - message: '"Pixel 9" in devices.xml is not used', - platform: "android" as const, - }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), - }), - ), - ).resolves.toBe(0); - - expect(output.stderr.split("\n")).toContain( - 'Warning [android] device-profile-hidden: "Pixel 9" in devices.xml is not used', - ); - }); - - it("simlock doctor writes `Unavailable [] : ` on stderr for a driver-unavailable finding and prints the report on stdout unchanged", async () => { - const output = outputCapture(); - const findings = [ - { - detail: "Refusing the ios device set: bad. Restart the daemon.", - kind: "driver-unavailable" as const, - platform: "ios" as const, - reason: "invalid-device-set", - }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), - }), - ), - ).resolves.toBe(0); - - expect(output.stderr.split("\n")).toContain( - "Unavailable [ios] invalid-device-set: Refusing the ios device set: bad. Restart the daemon.", - ); - expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); - }); - - it("simlock doctor writes agent-device fail ; on stderr for a module-unavailable finding, one line of at most 500 characters per part, after the advisory, prerequisite and warm-pool lines, prints the report on stdout unchanged, and exits 0", async () => { - const output = outputCapture(); - const findings = [ - { - kind: "module-unavailable" as const, - module: "agent-device", - message: `found 0.20.3,\n need ${"x".repeat(600)}`, - remedy: `install agent-device 0.21.28 or later,\nthen ${"y".repeat(600)}`, - }, - { - kind: "warm-pool-target-unreachable" as const, - message: "iOS 27.0 is not installed", - platform: "ios" as const, - reason: "runtime-missing" as const, - remedy: "run simlock component install ios 27.0", - target: "iPhone 17 / 27.0 / full", - }, - { - kind: "prerequisite-missing" as const, - message: "Xcode is not installed.", - platform: "ios" as const, - prerequisite: "xcode", - remedy: "Run `xcode-select`.", - }, - { - code: "slim-runtime-unsupported", - kind: "driver-advisory" as const, - message: "detail", - platform: "ios" as const, - }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), - }), - ), - ).resolves.toBe(0); - - const lines = output.stderr.split("\n"); - const index = lines.findIndex((line) => line.startsWith("agent-device fail ")); - expect(index).toBeGreaterThan(-1); - expect(lines[index]).toBe( - `agent-device fail found 0.20.3, need ${"x".repeat(500 - "found 0.20.3, need ".length)}; ${("install agent-device 0.21.28 or later, then " + "y".repeat(600)).slice(0, 500)}`, - ); - expect(lines.filter((line) => line.startsWith("agent-device"))).toHaveLength(1); - for (const earlier of ["Warning [ios]", "Missing [ios]", "warm-pool-target-unreachable"]) { - expect(lines.findIndex((line) => line.startsWith(earlier))).toBeLessThan(index); - expect(lines.findIndex((line) => line.startsWith(earlier))).toBeGreaterThan(-1); - } - expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); - }); - - it("simlock doctor writes fail … for a finding of another module", async () => { - const output = outputCapture(); - const findings = [ - { - kind: "module-unavailable" as const, - message: "needs\n tmux", - module: "other-mod", - remedy: "install tmux", - }, - ]; - - await runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] } as never) }), - }), - ); - - expect(output.stderr.split("\n")).toContain("other-mod fail needs tmux; install tmux"); - expect(output.stderr).not.toContain("agent-device"); - }); - - it("keeps --purge-orphans off the wire when --fix is all that was asked for", async () => { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["doctor", "--fix"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - runDoctor: (input) => { - seen.push(input); - return Promise.resolve({ findings: [], sources: [] }); - }, - }), - }), - ), - ).resolves.toBe(0); - - // why: The upgrade contract: `doctor --fix` running unattended in CI never becomes - // destructive on its own (ADR 0001, decision 6). - expect(seen).toEqual([{ fix: true, purgeOrphans: false }]); - }); - - it.each([ - ["declined", async () => false], - // why: Explicitly absent: the shared harness defaults `confirm` to an auto-yes, so leaving it - // out would test the default rather than the no-terminal case. - ["unavailable", undefined], - ] as const)( - "refuses --purge-orphans and contacts nobody when confirmation is %s", - async (_case, confirm) => { - const output = outputCapture(); - let connected = false; - - await expect( - runCli( - ["doctor", "--purge-orphans"], - output.environmentWith({ - // why: `exactOptionalPropertyTypes`: an explicitly-absent confirm has to be spread in, - // not passed as `undefined`. - ...(confirm === undefined ? { confirm: undefined } : { confirm }), - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(2); - - expect(connected).toBe(false); - expect(JSON.parse(output.stderr)).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("--yes") }, - }); - }, - ); - - it("takes --yes as the confirmation, so an unattended purge needs no terminal", async () => { - const output = outputCapture(); - const seen: unknown[] = []; - - await expect( - runCli( - ["doctor", "--purge-orphans", "--yes"], - output.environmentWith({ - confirm: async () => { - throw new Error("--yes must not ask"); - }, - connectAdmin: async () => - fakeClient({ - runDoctor: (input) => { - seen.push(input); - return Promise.resolve({ findings: [], sources: [] }); - }, - }), - }), - ), - ).resolves.toBe(0); - - expect(seen).toEqual([{ fix: false, purgeOrphans: true }]); - }); -}); - -describe("CLI: device add and device remove (ADR 0022 §2)", () => { - async function runDevice(argv: string[]) { - const output = outputCapture(); - let connected = false; - const code = await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ); - return { code, connected, stderr: output.stderr, stdout: output.stdout }; - } - - it.each([ - ["an unknown subcommand", ["device", "enroll", "X"]], - ["two IDs", ["device", "add", "--platform", "ios", "A", "B"]], - ["--yes, which remove does not take", ["device", "remove", "--yes", "X"]], - ["no --platform", ["device", "add", "X"]], - ])("exits 2 with USAGE and contacts nobody for %s", async (_case, argv) => { - const result = await runDevice(argv); - - expect(result.code).toBe(2); - expect(result.connected).toBe(false); - expect(JSON.parse(result.stderr)).toEqual({ - error: { code: "USAGE", message: expect.any(String) }, - }); - }); - - it("prints one line for an enrollment, naming the platform's OS, and the JSON with --json", async () => { - const line = await runDevice(["device", "add", "--platform", "ios", "UDID-1"]); - const json = await runDevice(["device", "add", "--platform", "android", "SER-1", "--json"]); - - expect(line.stdout).toBe( - "Enrolled iPhone 15 Pro (iOS 18.4) as UDID-1, 2 user apps recorded.\n", - ); - expect(JSON.parse(json.stdout)).toMatchObject({ driverDeviceId: "SER-1", platform: "android" }); - }); - - it.each([ - ["device", ["device"]], - ["device --help", ["device", "--help"]], - ["device -h", ["device", "-h"]], - ["device add --help", ["device", "add", "--help"]], - ["device remove -h", ["device", "remove", "-h"]], - ])("prints the usage and contacts nobody for %s", async (_case, argv) => { - const result = await runDevice(argv); - - expect(result.code).toBe(0); - expect(result.connected).toBe(false); - expect(result.stdout).toBe( - "Usage: simlock device add --platform [--json]\n" + - " simlock device remove [--json]\n", - ); - }); - - it("says what is wrong with a usage error, and refuses --platform on remove", async () => { - const messages = async (argv: string[]) => { - const result = await runDevice(argv); - expect(result.code).toBe(2); - return (JSON.parse(result.stderr) as { error: { message: string } }).error.message; - }; - - expect(await messages(["device", "enroll", "X"])).toContain("Unknown device command: enroll"); - expect(await messages(["device", "add", "--platform", "ios"])).toContain( - "device add takes exactly one ", - ); - expect(await messages(["device", "remove", "A", "B"])).toContain( - "device remove takes exactly one ", - ); - expect(await messages(["device", "add", "--platform", "windows", "X"])).toContain( - "device add needs --platform ios or --platform android", - ); - expect(await messages(["device", "remove", "--platform", "ios", "X"])).toContain( - "Unknown option '--platform'", - ); - }); - - it("closes the connection after an enrollment and after a removal", async () => { - const closed: string[] = []; - for (const argv of [ - ["device", "add", "--platform", "ios", "U"], - ["device", "remove", "U"], - ]) { - const output = outputCapture(); - await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => ({ - ...fakeClient(), - close: () => { - closed.push(argv[1] ?? ""); - return Promise.resolve(); - }, - }), - }), - ); - } - - expect(closed).toEqual(["add", "remove"]); - }); - - it("says again for a re-enrollment, one user app in the singular, and the Android OS name", async () => { - const output = outputCapture(); - await runCli( - ["device", "add", "--platform", "android", "SER-1"], - output.environmentWith({ - connectAdmin: async () => ({ - ...fakeClient(), - addDevice: () => - Promise.resolve({ - appCount: 1, - class: "tablet" as const, - driverDeviceId: "SER-1", - id: "dev_1", - model: "Pixel Tablet", - osVersion: "15", - platform: "android" as const, - reenrolled: true, - }), - }), - }), - ); - - expect(output.stdout).toBe( - "Enrolled Pixel Tablet (Android 15) as SER-1 again, 1 user app recorded.\n", - ); - }); - - it("prints the operation's output for remove with --json", async () => { - const result = await runDevice(["device", "remove", "UDID-1", "--json"]); - - expect(JSON.parse(result.stdout)).toEqual({ driverDeviceId: "UDID-1", id: "UDID-1" }); - }); - - it("prints one line for a removal, with no confirmation", async () => { - const result = await runDevice(["device", "remove", "UDID-1"]); - - expect(result.code).toBe(0); - expect(result.stdout).toBe( - "Removed UDID-1. It is no longer enrolled; nothing on the device changed.\n", - ); - }); -}); - -describe("CLI: component remove (ADR 0010 §8)", () => { - function removingClient(asked: unknown[]) { - return fakeClient({ - removeComponent: async (input) => { - asked.push(input); - return { - outcome: input.dryRun === true ? "would-remove" : "removed", - platform: input.platform, - sizeBytes: 7_000_000_000, - version: input.version, - }; - }, - }); - } - - it("removes with --yes and prints the daemon's answer as one JSON line on stdout", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - - await expect( - runCli( - ["component", "remove", "ios", "26.4", "--yes"], - output.environmentWith({ - confirm: async () => { - throw new Error("--yes must not ask"); - }, - connectAdmin: async () => removingClient(asked), - }), - ), - ).resolves.toBe(0); - - expect(asked).toEqual([{ platform: "ios", version: "26.4" }]); - expect(output.stdout.trim().split("\n")).toHaveLength(1); - expect(JSON.parse(output.stdout)).toEqual({ - outcome: "removed", - platform: "ios", - sizeBytes: 7_000_000_000, - version: "26.4", - }); - }); - - it("asks for confirmation without --yes, and removes once it is given", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const questions: string[] = []; - - await expect( - runCli( - ["component", "remove", "android", "35"], - output.environmentWith({ - confirm: async (question) => { - questions.push(question); - return true; - }, - connectAdmin: async () => removingClient(asked), - }), - ), - ).resolves.toBe(0); - - expect(questions).toEqual(["Remove android 35? [y/N] "]); - expect(asked).toEqual([{ platform: "android", version: "35" }]); - }); - - it.each([ - ["declined", async () => false], - // why: No terminal to ask on: the CLI's own `confirmTerminal` is not installed, so this is the - // no-terminal case rather than the default. - ["unavailable", undefined], - ] as const)( - "refuses without --yes when confirmation is %s, and contacts nobody", - async (_case, confirm) => { - const output = outputCapture(); - let connected = false; - - await expect( - runCli( - ["component", "remove", "ios", "26.4"], - output.environmentWith({ - ...(confirm === undefined ? { confirm: undefined } : { confirm }), - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(2); - - expect(connected).toBe(false); - expect(JSON.parse(output.stderr)).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("--yes") }, - }); - }, - ); - - it("does not ask under --dry-run, and asks the daemon for a dry run", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - - await expect( - runCli( - ["component", "remove", "ios", "26.4", "--dry-run"], - output.environmentWith({ - confirm: async () => { - throw new Error("--dry-run must not ask"); - }, - connectAdmin: async () => removingClient(asked), - }), - ), - ).resolves.toBe(0); - - expect(asked).toEqual([{ dryRun: true, platform: "ios", version: "26.4" }]); - expect(JSON.parse(output.stdout)).toMatchObject({ outcome: "would-remove" }); - }); - - it.each([ - ["COMPONENT_NOT_OWNED", {}, 12], - ["COMPONENT_IN_USE", { devices: 1, foreignDevices: 2 }, 12], - ["COMPONENT_BUSY", {}, 11], - ] as const)( - "exits with the table's code when the daemon refuses with %s", - async (code, details, exit) => { - const output = outputCapture(); - const client = fakeClient({ - removeComponent: () => - Promise.reject(new SimlockError(code, "domain", "refused", details as never)), - }); - - await expect( - runCli( - ["component", "remove", "ios", "26.4", "--yes"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(exit); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ - error: { code }, - }); - }, - ); - - it.each([ - ["no version", ["component", "remove", "ios"]], - ["an unknown platform", ["component", "remove", "tvos", "1.0", "--yes"]], - ["an extra argument", ["component", "remove", "ios", "26.4", "extra", "--yes"]], - ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { - const output = outputCapture(); - let connected = false; - - await expect( - runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(2); - expect(connected).toBe(false); - }); -}); - -describe("CLI: config set validates with the real config loader (ADR 0003 §11, B9)", () => { - it.each([ - ["an unknown key", "capacty.limits.ios.maxDevices", "2"], - ["a key missing the Ms suffix", "lease.defaultTtl", "60000"], - ])( - "rejects %s instead of silently dropping it or validating clean", - async (_name, key, value) => { - const output = outputCapture(realCliEnvironmentPorts()); - let wrote = false; - const exitCode = await runCli( - ["config", "set", key, value], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async () => { - wrote = true; - }, - }), - ); - expect(exitCode).toBe(2); - expect(wrote).toBe(false); - expect(JSON.parse(output.stderr).error.code).toBe("USAGE"); - }, - ); - - it.each([ - ["ios.simulator.audio", "false", "loud"], - ["android.emulator.headless", "true", "yes"], - ])( - "writes a good value of %s, a key its driver declares, and refuses a bad one naming the key", - async (key, good, bad) => { - const run = async (value: string) => { - const output = outputCapture(realCliEnvironmentPorts()); - let wrote = false; - const exitCode = await runCli( - ["config", "set", key, value], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async () => { - wrote = true; - }, - }), - ); - return { exitCode, stderr: output.stderr, wrote }; - }; - - const accepted = await run(good); - const refused = await run(bad); - - expect([accepted.exitCode, accepted.wrote]).toEqual([0, true]); - expect([refused.exitCode, refused.wrote]).toEqual([2, false]); - expect(refused.stderr).toContain(key); - }, - ); - - it("--gateway config set validates against the gateway's rules, not the worker's", async () => { - const gatewayHome = { - role: "gateway", - dataDirectory: "/simlock/gateway", - gatewayPresent: true, - } as const; - const output = outputCapture({ ...realCliEnvironmentPorts(), daemonHome: gatewayHome }); - let wrote = false; - const exitCode = await runCli( - ["--gateway", "config", "set", "http.enabled", "false"], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async () => { - wrote = true; - }, - }), - ); - expect(exitCode).toBe(2); - expect(wrote).toBe(false); - expect(output.stderr).toContain("http.enabled"); - }); - - it("--gateway daemon start refuses a config.json that fails gateway validation with exit 2 and USAGE, before any launch", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock/gateway"); - await filesystem.writeFileAtomic( - "/simlock/gateway/config.json", - JSON.stringify({ http: { enabled: false } }), - ); - const launcher = new FakeDaemonLauncher(); - const output = outputCapture({ - ...realCliEnvironmentPorts(filesystem), - daemonHome: { role: "gateway", dataDirectory: "/simlock/gateway", gatewayPresent: true }, - launcher, - }); - const exitCode = await runCli( - ["--gateway", "daemon", "start", "--token", "probe"], - output.environmentWith(), - ); - expect(exitCode).toBe(2); - expect(launcher.launches).toBe(0); - expect(JSON.parse(output.stderr).error).toEqual({ - code: "USAGE", - message: - 'Invalid config value for "http.enabled": expected true in gateway mode (a gateway is reached over HTTP)', - }); - }); - - describe("a joined worker's own HTTP listener (#811)", () => { - const joined = { url: "wss://fleet.example", token: "join-secret" }; - const homeWithGateway = { - role: "worker", - dataDirectory: "/simlock", - gatewayPresent: true, - } as const; - - async function workerFiles(contents: unknown) { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic("/simlock/config.json", JSON.stringify(contents)); - return filesystem; - } - - it.each([ - [ - "enabling HTTP on a joined worker", - ["http.enabled", "true"], - { gateway: joined }, - 'Invalid config after setting http.enabled: Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', - ], - [ - "adding the gateway object to a standalone worker with HTTP on", - ["gateway", JSON.stringify(joined)], - { http: { enabled: true } }, - 'Invalid config after setting gateway: Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', - ], - ] as const)( - "refuses a config set that creates the conflict with exit 2, USAGE and the file untouched: %s", - async (_name, [key, value], contents, message) => { - const filesystem = await workerFiles(contents); - const before = await filesystem.readFile("/simlock/config.json"); - const output = outputCapture({ - ...realCliEnvironmentPorts(filesystem), - daemonHome: homeWithGateway, - }); - const exitCode = await runCli( - ["--local", "config", "set", key, value], - output.environmentWith(), - ); - expect(exitCode).toBe(2); - expect(JSON.parse(output.stderr).error).toMatchObject({ code: "USAGE", message }); - expect(await filesystem.readFile("/simlock/config.json")).toBe(before); - }, - ); - - it("repairs a conflicting file with config set http.enabled false and prints the restart notice", async () => { - const filesystem = await workerFiles({ gateway: joined, http: { enabled: true } }); - const output = outputCapture({ - ...realCliEnvironmentPorts(filesystem), - daemonHome: homeWithGateway, - }); - const exitCode = await runCli( - ["--local", "config", "set", "http.enabled", "false"], - output.environmentWith(), - ); - expect(exitCode).toBe(0); - expect(output.stdout).toBe( - "Updated http.enabled in /simlock/config.json; takes effect on daemon restart.\n", - ); - expect(JSON.parse(await filesystem.readFile("/simlock/config.json"))).toMatchObject({ - http: { enabled: false }, - }); - }); - - it("--local daemon start with no daemon listening refuses a joined worker's HTTP conflict with exit 2 and USAGE, before any launch", async () => { - const filesystem = await workerFiles({ - gateway: { url: "ws://127.0.0.1:1", token: "join-secret" }, - http: { enabled: true }, - }); - // why: a launcher that exits at once, so a start that wrongly launches fails the launches assertion instead of waiting out the test. - const launcher = new FakeDaemonLauncher(undefined, { exit: { code: 1, signal: null } }); - const output = outputCapture({ - ...realCliEnvironmentPorts(filesystem), - daemonHome: homeWithGateway, - launcher, - }); - const exitCode = await runCli(["--local", "daemon", "start"], output.environmentWith()); - expect(exitCode).toBe(2); - expect(launcher.launches).toBe(0); - expect(JSON.parse(output.stderr).error).toEqual({ - code: "USAGE", - message: - 'Invalid config value for "http.enabled": expected false when gateway.url is set (joined workers are reached through the gateway)', - }); - }); - }); - - it("simlock config set eventLog.retention rejects a non-positive value, naming the key", async () => { - for (const value of ["0", "1.5"]) { - const output = outputCapture(realCliEnvironmentPorts()); - let wrote = false; - const exitCode = await runCli( - ["config", "set", "eventLog.retention", value], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async () => { - wrote = true; - }, - }), - ); - expect(exitCode).toBe(2); - expect(wrote).toBe(false); - expect(output.stderr).toContain("eventLog.retention"); - } - }); - - it("still writes a genuinely valid key", async () => { - const output = outputCapture(realCliEnvironmentPorts()); - let written: Record | undefined; - const exitCode = await runCli( - // why: Comfortably under the default `lease.maxTtlMs` (4h), so this doesn't also trip the - // TTL pair rule -- this test is only about the key itself validating clean. - ["config", "set", "lease.defaultTtlMs", "2400000"], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async (contents) => { - written = contents; - }, - }), - ); - expect(exitCode).toBe(0); - expect(written).toEqual({ lease: { defaultTtlMs: 2400000 } }); - }); - - it.each(["teamId", "bundleId", "provisioningProfile", "signingIdentity"])( - "simlock config set capabilities.agent-device.ios.%s stores a non-empty string and refuses an empty string or a number with the usage error", - async (key) => { - const path = `capabilities.agent-device.ios.${key}`; - const set = async (value: string) => { - const output = outputCapture(realCliEnvironmentPorts()); - let written: Record | undefined; - const exitCode = await runCli( - ["config", "set", path, value], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async (contents) => { - written = contents; - }, - }), - ); - return { exitCode, output, written }; - }; - - const stored = await set("ABCDE12345"); - expect(stored.exitCode).toBe(0); - expect(stored.output.stdout).toBe( - `Updated ${path} in /simlock/config.json; takes effect on daemon restart.\n`, - ); - expect(stored.written).toEqual({ - capabilities: { "agent-device": { ios: { [key]: "ABCDE12345" } } }, - }); - for (const value of ["", "42"]) { - const refused = await set(value); - expect(refused.exitCode).toBe(2); - expect(JSON.parse(refused.output.stderr)).toEqual({ - error: { - code: "USAGE", - message: `Invalid config after setting ${path}: Invalid config value for "${path}": expected a non-empty string`, - }, - }); - expect(refused.written).toBeUndefined(); - } - }, - ); - - it("does not let a stray file at the scratch validation path change the outcome", async () => { - // why: Guards against B9's second bug: the validation scratch path must never be able to read a - // real file, whether one happens to already exist there or not. - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/"); - await filesystem.writeFileAtomic( - "/config-set-validation.json", - `${JSON.stringify({ totally: { bogus: true } })}\n`, - ); - const output = outputCapture(realCliEnvironmentPorts(filesystem)); - let wrote = false; - const exitCode = await runCli( - ["config", "set", "downloads.policy", "never"], - output.environmentWith({ - readConfigFile: async () => ({}), - writeConfigFile: async () => { - wrote = true; - }, - }), - ); - expect(exitCode).toBe(0); - expect(wrote).toBe(true); - }); -}); - -describe("CLI: config set and the retired eventLog.rotateBytes", () => { - const configSet = async (file: Record) => { - const output = outputCapture(realCliEnvironmentPorts()); - let written: Record | undefined; - const environment = output.environmentWith({ - readConfigFile: async () => structuredClone(file), - writeConfigFile: async (contents) => { - written = contents; - }, - }); - const exitCode = await runCli(["config", "set", "log.level", "debug"], environment); - return { configPath: environment.configPath, exitCode, output, written }; - }; - - it("simlock config set on a file holding eventLog.rotateBytes writes it without that key and prints the Removed line once; on a file without it, no such line", async () => { - const holding = await configSet({ - eventLog: { retention: 86_400_000, rotateBytes: 1_048_576 }, - log: { level: "info" }, - }); - const without = await configSet({ - eventLog: { retention: 86_400_000 }, - log: { level: "info" }, - }); - - expect(holding.exitCode).toBe(0); - expect(holding.written).toEqual({ - eventLog: { retention: 86_400_000 }, - log: { level: "debug" }, - }); - expect(holding.output.stderr).toBe( - `Removed eventLog.rotateBytes from ${holding.configPath}: it is no longer used.\n`, - ); - expect(holding.output.stdout).toBe( - `Updated log.level in ${holding.configPath}; takes effect on daemon restart.\n`, - ); - expect(without.exitCode).toBe(0); - expect(without.written).toEqual({ - eventLog: { retention: 86_400_000 }, - log: { level: "debug" }, - }); - expect(without.output.stderr).toBe(""); - }); -}); - -describe("CLI: config set and the file mode of config.json (#252)", () => { - it("writes the config.json holding gateway.token owner-only, whether it creates the file or replaces an owner-only one", async () => { - // why: Two homes, one per way `config set` comes to write the join token. `MemoryFilesystem` - // gives a file written without a `mode` Node's default (0644), so the assertion pins what - // the CLI asks for, independent of the umask of whoever runs the suite. - const fresh = new MemoryFilesystem(); - const freshExit = await runCli( - ["config", "set", "gateway", '{"url":"wss://gw.example:4700","token":"slk_fresh"}'], - outputCapture(realCliEnvironmentPorts(fresh)).environmentWith(), - ); - - const tightened = new MemoryFilesystem(); - await tightened.mkdirp("/simlock"); - await tightened.writeFileAtomic( - "/simlock/config.json", - `${JSON.stringify({ gateway: { url: "wss://gw.example:4700" } })}\n`, - { mode: 0o600 }, - ); - const tightenedExit = await runCli( - ["config", "set", "gateway.token", "slk_replaced"], - outputCapture(realCliEnvironmentPorts(tightened)).environmentWith(), - ); - - expect([freshExit, tightenedExit]).toEqual([0, 0]); - expect([ - JSON.parse(await fresh.readFile("/simlock/config.json")).gateway.token, - JSON.parse(await tightened.readFile("/simlock/config.json")).gateway.token, - ]).toEqual(["slk_fresh", "slk_replaced"]); - expect({ - created: (await fresh.stat("/simlock/config.json")).mode?.toString(8), - replaced: (await tightened.stat("/simlock/config.json")).mode?.toString(8), - }).toEqual({ created: "600", replaced: "600" }); - }); - - it("makes a config.json that others could read owner-only on the next config set", async () => { - // why: A file written by hand or by an older simlock: readable by group and others. `config set` - // must not carry that mode over to the file it writes in its place. - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic( - "/simlock/config.json", - `${JSON.stringify({ gateway: { url: "wss://gw.example:4700", token: "slk_old" } })}\n`, - { mode: 0o644 }, - ); - - const exit = await runCli( - ["config", "set", "gateway.label", "build-mac"], - outputCapture(realCliEnvironmentPorts(filesystem)).environmentWith(), - ); - - expect(exit).toBe(0); - expect(JSON.parse(await filesystem.readFile("/simlock/config.json")).gateway).toEqual({ - url: "wss://gw.example:4700", - token: "slk_old", - label: "build-mac", - }); - expect((await filesystem.stat("/simlock/config.json")).mode?.toString(8)).toBe("600"); - }); -}); - -describe("CLI: --json shape is the contract, as-is (ADR 0003 §11)", () => { - it("status --json carries camelCase fields, not expires_at_ms, estimated_boot_ms, queue_position or device_unhealthy", async () => { - const output = outputCapture(); - await runCli( - ["status", "--json"], - output.environmentWith({ connectAdmin: async () => fakeClient() }), - ); - const text = output.stdout; - expect(text).not.toMatch(/expires_at_ms|estimated_boot_ms|queue_position|device_unhealthy/); - const parsed = JSON.parse(text) as StatusGetOutput; - expect(parsed.queueDepth).toBe(0); - }); - - it("lease output is the contract's LeaseGrant plus the resolved role", async () => { - const output = outputCapture(); - await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith({ connectAdmin: async () => fakeClient({ role: "admin" }) }), - ); - const parsed = JSON.parse(output.stdout) as LeaseGrant & { role: string }; - expect(parsed.lease.id).toBe("lse_1"); - expect(parsed.role).toBe("admin"); - expect(output.stdout).not.toMatch(/expires_at_ms|estimated_boot_ms/); - }); -}); - -describe("CLI: --yes / confirm guards (safety rule 5)", () => { - it("release --all without --yes and a confirm that declines refuses with exit 2 and releases nothing", async () => { - const output = outputCapture(); - let released = false; - await expect( - runCli( - ["release", "--all"], - output.environmentWith({ - confirm: async () => false, - connectAdmin: async () => - fakeClient({ - releaseAllLeases: () => { - released = true; - return Promise.resolve({ leaseIds: [] }); - }, - }), - }), - ), - ).resolves.toBe(2); - expect(released).toBe(false); - }); - - it("release --all --yes bypasses the interactive confirm", async () => { - const output = outputCapture(); - let released = false; - await expect( - runCli( - ["release", "--all", "--yes"], - output.environmentWith({ - confirm: async () => false, - connectAdmin: async () => - fakeClient({ - releaseAllLeases: () => { - released = true; - return Promise.resolve({ leaseIds: ["lse_1"] }); - }, - }), - }), - ), - ).resolves.toBe(0); - expect(released).toBe(true); - }); - - it("nuke without confirmation or --yes refuses before connecting", async () => { - const output = outputCapture(); - let connected = false; - await expect( - runCli( - ["nuke"], - output.environmentWith({ - confirm: async () => false, - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(2); - expect(connected).toBe(false); - }); -}); - -describe("CLI: token operations go through the daemon", () => { - it("token create/list/revoke all go through the client, not a local TokenStore", async () => { - const calls: string[] = []; - const client = fakeClient({ - createToken: (input) => { - calls.push("create"); - return Promise.resolve({ - secret: "sec_1", - token: { id: "tok_1", role: input.role, createdAt: 0 }, - }); - }, - listTokens: () => { - calls.push("list"); - return Promise.resolve({ tokens: [{ id: "tok_1", role: "operator", createdAt: 0 }] }); - }, - revokeToken: () => { - calls.push("revoke"); - return Promise.resolve({ revoked: true }); - }, - }); - const environment = outputCapture().environmentWith({ connectAdmin: async () => client }); - await runCli(["token", "create", "--role", "operator"], environment); - await runCli(["token", "list"], environment); - await runCli(["token", "revoke", "tok_1"], environment); - expect(calls).toEqual(["create", "list", "revoke"]); - }); -}); - -describe("CLI: catalog", () => { - it("prints a model named after an Object.prototype member with no paired runtime", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getCatalog: () => - Promise.resolve({ - platforms: [ - { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: {}, - models: ["constructor"], - platform: "ios", - runtimes: [], - }, - ], - }), - }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toContain(" (no class):\n constructor: (no paired runtime)\n"); - }); - - it("prints (none) for a platform with no models", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getCatalog: () => - Promise.resolve({ - platforms: [ - { - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: {}, - models: [], - platform: "android", - runtimes: ["35"], - }, - ], - }), - }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toBe( - ["Platform: android", " Runtimes: 35 (default: (none))", " Models: (none)", ""].join("\n"), - ); - }); - - it("prints each model with the runtimes it pairs with", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getCatalog: () => - Promise.resolve({ - platforms: [ - { - defaultRuntime: "26.0", - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: { - "iPhone 16": ["18.4", "26.0"], - "iPhone 8": [], - "iPhone XS": ["18.4"], - }, - models: ["iPhone 16", "iPhone XS", "iPhone 8"], - platform: "ios", - runtimes: ["18.4", "26.0"], - }, - ], - }), - }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toBe( - [ - "Platform: ios", - " Runtimes: 18.4, 26.0 (default: 26.0)", - " Models:", - " (no class):", - " iPhone 16: 18.4, 26.0", - " iPhone XS: 18.4", - " iPhone 8: (no paired runtime)", - "", - ].join("\n"), - ); - }); - - it("prints each model under its class in the order of the enum, and an unclassed model under (no class)", async () => { - const catalog = { - defaultRuntime: "26.0", - classDefaults: { phone: "iPhone 17", tablet: "iPad Pro" }, - modelAliases: { "iPhone 17": ["iphone-17", "i17"] }, - modelClasses: { - "Apple TV 4K": "tv" as const, - "Apple Watch Series 11 (46mm)": "watch" as const, - "iPad Pro": "tablet" as const, - "iPhone 17": "phone" as const, - "iPhone 16": "phone" as const, - }, - modelRuntimes: { - "Apple TV 4K": ["26.0"], - "Apple Watch Series 11 (46mm)": ["26.0"], - "iPad Pro": ["26.0"], - "iPhone 16": ["26.0"], - "iPhone 17": ["26.0"], - Mystery: [], - }, - models: [ - "Mystery", - "Apple Watch Series 11 (46mm)", - "iPhone 17", - "Apple TV 4K", - "iPad Pro", - "iPhone 16", - ], - platform: "ios" as const, - runtimes: ["26.0"], - customModels: ["iPhone 16"], - }; - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ getCatalog: () => Promise.resolve({ platforms: [catalog] }) }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toBe( - [ - "Platform: ios", - " Runtimes: 26.0 (default: 26.0)", - " Models:", - " phone (default: iPhone 17):", - " iPhone 17: 26.0", - " Other names: iphone-17, i17", - " iPhone 16 (custom): 26.0", - " tablet (default: iPad Pro):", - " iPad Pro: 26.0", - " watch (default: none):", - " Apple Watch Series 11 (46mm): 26.0", - " tv (default: none):", - " Apple TV 4K: 26.0", - " (no class):", - " Mystery: (no paired runtime)", - "", - ].join("\n"), - ); - }); - - it.each([ - { - field: "classDefaults", - catalog: { - classDefaults: { phone: "Pixel 9", tv: "Not Even Listed" }, - modelAliases: {}, - modelClasses: {}, - modelRuntimes: {}, - models: [], - platform: "android" as const, - runtimes: [], - }, - }, - { - field: "modelClasses", - catalog: { - modelAliases: {}, - classDefaults: {}, - modelClasses: { "Pixel 8": "phone" as const, "Odd One": "tv" as const }, - modelRuntimes: {}, - models: ["Pixel 8"], - platform: "android" as const, - runtimes: [], - }, - }, - ])("carries $field in --json as received", async ({ catalog }) => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ getCatalog: () => Promise.resolve({ platforms: [catalog] }) }), - }); - - await expect(runCli(["catalog", "--json"], environment)).resolves.toBe(0); - - expect(JSON.parse(output.stdout)).toEqual({ platforms: [catalog] }); - }); - - it("prints (none) for a platform that has images but none installed", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getCatalog: () => - Promise.resolve({ - platforms: [ - { - images: [], - modelAliases: {}, - classDefaults: {}, - modelClasses: {}, - modelRuntimes: {}, - models: [], - platform: "android", - runtimes: [], - }, - ], - }), - }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toBe( - [ - "Platform: android", - " Runtimes: (none) (default: (none))", - " Models: (none)", - " Images: (none)", - "", - ].join("\n"), - ); - }); - - it("prints a model's other names and each image", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getCatalog: () => - Promise.resolve({ - platforms: [ - { - defaultRuntime: "35", - images: [ - { abi: "x86_64", runtime: "34", tag: "default" }, - { abi: "arm64-v8a", runtime: "35", tag: "google_apis" }, - ], - modelAliases: { "Pixel 8": ["pixel_8"] }, - classDefaults: {}, - modelClasses: { "Pixel 8": "phone" }, - modelRuntimes: { "My Tablet": ["34", "35"], "Pixel 8": ["34", "35"] }, - models: ["My Tablet", "Pixel 8"], - platform: "android", - runtimes: ["34", "35"], - }, - ], - }), - }), - }); - - await expect(runCli(["catalog"], environment)).resolves.toBe(0); - - expect(output.stdout).toBe( - [ - "Platform: android", - " Runtimes: 34, 35 (default: 35)", - " Models:", - " phone (default: none):", - " Pixel 8: 34, 35", - " Other names: pixel_8", - " (no class):", - " My Tablet: 34, 35", - " Images (runtime, tag, ABI):", - " 34 default x86_64", - " 35 google_apis arm64-v8a", - "", - ].join("\n"), - ); - }); - - it("exits 12 when the daemon refuses with DOWNLOADS_DISABLED", async () => { - const output = outputCapture(); - const client = fakeClient({ - installComponent: () => - Promise.reject( - new SimlockError("DOWNLOADS_DISABLED", "domain", "downloads.policy is never", { - policy: "never", - }), - ), - }); - - await expect( - runCli( - ["component", "install", "ios", "26.4"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(12); - expect(output.stdout).toBe(""); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "DOWNLOADS_DISABLED", message: expect.any(String) }, - }); - }); - - it.each([ - ["a missing version", ["component", "install", "ios"]], - ["an extra argument", ["component", "install", "ios", "26.4", "26.5"]], - ["an unknown platform", ["component", "install", "tvos", "26.4"]], - ["--json", ["component", "install", "ios", "26.4", "--json"]], - ["an unknown subcommand", ["component", "uninstall", "ios", "26.4"]], - ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { - const output = outputCapture(); - let installs = 0; - const client = fakeClient({ - installComponent: (input) => { - installs += 1; - return Promise.resolve({ ...input, component: input.version, outcome: "installed" }); - }, - }); - - await expect( - runCli(argv, output.environmentWith({ connectAdmin: async () => client })), - ).resolves.toBe(2); - expect(installs).toBe(0); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ - error: { code: "USAGE" }, - }); - }); - - it.each([[["component"]], [["component", "--help"]], [["component", "install", "--help"]]])( - "prints the command's usage for %j without connecting to the daemon", - async (argv) => { - const output = outputCapture(); - let connected = false; - - await expect( - runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(0); - expect(output.stdout).toBe( - "Usage: simlock component install [--worker ... | --all-workers]\n" + - " simlock component list [--platform ]\n" + - " simlock component remove [--dry-run] [--yes]\n", - ); - expect(connected).toBe(false); - }, - ); -}); - -describe("CLI: component install on a gateway's workers (ADR 0010 §7)", () => { - function progressLines(stderr: string): unknown[] { - return stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as unknown) - .filter((line) => !Object.hasOwn(line as object, "notice")); - } - - it("--all-workers prints progress with each worker's id, one result line, and exits 15 when one worker is refused", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const results = [ - { outcome: "installed" as const, version: "35", workerId: "wrk_a" }, - { - error: { code: "DOWNLOADS_DISABLED", message: "Downloads are disabled" }, - outcome: "refused" as const, - workerId: "wrk_b", - }, - ]; - const client = fakeClient({ - installComponentOnWorkers: async (input, options) => { - asked.push(input); - options?.onProgress?.({ stage: "waiting", workerId: "wrk_a" }); - options?.onProgress?.({ fraction: 0.5, stage: "downloading", workerId: "wrk_b" }); - return { results }; - }, - }); - - await expect( - runCli( - ["component", "install", "android", "35", "--all-workers"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(15); - - expect(asked).toEqual([{ platform: "android", version: "35", workers: "all" }]); - expect(progressLines(output.stderr)).toEqual([ - { stage: "waiting", workerId: "wrk_a" }, - { fraction: 0.5, stage: "downloading", workerId: "wrk_b" }, - ]); - expect(output.stdout.trim().split("\n")).toHaveLength(1); - expect(JSON.parse(output.stdout)).toEqual({ results }); - }); - - it("exits 0 when every named worker ends installed or already-installed, and names each --worker given", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const client = fakeClient({ - installComponentOnWorkers: async (input) => { - asked.push(input); - return { - results: [ - { outcome: "installed", version: "26.4", workerId: "wrk_a" }, - { outcome: "already-installed", version: "26.4", workerId: "wrk_b" }, - ], - }; - }, - }); - - await expect( - runCli( - ["component", "install", "ios", "26.4", "--worker", "wrk_a", "--worker", "wrk_b"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(0); - - expect(asked).toEqual([{ platform: "ios", version: "26.4", workers: ["wrk_a", "wrk_b"] }]); - }); - - it.each(["skipped", "failed", "unknown"] as const)( - "exits 15 when a worker ends %s", - async (outcome) => { - const output = outputCapture(); - const client = fakeClient({ - installComponentOnWorkers: async () => ({ - results: [ - { outcome: "installed", version: "35", workerId: "wrk_a" }, - { error: { code: "WORKER_UNREACHABLE", message: "gone" }, outcome, workerId: "wrk_b" }, - ], - }), - }); - - await expect( - runCli( - ["component", "install", "android", "35", "--all-workers"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(15); - }, - ); - - it("refuses --worker together with --all-workers as USAGE, without connecting", async () => { - const output = outputCapture(); - let connected = false; - - await expect( - runCli( - ["component", "install", "android", "35", "--worker", "wrk_a", "--all-workers"], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ), - ).resolves.toBe(2); - expect(connected).toBe(false); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ - error: { code: "USAGE" }, - }); - }); - - it.each([ - "component install ", - "events, stats,", - "component list [--platform ]", - ])("lists the command in the usage banner: %s", async (fragment) => { - const output = outputCapture(); - await runCli(["--help"], output.environmentWith({})); - expect(output.stdout).toContain(fragment); - }); -}); - -describe("CLI: component list (ADR 0010 §8)", () => { - const listing = { - components: [ - { - devices: 2, - foreignDevices: 1, - installedAt: 1_790_864_071_200, - installedBySimlock: true, - platform: "android" as const, - sizeBytes: 4_200_000_000, - variant: "google_apis/arm64-v8a", - version: "35", - }, - ], - }; - - it("prints the daemon's component.list answer as one JSON object on stdout, and exits 0", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const client = fakeClient({ - listComponents: async (input) => { - asked.push(input); - return listing; - }, - }); - - await expect( - runCli(["component", "list"], output.environmentWith({ connectAdmin: async () => client })), - ).resolves.toBe(0); - - expect(asked).toEqual([{}]); - expect(output.stdout.trim().split("\n")).toHaveLength(1); - expect(JSON.parse(output.stdout)).toEqual(listing); - }); - - it("asks for one platform with --platform", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const client = fakeClient({ - listComponents: async (input) => { - asked.push(input); - return { components: [] }; - }, - }); - - await expect( - runCli( - ["component", "list", "--platform", "ios"], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(0); - - expect(asked).toEqual([{ platform: "ios" }]); - }); - - it.each([ - ["an unknown platform", ["component", "list", "--platform", "tvos"]], - ["a positional argument", ["component", "list", "ios"]], - ["an unknown flag", ["component", "list", "--json"]], - ])("exits 2 with USAGE for %s, without asking the daemon", async (_label, argv) => { - const output = outputCapture(); - let listed = 0; - const client = fakeClient({ - listComponents: () => { - listed += 1; - return Promise.resolve({ components: [] }); - }, - }); - - await expect( - runCli(argv, output.environmentWith({ connectAdmin: async () => client })), - ).resolves.toBe(2); - expect(listed).toBe(0); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ - error: { code: "USAGE" }, - }); - }); -}); - -describe("CLI: worker commands (ADR 0005 §8/§23)", () => { - const connectedWorker = { - capacity: { - android: { - atRamBudget: false, - limit: 2, - maxRunning: 2, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }, - global: { maxRunning: 4, overLimit: false, reserved: 0, running: 1, warm: 1 }, - ios: { - atRamBudget: false, - limit: 2, - maxRunning: 2, - overLimit: false, - reserved: 0, - running: 1, - used: 1, - warm: 1, - }, - }, - catalog: [], - connection: "connected" as const, - devices: [], - drained: false, - id: "wrk_1", - label: "mac-mini-1", - lastSeenAt: 1_000, - leases: [ - { - deviceId: "dev_1", - grantedAt: 1, - id: "lease_1", - lastRenewedAt: 1, - idChosenByRequester: false, - ownerId: "agent-1", - requesterId: "agent-1", - ttlDeadline: 2, - ttlMs: 1, - }, - ], - }; - - it("renders each worker's state, capacity and lease count", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ listWorkers: () => Promise.resolve({ workers: [connectedWorker] }) }), - }); - - await expect(runCli(["worker", "list"], environment)).resolves.toBe(0); - - expect(output.stdout).toContain("wrk_1 (mac-mini-1): connected -- ios 1/2"); - // why: No host facts on this view yet, so the line ends at the lease count. - expect(output.stdout).toContain("1 lease(s)\n"); - }); - - it("prints each worker's operating system, architecture, and tool versions", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - listWorkers: () => - Promise.resolve({ - workers: [ - { - ...connectedWorker, - host: { - arch: "arm64", - os: "macOS", - osVersion: "15.5", - tools: [ - { build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }, - { name: "emulator", platform: "android" as const, version: "35.4.9" }, - ], - }, - }, - ], - }), - }), - }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout).toContain( - "1 lease(s) -- macOS 15.5 arm64; xcode 16.4 (16F6), emulator 35.4.9", - ); - }); - - describe("a worker's Load line", () => { - const GIB = 2 ** 30; - const PARTS = { - cpu: { busyPercent: 34.2, cores: 10, limited: false }, - disk: { freeBytes: 120_688_581_018, totalBytes: 530_857_957_786 }, - memory: { - availableBytes: 10_522_669_875, - limited: false, - pressure: "normal" as const, - swapUsedBytes: 1_288_490_189, - totalBytes: 32 * GIB, - }, - }; - const READING = - "CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free"; - const NOW = 61_000; - const install = { - component: "26.4", - platform: "ios" as const, - since: 1_000, - state: "downloading" as const, - waiters: 2, - }; - const printWorkers = async (workers: unknown[]) => { - const output = outputCapture(); - await runCli( - ["worker", "list"], - output.environmentWith({ - clock: new FakeClock(NOW), - connectAdmin: async () => - fakeClient({ listWorkers: () => Promise.resolve({ workers } as never) }), - }), - ); - return output.stdout; - }; - - it("simlock worker list prints a connected or starting worker's Load line after its own line and before its install lines", async () => { - const stdout = await printWorkers([ - { - ...connectedWorker, - installs: [install], - load: { ...PARTS, readAt: NOW - 12_000 }, - }, - { - connection: "connected", - drained: false, - health: "starting", - id: "wrk_2", - lastSeenAt: 1, - load: { memory: PARTS.memory, readAt: NOW - 3_000 }, - }, - ]); - - const lines = stdout.trimEnd().split("\n"); - expect(lines[0]).toMatch(/^wrk_1 \(mac-mini-1\): connected -- /); - expect(lines[1]).toBe(` Load: ${READING} · read 12s ago`); - expect(lines[2]).toMatch(/^ {2}Install ios 26\.4: downloading for 60s, 2 waiters$/); - expect(lines[3]).toMatch(/^wrk_2: connected, starting/); - expect(lines[4]).toBe( - " Load: CPU unavailable · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk unavailable · read 3s ago", - ); - }); - - it("simlock worker list prints `no reading while disconnected` for a disconnected worker, and no Load line for an incompatible one or one that sent none", async () => { - const stdout = await printWorkers([ - { ...connectedWorker, connection: "disconnected", drained: true, id: "wrk_a" }, - { - connection: "incompatible", - drained: false, - id: "wrk_b", - lastSeenAt: 1, - protocol: { gateway: { max: 35, min: 35 }, worker: { max: 4, min: 4 } }, - }, - { ...connectedWorker, id: "wrk_c" }, - { ...connectedWorker, id: "wrk_d", load: { ...PARTS, readAt: NOW - 1_000 } }, - ]); - - const lines = stdout.trimEnd().split("\n"); - const loadAfter = (id: string) => { - const index = lines.findIndex( - (line) => line.startsWith(`${id}:`) || line.startsWith(`${id} (`), - ); - const next = lines[index + 1]; - return next?.startsWith(" Load:") === true ? next : undefined; - }; - expect(loadAfter("wrk_d")).toBe(` Load: ${READING} · read 1s ago`); - expect(loadAfter("wrk_a")).toBe(" Load: no reading while disconnected"); - expect(loadAfter("wrk_b")).toBeUndefined(); - expect(loadAfter("wrk_c")).toBeUndefined(); - }); - - it("simlock worker list prints `read 0s ago` for a readAt ahead of its clock", async () => { - const stdout = await printWorkers([ - { ...connectedWorker, load: { ...PARTS, readAt: NOW + 9_000 } }, - ]); - - expect(stdout).toContain(` Load: ${READING} · read 0s ago\n`); - }); - - it("simlock worker list rounds a reading's age up to whole seconds", async () => { - const stdout = await printWorkers([ - { ...connectedWorker, load: { ...PARTS, readAt: NOW - 1_001 } }, - ]); - - expect(stdout).toContain(` Load: ${READING} · read 2s ago\n`); - }); - - it("simlock status against a gateway prints the gateway's own Load line, and each worker's Load line aged from the status time", async () => { - const output = outputCapture(); - const status: StatusGetOutput = { - ...EMPTY_STATUS, - daemon: { - health: "running", - mode: "gateway", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - load: { ageMs: 4_000, ...PARTS }, - workers: [ - { - catalog: [], - connection: "connected", - devices: [], - drained: false, - id: "wrk_1", - lastSeenAt: 1, - leases: [], - load: { ...PARTS, readAt: NOW - 12_000 }, - }, - ], - }; - - await runCli( - ["status"], - output.environmentWith({ - clock: new FakeClock(NOW), - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - expect(output.stdout).toContain(`\nLoad: ${READING} · read 4s ago\n`); - expect(output.stdout).toContain(`\n Load: ${READING} · read 12s ago\n`); - }); - }); - - it("prints a starting worker as starting, with its leases unknown rather than none", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - listWorkers: () => - Promise.resolve({ - workers: [ - { - connection: "connected" as const, - drained: true, - health: "starting" as const, - id: "wrk_1", - lastSeenAt: 1, - }, - ], - }), - }), - }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout).toBe( - "wrk_1: connected, starting, drained -- capacity unknown, leases unknown\n", - ); - }); - - it("prints each worker's installs in progress under it", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - clock: new FakeClock(61_000), - connectAdmin: async () => - fakeClient({ - listWorkers: () => - Promise.resolve({ - workers: [ - { - ...connectedWorker, - installs: [ - { - component: "26.4", - platform: "ios" as const, - since: 1_000, - state: "downloading" as const, - waiters: 2, - }, - ], - }, - { - ...connectedWorker, - id: "wrk_2", - installs: [ - { - component: "35", - platform: "android" as const, - since: 31_000, - state: "waiting" as const, - waiters: 1, - }, - ], - }, - { ...connectedWorker, id: "wrk_3", installs: [] }, - ], - }), - }), - }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout).toContain( - "1 lease(s)\n Install ios 26.4: downloading for 60s, 2 waiters\nwrk_2 (mac-mini-1): connected", - ); - expect(output.stdout).toContain( - "1 lease(s)\n Install android 35: waiting for 30s, 1 waiter\nwrk_3 (mac-mini-1): connected", - ); - expect(output.stdout.match(/Install /g)).toHaveLength(2); - }); - - it("says so plainly when no worker has ever connected", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ connectAdmin: async () => fakeClient() }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout).toContain("No workers have connected"); - }); - - it("names the protocol skew on an incompatible worker", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - listWorkers: () => - Promise.resolve({ - workers: [ - { - ...connectedWorker, - connection: "incompatible" as const, - leases: [], - protocol: { gateway: { max: 5, min: 5 }, worker: { max: 4, min: 4 } }, - }, - ], - }), - }), - }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout).toContain("incompatible"); - expect(output.stdout).toContain("protocol 4-4 vs gateway 5-5"); - }); - - it("worker list shows a worker's version, the latest only when newer, and its update stage", async () => { - const waiting = { - forced: false, - from: "0.3.0", - leases: 2, - stage: "waiting", - startedAt: 1_000, - to: "0.4.0", - } as const; - const cases: readonly { - expected: string; - latestVersion: string | null; - update: WorkerView["update"]; - version: string | undefined; - }[] = [ - { - expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases", - latestVersion: "0.4.0", - update: waiting, - version: "0.3.0", - }, - { - expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: installing", - latestVersion: "0.4.0", - update: { - forced: false, - from: "0.3.0", - stage: "installing", - startedAt: 1_000, - to: "0.4.0", - }, - version: "0.3.0", - }, - { - expected: " -- updating to 0.4.0: waiting for 1 lease", - latestVersion: null, - update: { ...waiting, leases: 1 }, - version: undefined, - }, - { expected: " -- simlock 0.4.0", latestVersion: "0.4.0", update: null, version: "0.4.0" }, - { expected: " -- simlock 0.5.0", latestVersion: "0.4.0", update: null, version: "0.5.0" }, - { expected: " -- simlock 0.4.0", latestVersion: null, update: null, version: "0.4.0" }, - ]; - for (const { expected, latestVersion, update, version } of cases) { - const output = outputCapture(); - const worker = { - ...connectedWorker, - latestVersion, - update, - ...(version === undefined ? {} : { version }), - }; - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ listWorkers: () => Promise.resolve({ workers: [worker] }) }), - }); - - await runCli(["worker", "list"], environment); - - expect(output.stdout.trimEnd().slice(-expected.length)).toBe(expected); - if (!expected.includes("available")) expect(output.stdout).not.toContain("available"); - if (update === null) expect(output.stdout).not.toContain("updating"); - } - }); - - it("emits the raw contract value under --json", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ listWorkers: () => Promise.resolve({ workers: [connectedWorker] }) }), - }); - - await runCli(["worker", "list", "--json"], environment); - - expect(JSON.parse(output.stdout)).toMatchObject({ workers: [{ id: "wrk_1" }] }); - }); - - it("drains, undrains and removes by id", async () => { - const calls: string[] = []; - const client = fakeClient({ - drainWorker: (input) => { - calls.push(`drain:${input.workerId}`); - return Promise.resolve({ drained: true as const, workerId: input.workerId }); - }, - removeWorker: (input) => { - calls.push(`remove:${input.workerId}`); - return Promise.resolve({ removed: true, workerId: input.workerId }); - }, - undrainWorker: (input) => { - calls.push(`undrain:${input.workerId}`); - return Promise.resolve({ drained: false as const, workerId: input.workerId }); - }, - }); - const environment = outputCapture().environmentWith({ connectAdmin: async () => client }); - - await expect(runCli(["worker", "drain", "wrk_1"], environment)).resolves.toBe(0); - await expect(runCli(["worker", "undrain", "wrk_1"], environment)).resolves.toBe(0); - await expect(runCli(["worker", "remove", "wrk_2"], environment)).resolves.toBe(0); - - expect(calls).toEqual(["drain:wrk_1", "undrain:wrk_1", "remove:wrk_2"]); - }); - - it("requires a worker id, and rejects an unknown subcommand", async () => { - const environment = outputCapture().environmentWith({ - connectAdmin: async () => fakeClient(), - }); - - await expect(runCli(["worker", "drain"], environment)).resolves.toBe(2); - await expect(runCli(["worker", "explode"], environment)).resolves.toBe(2); - }); - - it("prints usage without connecting to anything", async () => { - const output = outputCapture(); - let connected = false; - const environment = output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }); - - await expect(runCli(["worker"], environment)).resolves.toBe(0); - - expect(output.stdout).toContain("simlock worker list"); - expect(connected).toBe(false); - }); - - it("mints a worker join token (ADR 0005 §8)", async () => { - const roles: string[] = []; - const client = fakeClient({ - createToken: (input) => { - roles.push(input.role); - return Promise.resolve({ - secret: "sec_join", - token: { createdAt: 0, id: "tok_1", role: input.role }, - }); - }, - }); - const output = outputCapture(); - const environment = output.environmentWith({ connectAdmin: async () => client }); - - await expect(runCli(["token", "create", "--role", "worker"], environment)).resolves.toBe(0); - - expect(roles).toEqual(["worker"]); - expect(JSON.parse(output.stdout)).toMatchObject({ token: { role: "worker" } }); - }); - - it("still rejects a role that is not one of the three", async () => { - const environment = outputCapture().environmentWith({ - connectAdmin: async () => fakeClient(), - }); - - await expect(runCli(["token", "create", "--role", "root"], environment)).resolves.toBe(2); - }); -}); - -describe("CLI: alerts test (ADR 0035 §1)", () => { - const USAGE = "Usage: simlock alerts test [--json]\n"; - const errorLine = (stderr: string): unknown => JSON.parse(stderr.trim().split("\n").at(-1) ?? ""); - - async function runAlerts(argv: string[], overrides: Partial = {}) { - const output = outputCapture(); - const inputs: unknown[] = []; - let connected = false; - const code = await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient({ - testAlertDestination: (input) => { - inputs.push(input); - return Promise.resolve({ destination: input.destination, status: 204 }); - }, - ...overrides, - }); - }, - }), - ); - return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; - } - - it("simlock alerts test prints the sent line, --json prints the output, a missing name prints the Expected line, simlock alerts and --help print the usage, and an unknown subcommand is a usage error", async () => { - const sent = await runAlerts(["alerts", "test", "ops-slack"]); - expect([sent.code, sent.stdout, sent.inputs]).toEqual([ - 0, - "Sent a test message to ops-slack: HTTP 204\n", - [{ destination: "ops-slack" }], - ]); - - const json = await runAlerts(["alerts", "test", "ops-slack", "--json"]); - expect([json.code, json.stdout]).toEqual([0, '{"destination":"ops-slack","status":204}\n']); - - for (const argv of [["alerts"], ["alerts", "--help"], ["alerts", "test", "--help"]]) { - const usage = await runAlerts(argv); - expect([usage.code, usage.stdout, usage.stderr, usage.connected]).toEqual([ - 0, - USAGE, - "", - false, - ]); - } - - for (const [argv, message] of [ - [["alerts", "test"], "Expected destination (run `simlock --help` for usage)"], - [["alerts", "foo"], "Unknown alerts command: foo (run `simlock --help` for usage)"], - ] as const) { - const refused = await runAlerts([...argv]); - expect(refused.code).toBe(2); - expect(refused.connected).toBe(false); - expect(JSON.parse(refused.stderr)).toEqual({ error: { code: "USAGE", message } }); - } - }); - - it("simlock alerts test exits 1 on ALERT_SEND_FAILED and 12 on UNKNOWN_ALERT_DESTINATION with the error line", async () => { - const failed = await runAlerts(["alerts", "test", "ops-slack"], { - testAlertDestination: () => - Promise.reject( - new SimlockError("ALERT_SEND_FAILED", "transport", "ops-slack answered HTTP 404", { - destination: "ops-slack", - status: 404, - }), - ), - }); - expect([failed.code, failed.stdout, errorLine(failed.stderr)]).toEqual([ - 1, - "", - { error: { code: "ALERT_SEND_FAILED", message: "ops-slack answered HTTP 404" } }, - ]); - - const unknown = await runAlerts(["alerts", "test", "nope"], { - testAlertDestination: () => - Promise.reject( - new SimlockError("UNKNOWN_ALERT_DESTINATION", "domain", "No alert destination named", { - destination: "nope", - }), - ), - }); - expect([unknown.code, errorLine(unknown.stderr)]).toEqual([ - 12, - { error: { code: "UNKNOWN_ALERT_DESTINATION", message: "No alert destination named" } }, - ]); - }); -}); - -describe("CLI: baseline save", () => { - const saved = (overrides: Record = {}) => ({ - baseline: { - mode: "full", - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android", - savedAt: "2026-11-02T13:40:00.000Z", - version: 1, - ...overrides, - }, - leaseId: "L1", - }); - - async function runSave(argv: string[], answer = saved()) { - const output = outputCapture(); - const inputs: unknown[] = []; - let connected = false; - const code = await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient({ - saveBaseline: (input) => { - inputs.push(input); - return Promise.resolve(answer as never); - }, - }); - }, - }), - ); - return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; - } - - it("simlock baseline save sends the name, lease and note and prints the baseline's version, spec and size with the lease it ended, and refuses a missing --lease as USAGE", async () => { - const android = await runSave( - ["baseline", "save", "checkout", "--lease", "L1", "--note", "app 4.2, test user"], - saved({ imageTag: "google_apis", sizeBytes: 4_800_000_000 }), - ); - expect([android.code, android.stdout, android.inputs]).toEqual([ - 0, - "Saved baseline checkout v1: android Pixel 9, Android 36 (google_apis), full, 4.8 GB. Lease L1 ended.\n", - [{ leaseId: "L1", name: "checkout", note: "app 4.2, test user" }], - ]); - - const ios = await runSave( - ["baseline", "save", "login", "--lease", "L5"], - saved({ - mode: "slim", - model: "iPhone 17", - name: "login", - osVersion: "27.0", - platform: "ios", - sizeBytes: 3_100_000_000, - }), - ); - expect(ios.stdout).toBe( - "Saved baseline login v1: ios iPhone 17, iOS 27.0, slim, 3.1 GB. Lease L1 ended.\n", - ); - expect(ios.inputs).toEqual([{ leaseId: "L5", name: "login" }]); - - const small = await runSave( - ["baseline", "save", "tiny", "--lease", "L1"], - saved({ sizeBytes: 250_000_000 }), - ); - expect(small.stdout).toContain("full, 250.0 MB. Lease L1 ended."); - const uncounted = await runSave(["baseline", "save", "tiny", "--lease", "L1"], saved()); - expect(uncounted.stdout).toBe( - "Saved baseline checkout v1: android Pixel 9, Android 36, full. Lease L1 ended.\n", - ); - - const missing = await runSave(["baseline", "save", "checkout"]); - expect([missing.code, missing.connected, missing.stdout]).toEqual([2, false, ""]); - expect(JSON.parse(missing.stderr)).toMatchObject({ error: { code: "USAGE" } }); - - const twoNames = await runSave(["baseline", "save", "checkout", "other", "--lease", "L1"]); - expect([twoNames.code, twoNames.connected, twoNames.inputs]).toEqual([2, false, []]); - }); -}); - -describe("CLI: baseline list", () => { - const checkout = { - imageTag: "google_apis", - mode: "full", - model: "Pixel 9", - name: "checkout", - osVersion: "36", - platform: "android", - savedAt: new Date(2026, 10, 2, 13, 40).toISOString(), - sizeBytes: 4_900_000_000, - version: 2, - } as const; - const login = { - mode: "slim", - model: "iPhone 17", - name: "login", - note: "signed in", - osVersion: "27.0", - platform: "ios", - savedAt: new Date(2026, 10, 1, 9, 12).toISOString(), - staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - version: 1, - } as const; - - async function runList(argv: string[], baselines: readonly (typeof checkout | typeof login)[]) { - const output = outputCapture(); - const code = await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => - fakeClient({ listBaselines: () => Promise.resolve({ baselines: [...baselines] }) }), - }), - ); - return { code, stderr: output.stderr, stdout: output.stdout }; - } - - it("simlock baseline list prints a table of each baseline with a - for an uncounted size and its stale reason under the row, --json the baselines, and says when there are none", async () => { - const table = await runList(["baseline", "list"], [checkout, login]); - expect([table.code, table.stdout]).toEqual([ - 0, - [ - "NAME VERSION PLATFORM MODEL OS MODE IMAGE SIZE SAVED NOTE", - "checkout 2 android Pixel 9 36 full google_apis 4.9 GB 2026-11-02 13:40", - "login 1 ios iPhone 17 27.0 slim - - 2026-11-01 09:12 signed in", - " stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - "", - ].join("\n"), - ]); - - const json = await runList(["baseline", "list", "--json"], [checkout, login]); - expect([json.code, JSON.parse(json.stdout)]).toEqual([0, { baselines: [checkout, login] }]); - - const none = await runList(["baseline", "list"], []); - expect([none.code, none.stdout]).toEqual([0, "No baselines on this worker.\n"]); - }); -}); - -describe("CLI: baselines through a gateway (#1208)", () => { - const GATEWAY = { - daemonHome: { dataDirectory: "/simlock/gateway", gatewayPresent: true, role: "gateway" }, - } as const; - const checkout = { - imageTag: "google_apis", - mode: "full", - model: "Pixel 9", - name: "checkout", - note: "app 4.2, test user", - osVersion: "36", - platform: "android", - savedAt: new Date(2026, 10, 2, 13, 10).toISOString(), - sizeBytes: 4_800_000_000, - version: 2, - workerId: "wrk_a", - } as const; - const smoke = { - mode: "slim", - model: "iPhone 17", - name: "smoke", - osVersion: "27.0", - platform: "ios", - savedAt: new Date(2026, 9, 30, 9, 2).toISOString(), - staleReason: "runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - version: 1, - workerId: "wrk_a", - } as const; - - it("simlock worker list prints a worker's Baselines line, its image tag in brackets and a stale one marked, none for a worker without any, and no line for one that has not read them", async () => { - const output = outputCapture(); - const worker = { connection: "connected", drained: false, lastSeenAt: 1 } as const; - await runCli( - ["worker", "list"], - output.environmentWith({ - clock: new FakeClock(1_000), - connectAdmin: async () => - fakeClient({ - listWorkers: () => - Promise.resolve({ - workers: [ - { ...worker, baselines: [checkout, smoke], id: "wrk_a" }, - { ...worker, baselines: [], id: "wrk_b" }, - { ...worker, health: "starting", id: "wrk_c" }, - ], - } as never), - }), - }), - ); - - expect(output.stdout.split("\n").filter((line) => /^\s+Baselines:/.test(line))).toEqual([ - " Baselines: checkout v2: android Pixel 9, Android 36 (google_apis), full; smoke v1: ios iPhone 17, iOS 27.0, slim, stale", - " Baselines: none", - ]); - }); - - it("simlock --gateway baseline list puts the worker first in each row, and says when no worker holds a baseline", async () => { - const run = async (baselines: readonly (typeof checkout | typeof smoke)[]) => { - const output = outputCapture(); - await runCli( - ["--gateway", "baseline", "list"], - output.environmentWith({ - ...GATEWAY, - connectAdmin: async () => - fakeClient({ listBaselines: () => Promise.resolve({ baselines: [...baselines] }) }), - }), - ); - return output.stdout; - }; - - expect(await run([checkout, smoke])).toBe( - [ - "WORKER NAME VERSION PLATFORM MODEL OS MODE IMAGE SIZE SAVED NOTE", - "wrk_a checkout 2 android Pixel 9 36 full google_apis 4.8 GB 2026-11-02 13:10 app 4.2, test user", - "wrk_a smoke 1 ios iPhone 17 27.0 slim - - 2026-10-30 09:02", - " stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed", - "", - ].join("\n"), - ); - expect(await run([])).toBe("No baselines on any worker.\n"); - }); - - it("simlock --gateway baseline remove --worker sends the worker, asks and prints ending in from , and without --worker asks nothing and sends the remove", async () => { - const run = async (argv: string[]) => { - const output = outputCapture(); - const sent: unknown[] = []; - const questions: string[] = []; - await runCli( - ["--gateway", ...argv], - output.environmentWith({ - ...GATEWAY, - confirm: async (question) => { - questions.push(question); - return true; - }, - connectAdmin: async () => - fakeClient({ - listBaselines: () => - Promise.resolve({ - baselines: [{ ...smoke, version: 3, workerId: "wrk_b" }, smoke], - }), - removeBaseline: (input) => { - sent.push(input); - return Promise.resolve({ deletedDevices: [], name: input.name, versions: [1] }); - }, - }), - }), - ); - return { questions, sent, stdout: output.stdout }; - }; - - expect(await run(["baseline", "remove", "smoke", "--worker", "wrk_a"])).toEqual({ - questions: ["Remove baseline smoke v1 from wrk_a? [y/N] "], - sent: [{ name: "smoke", workerId: "wrk_a" }], - stdout: "Removed baseline smoke v1 from wrk_a.\n", - }); - expect(await run(["baseline", "remove", "smoke"])).toMatchObject({ - questions: [], - sent: [{ name: "smoke" }], - }); - }); -}); - -describe("CLI: baseline remove", () => { - const login = { - mode: "slim", - model: "iPhone 17", - name: "login", - osVersion: "27.0", - platform: "ios", - savedAt: new Date(2026, 10, 1, 9, 12).toISOString(), - version: 1, - } as const; - - async function runRemove( - argv: string[], - options: { confirm?: ((question: string) => Promise) | undefined; deleted?: string[] }, - ) { - const output = outputCapture(); - const removed: unknown[] = []; - const code = await runCli( - argv, - output.environmentWith({ - confirm: options.confirm, - connectAdmin: async () => - fakeClient({ - listBaselines: () => Promise.resolve({ baselines: [login] }), - removeBaseline: (input) => { - removed.push(input); - return Promise.resolve({ - deletedDevices: options.deleted ?? [], - name: input.name, - versions: [1], - }); - }, - }), - }), - ); - return { code, removed, stderr: output.stderr, stdout: output.stdout }; - } - - it("simlock baseline remove asks Remove baseline v? first, removes with --yes or an answered yes and prints how many idle devices it deleted, and refuses with USAGE when there is no terminal or the answer is no", async () => { - const questions: string[] = []; - const asked = await runRemove(["baseline", "remove", "login"], { - confirm: async (question) => { - questions.push(question); - return true; - }, - }); - expect([asked.code, asked.stdout, asked.removed, questions]).toEqual([ - 0, - "Removed baseline login v1.\n", - [{ name: "login" }], - ["Remove baseline login v1? [y/N] "], - ]); - - const yes = await runRemove(["baseline", "remove", "login", "--yes"], { - confirm: async () => { - throw new Error("--yes must not ask"); - }, - deleted: ["dev_1"], - }); - expect(yes.stdout).toBe("Removed baseline login v1. Deleted 1 idle device made from it.\n"); - const many = await runRemove(["baseline", "remove", "login", "--yes"], { - deleted: ["dev_1", "dev_2"], - }); - expect(many.stdout).toBe("Removed baseline login v1. Deleted 2 idle devices made from it.\n"); - - for (const confirm of [undefined, async () => false]) { - const refused = await runRemove(["baseline", "remove", "login"], { confirm }); - expect([refused.code, refused.removed, refused.stdout]).toEqual([2, [], ""]); - expect(JSON.parse(refused.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: "baseline remove requires confirmation or --yes" }, - }); - } - }); -}); - -describe("CLI: warm forget", () => { - const USAGE = - "Usage: simlock warm forget [] [--mode ] [--image-tag ] [--json]\n"; - const pixel = { model: "Pixel", mode: "full", osVersion: "36", platform: "android" } as const; - - const checkoutKind = { ...pixel, baseline: "checkout" } as const; - - async function runWarm( - argv: string[], - forgotten: (typeof pixel | typeof checkoutKind)[] = [pixel], - ) { - const output = outputCapture(); - const inputs: unknown[] = []; - let connected = false; - const code = await runCli( - argv, - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient({ - forgetWarmKind: (input) => { - inputs.push(input); - return Promise.resolve({ forgotten }); - }, - }); - }, - }), - ); - return { code, connected, inputs, stderr: output.stderr, stdout: output.stdout }; - } - - it("simlock warm forget sends the platform, model, OS, mode and image tag it was given and prints each kind forgotten, or that none was learned", async () => { - const forgot = await runWarm(["warm", "forget", "android", "Pixel"]); - expect([forgot.code, forgot.stdout, forgot.inputs]).toEqual([ - 0, - "Forgot android Pixel 36 (full).\nIts idle devices are no longer kept warm: the idle rules shut them down and delete them, unless a configured warmPool.targets entry keeps them.\n", - [{ model: "Pixel", platform: "android" }], - ]); - - const narrowed = await runWarm( - ["warm", "forget", "android", "Pixel", "36", "--mode", "full", "--image-tag", "google_apis"], - [], - ); - expect(narrowed.stdout).toBe( - "The warm pool has learned no kind matching android Pixel 36 (full, google_apis).\n", - ); - expect(narrowed.inputs).toEqual([ - { - imageTag: "google_apis", - mode: "full", - model: "Pixel", - osVersion: "36", - platform: "android", - }, - ]); - - const named = await runWarm(["warm", "forget", "android", "Pixel"], [checkoutKind]); - expect(named.stdout).toContain("Forgot android Pixel 36 (full, baseline checkout).\n"); - - const none = await runWarm(["warm", "forget", "ios", "iPhone 16"], []); - expect([none.code, none.stdout]).toEqual([ - 0, - "The warm pool has learned no kind matching ios iPhone 16.\n", - ]); - - const json = await runWarm(["warm", "forget", "android", "Pixel", "--json"]); - expect([json.code, JSON.parse(json.stdout)]).toEqual([0, { forgotten: [pixel] }]); - }); - - it("simlock warm and --help print the usage without connecting; a bad platform, a missing model, extra words, a bad --mode and an unknown subcommand are usage errors", async () => { - for (const argv of [["warm"], ["warm", "--help"], ["warm", "forget", "--help"]]) { - const usage = await runWarm(argv); - expect([usage.code, usage.stdout, usage.stderr, usage.connected]).toEqual([ - 0, - USAGE, - "", - false, - ]); - } - - for (const [argv, message] of [ - [["warm", "forget", "tvos", "Pixel"], "warm forget needs a platform first: ios or android"], - [["warm", "forget", "android"], "warm forget takes and an optional "], - [ - ["warm", "forget", "android", "Pixel", "36", "extra"], - "warm forget takes and an optional ", - ], - [["warm", "forget", "android", "Pixel", "--mode", "tiny"], "--mode is slim or full"], - [["warm", "boot"], "Unknown warm command: boot"], - ] as const) { - const refused = await runWarm([...argv]); - expect([refused.code, refused.connected]).toEqual([2, false]); - expect(JSON.parse(refused.stderr)).toEqual({ - error: { code: "USAGE", message: `${message} (run \`simlock --help\` for usage)` }, - }); - } - }); -}); - -describe("CLI: status renders the fleet a gateway reports (ADR 0005 §20)", () => { - it("names the mode, each worker, and which worker a device and a lease live on", async () => { - const output = outputCapture(); - const status = { - capacity: { - android: { - atRamBudget: false, - limit: 0, - maxRunning: 0, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }, - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 1, warm: 0 }, - ios: { - atRamBudget: false, - limit: 2, - maxRunning: 2, - overLimit: false, - reserved: 0, - running: 1, - used: 1, - warm: 0, - }, - }, - daemon: { - health: "running" as const, - mode: "gateway" as const, - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] }, - devices: [ - { - physical: false, - id: "dev_1", - spec: { - physical: false, - model: "iPhone 17", - osVersion: "26.0", - platform: "ios" as const, - }, - mode: "full" as const, - servesDefaultMode: true, - state: "leased" as const, - workerId: "wrk_1", - }, - ], - leases: [ - { - deviceId: "dev_1", - grantedAt: 1, - id: "lease_1", - lastRenewedAt: 1, - idChosenByRequester: false, - ownerId: "agent-1", - requesterId: "agent-1", - ttlDeadline: 2, - ttlMs: 1, - workerId: "wrk_1", - }, - ], - queueDepth: 0, - workers: [ - { - capacity: undefined, - catalog: [], - connection: "connected" as const, - devices: [], - drained: true, - id: "wrk_1", - lastSeenAt: 1, - leases: [], - }, - ], - }; - const environment = output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }); - - await runCli(["status"], environment); - - expect(output.stdout).toContain("Daemon: running (gateway)"); - // why: A gateway runs no drivers: its host line names the machine and nothing after it. - expect(output.stdout).toContain("Host: Linux 6.8.0 x64\n"); - expect(output.stdout).toContain("wrk_1: connected, drained"); - expect(output.stdout).toContain( - "Device dev_1 on wrk_1: leased, mode full, serves default mode: yes", - ); - expect(output.stdout).toContain("Lease lease_1: agent-1 on wrk_1"); - }); - - describe("the warm pool block", () => { - const warmPool = { - enabled: true, - reserveRunning: { android: 0, ios: 1 }, - targets: [ - { - booting: 1, - count: 2, - mode: "full" as const, - model: "iPhone 17", - origin: "configured" as const, - osVersion: "26.0", - platform: "ios" as const, - ready: 1, - }, - { - booting: 0, - count: 1, - mode: "full" as const, - model: "iPhone 17", - origin: "learned" as const, - osVersion: "27.0", - platform: "ios" as const, - ready: 0, - short: "runtime-missing" as const, - }, - { - booting: 0, - count: 1, - imageTag: "google_apis_playstore", - mode: "full" as const, - model: "Pixel 8", - origin: "learned" as const, - osVersion: "35", - platform: "android" as const, - ready: 1, - }, - { - baseline: "checkout", - booting: 0, - count: 1, - mode: "full" as const, - model: "iPhone 17", - origin: "learned" as const, - osVersion: "26.0", - platform: "ios" as const, - ready: 1, - }, - ], - }; - const statusWith = (block: StatusGetOutput["warmPool"]) => - fakeClient({ getStatus: () => Promise.resolve({ ...EMPTY_STATUS, warmPool: block }) }); - - it("simlock status prints the origin word on every warm pool line, learned lines included, the tag in the name, and a learned line's short:", async () => { - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ connectAdmin: async () => statusWith(warmPool) }), - ); - - expect(output.stdout).toContain( - "warm pool: enabled, reserve ios 1 android 0\n" + - " iPhone 17 / 26.0 / full configured wanted 2 ready 1 booting 1\n" + - " iPhone 17 / 27.0 / full learned wanted 1 ready 0 booting 0 short: runtime-missing\n" + - " Pixel 8 / 35 / full / google_apis_playstore learned wanted 1 ready 1 booting 0\n" + - " iPhone 17 / 26.0 / full / baseline checkout learned wanted 1 ready 1 booting 0\n", - ); - }); - - it("simlock status leaves the runtime out of a target that names none", async () => { - const output = outputCapture(); - const unversioned = { - enabled: true, - reserveRunning: { android: 0, ios: 0 }, - targets: [ - { - booting: 0, - count: 1, - mode: "slim" as const, - model: "iPhone 17", - origin: "configured" as const, - platform: "ios" as const, - ready: 0, - }, - ], - }; - - await runCli( - ["status"], - output.environmentWith({ connectAdmin: async () => statusWith(unversioned) }), - ); - - expect(output.stdout).toContain( - "warm pool: enabled, reserve ios 0 android 0\n" + - " iPhone 17 / slim configured wanted 1 ready 0 booting 0\n", - ); - }); - - it("simlock status prints a pool that is off as disabled, with every target short disabled", async () => { - const output = outputCapture(); - const off = { - enabled: false, - reserveRunning: { android: 0, ios: 0 }, - targets: warmPool.targets.slice(0, 1).map((target) => ({ - ...target, - booting: 0, - ready: 0, - short: "disabled" as const, - })), - }; - - await runCli( - ["status"], - output.environmentWith({ connectAdmin: async () => statusWith(off) }), - ); - - expect(output.stdout).toContain( - "warm pool: disabled, reserve ios 0 android 0\n" + - " iPhone 17 / 26.0 / full configured wanted 2 ready 0 booting 0 short: disabled\n", - ); - }); - - it.each([ - { - name: "worker whose answer has no block", - mode: "worker" as const, - warmPool: undefined, - }, - { - name: "gateway whose own block is empty", - mode: "gateway" as const, - warmPool: { enabled: false, reserveRunning: { android: 0, ios: 0 }, targets: [] }, - }, - ])("simlock status prints no warm pool line for a $name", async ({ mode, warmPool }) => { - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => - Promise.resolve({ - ...EMPTY_STATUS, - daemon: { - health: "running" as const, - mode, - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - warmPool, - }), - }), - }), - ); - - expect(output.stdout).toBe( - `Daemon: running (${mode})\n` + - "Simlock: 0.0.0-test\n" + - `CLI: ${PACKAGE_VERSION}\n` + - "Host: macOS 15.5 arm64\n" + - "Running global: 0 + 0 reserved/2, warm 0\n" + - "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + - "Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + - "Queue depth: 0\n", - ); - }); - - it.each(["worker", "gateway"] as const)( - "simlock status of a %s with queueDepth 2 and physicalQueueDepth 1 prints Queue depth: 2", - async (mode) => { - const output = outputCapture(); - const answer = { - ...EMPTY_STATUS, - daemon: { ...EMPTY_STATUS.daemon, mode }, - physicalQueueDepth: 1, - queueDepth: 2, - } as unknown as StatusGetOutput; - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(answer) }), - }), - ); - - expect(output.stdout).toBe( - `Daemon: running (${mode})\n` + - "Simlock: 0.0.0-test\n" + - `CLI: ${PACKAGE_VERSION}\n` + - "Host: macOS 15.5 arm64\n" + - "Running global: 0 + 0 reserved/2, warm 0\n" + - "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + - "Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0\n" + - "Queue depth: 2\n", - ); - }, - ); - - it("simlock status --json carries the block as the daemon sent it", async () => { - const output = outputCapture(); - - await runCli( - ["status", "--json"], - output.environmentWith({ connectAdmin: async () => statusWith(warmPool) }), - ); - - expect(JSON.parse(output.stdout).warmPool).toEqual(warmPool); - expect(JSON.parse(output.stdout).cliVersion).toBe(PACKAGE_VERSION); - }); - - it("simlock doctor prints one stderr line per unreachable warm pool target, keeps the JSON report on stdout, and exits 0", async () => { - const output = outputCapture(); - const findings = [ - { - kind: "warm-pool-target-unreachable" as const, - message: "iOS 27.0 is not installed", - platform: "ios" as const, - reason: "runtime-missing" as const, - remedy: "run simlock component install ios 27.0", - target: "iPhone 17 / 27.0 / full", - }, - { - kind: "warm-pool-target-unreachable" as const, - message: "the targets want 4 running devices, and the running limit leaves room for 3", - reason: "over-limit" as const, - remedy: "lower the counts of the warmPool.targets, or raise the running limit", - target: "all targets", - }, - { deviceId: "d-1", kind: "expired-live-lease" as const, leaseId: "l-1" }, - ]; - - await expect( - runCli( - ["doctor"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ runDoctor: () => Promise.resolve({ findings, sources: [] }) }), - }), - ), - ).resolves.toBe(0); - - expect( - output.stderr - .split("\n") - .filter((line) => line.includes("warm-pool-target-unreachable") || line.includes("d-1")), - ).toEqual([ - "warm-pool-target-unreachable iPhone 17 / 27.0 / full: iOS 27.0 is not installed; run simlock component install ios 27.0", - "warm-pool-target-unreachable all targets: the targets want 4 running devices, and the running limit leaves room for 3; lower the counts of the warmPool.targets, or raise the running limit", - ]); - expect(output.stderr).not.toContain("undefined"); - expect(JSON.parse(output.stdout)).toEqual({ findings, sources: [] }); - }); - }); - - describe("against a starting daemon", () => { - const STARTING: StatusGetOutput = { - daemon: { - health: "starting", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, - }; - - it("simlock status prints the starting line and no device, lease or capacity line", async () => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(STARTING) }), - }), - ); - - expect(output.stdout).toBe( - "Daemon: starting (worker)\n" + - "Simlock: 0.0.0-test\n" + - `CLI: ${PACKAGE_VERSION}\n` + - "Host: macOS 15.5 arm64\n" + - "Devices, leases and capacity appear once startup finishes.\n", - ); - }); - - it.each(["capacity", "devices", "leases", "queueDepth"] as const)( - "simlock status prints the starting line for an answer with no %s, though the rest is there", - async (missing) => { - const output = outputCapture(); - const { [missing]: _left, ...answer } = EMPTY_STATUS; - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ getStatus: () => Promise.resolve(answer as StatusGetOutput) }), - }), - ); - - expect(output.stdout).toContain( - "Devices, leases and capacity appear once startup finishes.\n", - ); - expect(output.stdout).not.toContain("Queue depth"); - }, - ); - - it("simlock status --json prints the answer with those fields absent", async () => { - const output = outputCapture(); - await runCli( - ["status", "--json"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(STARTING) }), - }), - ); - - expect(JSON.parse(output.stdout)).toEqual({ ...STARTING, cliVersion: PACKAGE_VERSION }); - }); - }); - - it("prints the host line", async () => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => - Promise.resolve({ - ...EMPTY_STATUS, - host: { - arch: "arm64", - os: "macOS", - osVersion: "15.5", - tools: [ - { build: "16F6", name: "xcode", platform: "ios" as const, version: "16.4" }, - ], - }, - }), - }), - }); - - await runCli(["status"], environment); - - expect(output.stdout).toContain("Host: macOS 15.5 arm64; xcode 16.4 (16F6)\n"); - }); - - describe("the Load line", () => { - const GIB = 2 ** 30; - const MIB = 2 ** 20; - const LOAD = { - ageMs: 4_000, - cpu: { busyPercent: 34.2, cores: 10, limited: false }, - disk: { freeBytes: 120_688_581_018, totalBytes: 530_857_957_786 }, - memory: { - availableBytes: 10_522_669_875, - limited: false, - pressure: "normal" as const, - swapUsedBytes: 1_288_490_189, - totalBytes: 32 * GIB, - }, - }; - const FULL_LINE = - "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago\n"; - const STARTING = { - daemon: { - health: "starting" as const, - mode: "worker" as const, - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, - }; - - const print = async (status: StatusGetOutput): Promise => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - return output.stdout; - }; - const loadLine = async (load: StatusGetOutput["load"]): Promise => { - const stdout = await print({ ...EMPTY_STATUS, load }); - return stdout.split("\n").find((line) => line.startsWith("Load:")) ?? ""; - }; - - it("simlock status prints the Load line after the Host line, on a starting daemon and a running one", async () => { - expect(await print({ ...STARTING, load: LOAD })).toBe( - "Daemon: starting (worker)\n" + - "Simlock: 0.0.0-test\n" + - `CLI: ${PACKAGE_VERSION}\n` + - "Host: macOS 15.5 arm64\n" + - FULL_LINE + - "Devices, leases and capacity appear once startup finishes.\n", - ); - - expect(await print({ ...EMPTY_STATUS, load: LOAD })).toContain( - `Host: macOS 15.5 arm64\n${FULL_LINE}`, - ); - }); - - it("simlock status prints CPU unavailable, memory unavailable or disk unavailable for an absent part, and no swap or pressure part when memory is absent", async () => { - const { cpu: _cpu, ...withoutCpu } = LOAD; - const { memory: _memory, ...withoutMemory } = LOAD; - const { disk: _disk, ...withoutDisk } = LOAD; - - expect(await loadLine(withoutCpu)).toBe( - "Load: CPU unavailable · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago", - ); - expect(await loadLine(withoutMemory)).toBe( - "Load: CPU 34% busy (10 cores) · memory unavailable · disk 112.4 GiB of 494.4 GiB free · read 4s ago", - ); - expect(await loadLine(withoutDisk)).toBe( - "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · pressure normal · disk unavailable · read 4s ago", - ); - }); - - it("simlock status prints no pressure part when the reading has none, and limit on a limited CPU or memory", async () => { - const { pressure: _pressure, ...noPressure } = LOAD.memory; - - expect(await loadLine({ ...LOAD, memory: noPressure })).toBe( - "Load: CPU 34% busy (10 cores) · memory 9.8 GiB of 32.0 GiB available · swap 1.2 GiB used · disk 112.4 GiB of 494.4 GiB free · read 4s ago", - ); - expect( - await loadLine({ - ...LOAD, - cpu: { busyPercent: 34.2, cores: 2, limited: true }, - memory: { ...LOAD.memory, availableBytes: 3 * GIB, limited: true, totalBytes: 8 * GIB }, - }), - ).toBe( - "Load: CPU 34% busy (2 cores, limit) · memory 3.0 GiB of 8.0 GiB available (limit) · swap 1.2 GiB used · pressure normal · disk 112.4 GiB of 494.4 GiB free · read 4s ago", - ); - }); - - it.each([ - ["swap 0 B", { memory: { ...LOAD.memory, swapUsedBytes: 0 } }, "swap 0 B used"], - [ - "swap 300 MiB", - { memory: { ...LOAD.memory, swapUsedBytes: 300 * MIB } }, - "swap 300 MiB used", - ], - [ - "swap 1023.6 MiB", - { memory: { ...LOAD.memory, swapUsedBytes: 1_073_322_394 } }, - "swap 1.0 GiB used", - ], - ["busy 34.5", { cpu: { ...LOAD.cpu, busyPercent: 34.5 } }, "CPU 35% busy"], - ["cores 1", { cpu: { ...LOAD.cpu, cores: 1 } }, "(1 core)"], - ["cores 0.5", { cpu: { ...LOAD.cpu, cores: 0.5 } }, "(0.5 cores)"], - ["cores 10", { cpu: { ...LOAD.cpu, cores: 10 } }, "(10 cores)"], - ["cores 0.01", { cpu: { ...LOAD.cpu, cores: 0.01 } }, "(0.01 cores)"], - ["cores 0.001", { cpu: { ...LOAD.cpu, cores: 0.001 } }, "(<0.01 cores)"], - ["ageMs 4001", { ageMs: 4_001 }, "read 5s ago"], - ["ageMs 0", { ageMs: 0 }, "read 0s ago"], - ])("simlock status formats %s as in the inputs table", async (_row, change, fragment) => { - expect(await loadLine(LOAD)).toContain("CPU 34% busy (10 cores)"); - - expect(await loadLine({ ...LOAD, ...change })).toContain(fragment); - }); - - it("simlock status prints no Load line when the answer has no load block", async () => { - expect(await print({ ...EMPTY_STATUS, load: LOAD })).toContain("Load: "); - - expect(await print(EMPTY_STATUS)).not.toContain("Load:"); - expect(await print(STARTING)).not.toContain("Load:"); - }); - - it("simlock status prints the Memory pressure line with the step, when it began and why, and none when there is no step", async () => { - const line = async (memoryPressure: unknown) => - ( - await print({ - ...EMPTY_STATUS, - capacity: { ...EMPTY_STATUS.capacity, memoryPressure }, - load: LOAD, - } as StatusGetOutput) - ) - .split("\n") - .find((row) => row.startsWith("Memory pressure:")); - const since = new Date(2026, 10, 6, 9, 12).getTime(); - const warn = { availablePercent: 34, level: "warn", trigger: "level" }; - const floor = (availablePercent: number, extra = {}) => - line({ - cause: { availablePercent, floorPercent: 10, trigger: "floor", ...extra }, - since, - step: "new-devices-paused", - }); - - expect(await line({ cause: warn, since, step: "warming-paused" })).toBe( - "Memory pressure: warming paused since 09:12 (pressure warn)", - ); - const floored = "Memory pressure: new devices paused since 09:12 (8% available, floor 10%)"; - expect(await floor(8)).toBe(floored); - expect(await floor(8, { level: "warn" })).toBe(floored); - expect(await floor(9)).toBe(floored.replace("8%", "9%")); - expect(await line({ step: "none" })).toBeUndefined(); - expect(await line(undefined)).toBeUndefined(); - - const stdout = await print({ - ...EMPTY_STATUS, - capacity: { - ...EMPTY_STATUS.capacity, - memoryPressure: { cause: warn, since, step: "warming-paused" }, - }, - load: LOAD, - } as StatusGetOutput); - expect(stdout).toContain( - `${FULL_LINE}Memory pressure: warming paused since 09:12 (pressure warn)\nRunning global`, - ); - }); - }); - - it("simlock status prints the console address when HTTP is enabled", async () => { - const print = async (status: StatusGetOutput) => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - return output.stdout; - }; - const daemon = { ...EMPTY_STATUS.daemon, consoleUrl: "http://127.0.0.1:4700/" }; - - expect(await print({ ...EMPTY_STATUS, daemon })).toContain( - `Daemon: running (worker)\nSimlock: 0.0.0-test\nCLI: ${PACKAGE_VERSION}\nConsole: http://127.0.0.1:4700/\nHost: `, - ); - expect(await print(EMPTY_STATUS)).not.toContain("Console:"); - }); - - it("simlock daemon start prints the console address when HTTP is enabled", async () => { - const start = async (status: StatusGetOutput) => { - const output = outputCapture(); - await runCli( - ["daemon", "start"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - return output.stdout; - }; - const daemon = { ...EMPTY_STATUS.daemon, consoleUrl: "http://127.0.0.1:4700/" }; - - expect(await start({ ...EMPTY_STATUS, daemon })).toBe( - "Daemon running\nConsole: http://127.0.0.1:4700/\n", - ); - expect(await start(EMPTY_STATUS)).toBe("Daemon running\n"); - }); - - describe("the history lines of simlock status", () => { - const NOT_WRITTEN = "History: the event history is not being written; see daemon.log."; - const SET_ASIDE_AT = Date.UTC(2026, 9, 16, 9, 12, 30); - const pad = (value: number): string => String(value).padStart(2, "0"); - const local = new Date(SET_ASIDE_AT); - const setAside = `History: the event history was damaged and set aside at ${local.getFullYear()}-${pad(local.getMonth() + 1)}-${pad(local.getDate())} ${pad(local.getHours())}:${pad(local.getMinutes())}; a fresh one was started.`; - - const statusLines = async (history: StatusGetOutput["history"]): Promise => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => - Promise.resolve({ - ...EMPTY_STATUS, - ...(history === undefined ? {} : { history }), - } as StatusGetOutput), - }), - }), - ); - return output.stdout.trimEnd().split("\n"); - }; - - it("prints the not-written line when writing is false, the set-aside line with the local time when setAsideAt is set, both when both hold, and neither otherwise", async () => { - const healthy = await statusLines({ opened: true, writing: true }); - const notWritten = await statusLines({ opened: false, writing: false }); - const aside = await statusLines({ opened: true, setAsideAt: SET_ASIDE_AT, writing: true }); - const both = await statusLines({ opened: true, setAsideAt: SET_ASIDE_AT, writing: false }); - - expect(healthy.filter((line) => line.startsWith("History:"))).toEqual([]); - expect(notWritten.filter((line) => line.startsWith("History:"))).toEqual([NOT_WRITTEN]); - expect(aside.filter((line) => line.startsWith("History:"))).toEqual([setAside]); - expect(both.filter((line) => line.startsWith("History:")).sort()).toEqual( - [NOT_WRITTEN, setAside].sort(), - ); - }); - - it("prints the rebuilt line with the local time when rebuiltAt is set and the could-not-rebuild line when rebuildFailed is set, and neither otherwise", async () => { - const rebuiltLine = `History: figures were rebuilt from the event history at ${local.getFullYear()}-${pad(local.getMonth() + 1)}-${pad(local.getDate())} ${pad(local.getHours())}:${pad(local.getMinutes())}.`; - const failedLine = "History: figures could not be rebuilt; see daemon.log."; - const plain = await statusLines({ opened: true, writing: true }); - const rebuilt = await statusLines({ opened: true, rebuiltAt: SET_ASIDE_AT, writing: true }); - const failed = await statusLines({ opened: true, rebuildFailed: true, writing: true }); - - expect(plain.filter((line) => line.startsWith("History:"))).toEqual([]); - expect(rebuilt.filter((line) => line.startsWith("History:"))).toEqual([rebuiltLine]); - expect(failed.filter((line) => line.startsWith("History:"))).toEqual([failedLine]); - }); - }); - - it("prints an install line while one runs and none after", async () => { - const install = { - component: "26.4", - platform: "ios" as const, - since: 1_000, - state: "downloading" as const, - waiters: 1, - }; - const print = async (installs: StatusGetOutput["installs"]) => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - clock: new FakeClock(43_000), - connectAdmin: async () => - fakeClient({ getStatus: () => Promise.resolve({ ...EMPTY_STATUS, installs }) }), - }), - ); - return output.stdout; - }; - - expect(await print([install])).toContain("Install ios 26.4: downloading for 42s, 1 waiter\n"); - expect(await print([])).not.toContain("Install "); - }); - - it("prints an install whose start is ahead of this machine's clock as 0s old, not a negative age", async () => { - const output = outputCapture(); - const install = { - component: "26.4", - platform: "ios" as const, - since: 5_000, - state: "waiting" as const, - waiters: 1, - }; - await runCli( - ["status"], - output.environmentWith({ - clock: new FakeClock(1_000), - connectAdmin: async () => - fakeClient({ - getStatus: () => Promise.resolve({ ...EMPTY_STATUS, installs: [install] }), - }), - }), - ); - - expect(output.stdout).toContain("Install ios 26.4: waiting for 0s, 1 waiter\n"); - }); - - it("prints a gateway's installs once each, naming the worker, not again under the worker", async () => { - const output = outputCapture(); - const install = { - component: "35", - platform: "android" as const, - since: 0, - state: "waiting" as const, - waiters: 3, - }; - const status: StatusGetOutput = { - ...EMPTY_STATUS, - daemon: { - health: "running", - mode: "gateway", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - installs: [{ ...install, workerId: "wrk_1" }], - workers: [ - { - catalog: [], - connection: "connected", - devices: [], - drained: false, - id: "wrk_1", - installs: [install], - lastSeenAt: 1, - leases: [], - }, - ], - }; - const environment = output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }); - - await runCli(["status"], environment); - - expect(output.stdout.match(/Install /g)).toHaveLength(1); - expect(output.stdout).toContain("Install android 35 on wrk_1: waiting for 0s, 3 waiters\n"); - }); - - it("prints each device's mode and whether it serves a request naming no mode", async () => { - const output = outputCapture(); - const spec = { - physical: false, - model: "iPhone 17", - osVersion: "26.0", - platform: "ios" as const, - }; - const status: StatusGetOutput = { - ...EMPTY_STATUS, - devices: [ - { - physical: false, - id: "dev_slim", - mode: "slim", - servesDefaultMode: false, - spec, - state: "ready", - }, - { - physical: false, - id: "dev_full", - mode: "full", - servesDefaultMode: true, - spec, - state: "leased", - }, - { - physical: false, - id: "dev_copy", - mode: "full", - servesDefaultMode: true, - spec: { ...spec, baseline: { name: "checkout", version: 2 } }, - state: "leased", - }, - ], - }; - const environment = output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }); - - await runCli(["status"], environment); - - expect(output.stdout).toContain("Device dev_slim: ready, mode slim, serves default mode: no\n"); - expect(output.stdout).toContain( - "Device dev_full: leased, mode full, serves default mode: yes\n", - ); - expect(output.stdout).toContain( - "Device dev_copy: leased, mode full, serves default mode: yes, baseline checkout v2\n", - ); - }); - - it("prints the RAM budget, marked over limit when over, and no RAM line when the daemon reports none", async () => { - type RamBudget = NonNullable["ramBudget"]>; - const statusWith = (ramBudget?: RamBudget) => { - const status: StatusGetOutput = { - ...EMPTY_STATUS, - capacity: { ...EMPTY_STATUS.capacity, ...(ramBudget === undefined ? {} : { ramBudget }) }, - }; - return status; - }; - const print = async (status: StatusGetOutput) => { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - return output.stdout; - }; - const gib = 1024 ** 3; - - expect( - await print(statusWith({ limitBytes: 12 * gib, overLimit: false, usedBytes: 4.5 * gib })), - ).toContain("RAM budget: 4.50 GiB/12.00 GiB used\n"); - expect( - await print(statusWith({ limitBytes: 12 * gib, overLimit: true, usedBytes: 13 * gib })), - ).toContain("RAM budget: 13.00 GiB/12.00 GiB used (over limit)\n"); - expect(await print(statusWith())).not.toContain("RAM"); - }); - - it("prints, per platform, whether the worker is at its RAM budget, after any over-limit mark", async () => { - const status: StatusGetOutput = { - ...EMPTY_STATUS, - capacity: { - ...EMPTY_STATUS.capacity, - android: { ...EMPTY_STATUS.capacity.android, atRamBudget: false }, - ios: { ...EMPTY_STATUS.capacity.ios, atRamBudget: true, overLimit: true }, - }, - }; - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - const lines = output.stdout.split("\n"); - expect(lines).toContain( - "Capacity ios: managed 0/1, running 0 + 0 reserved/1, warm 0 (over limit) (at RAM budget)", - ); - expect(lines).toContain("Capacity android: managed 0/1, running 0 + 0 reserved/1, warm 0"); - }); - - it("simlock status prints a physical device as physical and its class, with today's markers, and a simulator's line as before", async () => { - const status: StatusGetOutput = { - ...EMPTY_STATUS, - devices: [ - { - class: "phone", - id: "dev_p1", - physical: true, - servesDefaultMode: false, - spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, - state: "ready", - }, - { - class: "tablet", - id: "dev_p2", - physical: true, - servesDefaultMode: false, - spec: { model: "iPad Pro", osVersion: "18.4", physical: true, platform: "ios" }, - state: "quarantined", - quarantineAttempts: 1, - workerId: "wrk_1", - }, - { - id: "dev_v1", - mode: "full", - physical: false, - servesDefaultMode: true, - spec: { model: "iPhone 16", osVersion: "18.4", physical: false, platform: "ios" }, - state: "ready", - }, - ], - }; - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - const lines = output.stdout.split("\n"); - expect(lines).toContain("Device dev_p1: ready, physical phone"); - expect(lines).toContain("Device dev_p2 on wrk_1: quarantined, physical tablet (purge retry 1)"); - expect(lines).toContain("Device dev_v1: ready, mode full, serves default mode: yes"); - }); - - it("status ends a quarantined device's marker with ; and the cause for each kind, and prints today's marker when there is no cause", async () => { - const quarantined = ( - id: string, - quarantineCause?: Record, - ): Record => ({ - id, - mode: "full", - physical: false, - quarantineAttempts: 1, - quarantineNextRetryAt: 1_760_619_150_000, - servesDefaultMode: true, - spec: { model: "iPhone 16", osVersion: "18.4", physical: false, platform: "ios" }, - state: "quarantined", - ...(quarantineCause === undefined ? {} : { quarantineCause }), - }); - const status = { - ...EMPTY_STATUS, - devices: [ - quarantined("dev_purge", { - kind: "purge-failed", - message: "DriverCrashError: simctl erase exited 164, then stalled", - }), - quarantined("dev_shutdown", { - kind: "shutdown-failed", - message: "Error: shutdown exploded", - }), - quarantined("dev_delete", { kind: "delete-failed", message: "Error: delete exploded" }), - quarantined("dev_prov", { kind: "stalled", stalledState: "provisioning" }), - quarantined("dev_recl", { kind: "stalled", stalledState: "reclaiming" }), - quarantined("dev_none"), - ], - } as unknown as StatusGetOutput; - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - const lines = output.stdout.split("\n"); - const head = - "quarantined, mode full, serves default mode: yes (purge retry 1, next retry at 1760619150000"; - expect(lines).toContain( - `Device dev_purge: ${head}; purge failed: DriverCrashError: simctl erase exited 164, then stalled)`, - ); - expect(lines).toContain( - `Device dev_shutdown: ${head}; shutdown failed: Error: shutdown exploded)`, - ); - expect(lines).toContain(`Device dev_delete: ${head}; delete failed: Error: delete exploded)`); - expect(lines).toContain(`Device dev_prov: ${head}; stalled while provisioning)`); - expect(lines).toContain(`Device dev_recl: ${head}; stalled while reclaiming)`); - expect(lines).toContain(`Device dev_none: ${head})`); - }); - - it("status prints absent () as the state of an absent device", async () => { - const physical = (id: string, state: string, absentReason?: string) => ({ - class: "phone", - id, - physical: true, - servesDefaultMode: false, - spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, - state, - ...(absentReason === undefined ? {} : { absentReason }), - }); - const status = { - ...EMPTY_STATUS, - devices: [ - physical("dev_ready", "ready"), - physical("dev_locked", "absent", "locked"), - physical("dev_untrusted", "absent", "untrusted"), - physical("dev_gone", "absent", "not-present"), - ], - } as unknown as StatusGetOutput; - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - const lines = output.stdout.split("\n"); - expect(lines).toContain("Device dev_ready: ready, physical phone"); - expect(lines).toContain("Device dev_locked: absent (locked), physical phone"); - expect(lines).toContain("Device dev_untrusted: absent (untrusted), physical phone"); - expect(lines).toContain("Device dev_gone: absent (not-present), physical phone"); - }); - - it("status prints a bare state for an absent device with no reason and for a device that is not absent", async () => { - const physical = (id: string, state: string, absentReason?: string) => ({ - class: "phone", - id, - physical: true, - servesDefaultMode: false, - spec: { model: "iPhone 15 Pro", osVersion: "18.4", physical: true, platform: "ios" }, - state, - ...(absentReason === undefined ? {} : { absentReason }), - }); - const status = { - ...EMPTY_STATUS, - devices: [physical("dev_bare", "absent"), physical("dev_stray", "ready", "locked")], - } as unknown as StatusGetOutput; - const output = outputCapture(); - - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - const lines = output.stdout.split("\n"); - expect(lines).toContain("Device dev_bare: absent, physical phone"); - expect(lines).toContain("Device dev_stray: ready, physical phone"); - }); - - it("simlock status marks a stalled device", async () => { - const device = (id: string, extra: object) => ({ - physical: false, - id, - mode: "full" as const, - servesDefaultMode: true, - spec: { physical: false, model: "iPhone 16", osVersion: "18.4", platform: "ios" as const }, - state: "provisioning" as const, - ...extra, - }); - const status: StatusGetOutput = { - ...EMPTY_STATUS, - devices: [ - device("dev_stuck", { stalled: true, transitionAgeMs: 120_000 }), - device("dev_fresh", { transitionAgeMs: 2_000 }), - ], - }; - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status) }), - }), - ); - - expect(output.stdout).toContain( - "Device dev_stuck: provisioning, mode full, serves default mode: yes (mid-transition 120000ms, stalled)\n", - ); - expect(output.stdout).toContain( - "Device dev_fresh: provisioning, mode full, serves default mode: yes (mid-transition 2000ms)\n", - ); - }); -}); - -describe("CLI: lease pushes and exit codes (own logic, not the dispatcher's)", () => { - it("ignores a lease-lost push for a lease this connection does not hold (own-lease-id filter)", async () => { - const output = outputCapture(); - let leaseLostListener: - | ((push: { leaseId: string; deviceId: string; reason: string }) => void) - | undefined; - let released = false; - const signals = new EventEmitter(); - const client = fakeClient({ - onLeaseLost: (listener) => { - leaseLostListener = listener; - return () => {}; - }, - releaseLease: () => { - released = true; - return Promise.resolve({ leaseId: "lse_1" }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - leaseLostListener?.({ leaseId: "some-other-lease", deviceId: "dev_2", reason: "ttl-backstop" }); - await settle(); - signals.emit("SIGINT"); - const exitCode = await runPromise; - expect(exitCode).toBe(0); - expect(released).toBe(true); - expect(output.stderr).not.toContain("lease-lost"); - }); - - it("writes {push:...} stderr lines for progress, device-unhealthy, and device-recovered as contract values", async () => { - const output = outputCapture(); - let unhealthyListener: ((push: DeviceUnhealthyPush) => void) | undefined; - let recoveredListener: ((push: DeviceRecoveredPush) => void) | undefined; - const client = fakeClient({ - onDeviceUnhealthy: (listener) => { - unhealthyListener = listener; - return () => {}; - }, - onDeviceRecovered: (listener) => { - recoveredListener = listener; - return () => {}; - }, - requestLease: (_input, options) => { - options?.onProgress?.({ stage: "provisioning", etaMs: 1_500 }); - unhealthyListener?.({ leaseId: "lse_1", deviceId: "dev_1" }); - recoveredListener?.({ leaseId: "lse_1", deviceId: "dev_1", attempts: 2 }); - return Promise.resolve({ - device: { - physical: false, - id: "dev_1", - driverDeviceId: "dev_1", - spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, - mode: "full", - }, - environment: {}, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }); - }, - }); - await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith({ connectAdmin: async () => client }), - ); - const lines = output.stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); - expect(lines).toContainEqual({ push: "progress", stage: "provisioning", etaMs: 1_500 }); - expect(lines).toContainEqual({ - push: "device-unhealthy", - leaseId: "lse_1", - deviceId: "dev_1", - }); - expect(lines).toContainEqual({ - push: "device-recovered", - leaseId: "lse_1", - deviceId: "dev_1", - attempts: 2, - }); - }); - - it("maps --mode and --no-wait onto the contract's mode/noWait input fields", async () => { - const output = outputCapture(); - let capturedInput: Record | undefined; - const client = fakeClient({ - requestLease: (input) => { - capturedInput = input as unknown as Record; - return Promise.resolve({ - device: { - physical: false, - id: "dev_1", - driverDeviceId: "dev_1", - spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, - mode: "full", - }, - environment: {}, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }); - }, - }); - await runCli( - [ - "lease", - "--platform", - "ios", - "--device", - "iPhone 17 Pro", - "--detach", - "--mode", - "slim", - "--no-wait", - ], - output.environmentWith({ connectAdmin: async () => client }), - ); - expect(capturedInput?.mode).toBe("slim"); - expect(capturedInput?.noWait).toBe(true); - }); - - // why: the real environment builder reads `SIMLOCK_BASELINE`, so the rows below prove the variable, not a hand-set field. - const fromEnv = (value: string): Partial => { - const { baselineFromEnv } = buildCliEnvironment(realCliEnvironmentPorts(), { - SIMLOCK_BASELINE: value, - }); - return baselineFromEnv === undefined ? {} : { baselineFromEnv }; - }; - - it.each([ - { - flag: "--image-tag", - field: "imageTag", - value: "google_apis_playstore", - lease: ["lease", "--platform", "android", "--device", "Pixel 8", "--detach"], - flagArgs: ["--image-tag", "google_apis_playstore"], - envOverride: () => ({}), - }, - { - flag: "--except", - field: "except", - value: ["store", "other"], - lease: ["lease", "--platform", "ios", "--mode", "slim", "--detach"], - flagArgs: ["--except", "store,other"], - envOverride: () => ({}), - }, - { - flag: "--baseline", - field: "baseline", - value: "checkout", - lease: ["lease", "--platform", "android", "--detach"], - flagArgs: ["--baseline", "checkout"], - envOverride: () => ({}), - }, - { - flag: "SIMLOCK_BASELINE", - field: "baseline", - value: "checkout", - lease: ["lease", "--platform", "android", "--detach"], - flagArgs: [], - envOverride: () => fromEnv("checkout"), - }, - { - flag: "--baseline over SIMLOCK_BASELINE", - field: "baseline", - value: "other", - lease: ["lease", "--platform", "android", "--detach"], - flagArgs: ["--baseline", "other"], - envOverride: () => fromEnv("checkout"), - }, - { - flag: "an empty SIMLOCK_BASELINE", - field: "baseline", - value: undefined, - lease: ["lease", "--platform", "android", "--detach"], - flagArgs: [], - envOverride: () => fromEnv(""), - }, - { - flag: "--physical", - field: "physical", - value: true, - lease: ["lease", "--platform", "ios", "--detach"], - flagArgs: ["--physical"], - envOverride: () => ({}), - }, - ])( - "maps $flag onto the contract's $field input field, and sends none without it", - async ({ field, value, lease, flagArgs, envOverride }) => { - const inputs: Record[] = []; - const client = fakeClient({ - requestLease: (input) => { - inputs.push(input as unknown as Record); - return Promise.resolve({ - device: { - class: "phone", - driverDeviceId: "00008110-001A2B3C4D5E", - id: "dev_1", - physical: true, - spec: { - class: "phone", - model: "iPhone 15 Pro", - osVersion: "18.4", - physical: true, - platform: "ios", - }, - }, - environment: {}, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }); - }, - }); - - await runCli( - [...lease, ...flagArgs], - outputCapture().environmentWith({ connectAdmin: async () => client, ...envOverride() }), - ); - await runCli(lease, outputCapture().environmentWith({ connectAdmin: async () => client })); - - expect(inputs[0]?.[field]).toEqual(value); - expect(inputs[1]).not.toHaveProperty(field); - }, - ); - - it("fails the removed full flag as an unknown option, before connecting", async () => { - const output = outputCapture(); - let connected = false; - const exitCode = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--full"], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient({}); - }, - }), - ); - expect(exitCode).toBe(2); - expect(connected).toBe(false); - expect(output.stderr).toContain("--full"); - }); - - it("omits mode and reports noWait: false when neither flag is given", async () => { - const output = outputCapture(); - let capturedInput: Record | undefined; - const client = fakeClient({ - requestLease: (input) => { - capturedInput = input as unknown as Record; - return Promise.resolve({ - device: { - physical: false, - id: "dev_1", - driverDeviceId: "dev_1", - spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, - mode: "full", - }, - environment: {}, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }); - }, - }); - await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith({ connectAdmin: async () => client }), - ); - expect(capturedInput).not.toHaveProperty("mode"); - expect(capturedInput?.noWait).toBe(false); - }); -}); - -describe("CLI: holder renew and release (ADR 0004 §2)", () => { - it("renews at a third of the TTL the daemon returned, re-deriving the cadence from each renewal", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - const signals = new EventEmitter(); - const renewals: Array<{ leaseId: string; at: number }> = []; - let released: string | undefined; - const client = fakeClient({ - renewLease: (input) => { - renewals.push({ at: clock.now(), leaseId: input.leaseId }); - // why: Half the grant's TTL comes back, so the next renewal must be half as far out too. - return Promise.resolve({ - deviceId: "dev_1", - grantedAt: 0, - id: input.leaseId, - ownerId: "test-requester", - requesterId: "test-requester", - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: clock.now() + 30_000, - }); - }, - releaseLease: (input) => { - released = input.leaseId; - return Promise.resolve({ leaseId: input.leaseId }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - clock, - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - - // why: The grant's deadline is 60_000, so the first renewal is due at 20_000 -- and not before. - clock.advance(19_999); - await settle(); - expect(renewals).toEqual([]); - clock.advance(1); - await settle(); - expect(renewals).toEqual([{ at: 20_000, leaseId: "lse_1" }]); - - // why: 30_000ms of TTL came back at 20_000, so the next renewal is at 30_000, not at 40_000. - clock.advance(10_000); - await settle(); - expect(renewals).toEqual([ - { at: 20_000, leaseId: "lse_1" }, - { at: 30_000, leaseId: "lse_1" }, - ]); - - signals.emit("SIGINT"); - expect(await runPromise).toBe(0); - // why: The release is the CLI's own call, not a side effect of the socket closing. - expect(released).toBe("lse_1"); - // why: And the timer is gone with it: no renewal can race the release, and nothing keeps the - // process alive after the command returns. - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - expect(renewals).toHaveLength(2); - }); - - it("exits 1 on a dead connection, naming the lease that outlived it, and releases nothing", async () => { - // why: ADR 0004 §3: the daemon released nothing when the socket died, so the lease is still - // granted -- the holder says so, says until when, and does not pretend to release it. - const output = outputCapture(); - let connectionLostListener: ((error: AnySimlockError) => void) | undefined; - let released = false; - const client = fakeClient({ - onConnectionLost: (listener) => { - connectionLostListener = listener; - return () => {}; - }, - releaseLease: () => { - released = true; - return Promise.resolve({ leaseId: "lse_1" }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ connectAdmin: async () => client }), - ); - // why: Let `requestLease` resolve and `runLease` reach its wait point, so the listener exists. - await settle(); - connectionLostListener?.( - new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), - ); - - expect(await runPromise).toBe(1); - const line = JSON.parse(output.stderr.trim().split("\n").at(-1) as string) as { - error: { code: string; message: string }; - }; - expect(line.error.code).toBe("DAEMON_CONNECTION_LOST"); - expect(line.error.message).toContain("lse_1"); - expect(line.error.message).toContain("60000"); - expect(released).toBe(false); - }); - - it("names the latest deadline in that line, not the one the grant carried", async () => { - // why: The deadline moves on every renewal, and this line is the only place a holder reports - // it. Quoting the grant-time one would, after about a TTL of uptime, name a moment in the - // past on a lease that is perfectly alive. - const clock = new FakeClock(0); - const output = outputCapture(); - let connectionLostListener: ((error: AnySimlockError) => void) | undefined; - const client = fakeClient({ - onConnectionLost: (listener) => { - connectionLostListener = listener; - return () => {}; - }, - renewLease: (input) => - Promise.resolve({ - deviceId: "dev_1", - grantedAt: 0, - id: input.leaseId, - ownerId: "test-requester", - requesterId: "test-requester", - lastRenewedAt: clock.now(), - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: clock.now() + 60_000, - }), - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - - // why: Two renewals: the grant's deadline was 60_000, so they land at 20_000 and 40_000, each - // answering with a deadline a full TTL further out. - clock.advance(20_000); - await settle(); - clock.advance(20_000); - await settle(); - - connectionLostListener?.( - new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), - ); - expect(await runPromise).toBe(1); - - const line = JSON.parse(output.stderr.trim().split("\n").at(-1) as string) as { - error: { message: string }; - }; - expect(line.error.message).toContain("100000"); - expect(line.error.message).not.toContain("60000"); - }); - - it("writes one line for a lost connection, even with a renewal in flight when it dies", async () => { - // why: The same socket carries both: the connection-lost listener writes the line that names - // the lease and its deadline, and the `lease.renew` that was in flight rejects with - // `DAEMON_CONNECTION_LOST`. That rejection is a retryable failure as far as - // `startLeaseRenewal` knows -- not a `lease-lost` -- so the holder must not answer it with - // a second line saying the daemon is gone, this one without the lease id a reader needs. - // Two things stop that, and this pins the outcome rather than either mechanism: the - // `finally` stops renewal (nothing is reported after `stop()`), and `onError` checks - // `connectionLost` so the answer does not depend on which of the two microtask chains - // wins. - const clock = new FakeClock(0); - const output = outputCapture(); - let connectionLostListener: ((error: AnySimlockError) => void) | undefined; - let rejectRenew: ((error: unknown) => void) | undefined; - const client = fakeClient({ - onConnectionLost: (listener) => { - connectionLostListener = listener; - return () => {}; - }, - renewLease: () => - new Promise((_resolve, reject) => { - rejectRenew = reject; - }), - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - - // why: The grant's deadline is 60_000: the first renewal starts at 20_000 and never answers. - clock.advance(20_000); - await settle(); - expect(rejectRenew).toBeDefined(); - - // why: The socket dies. In that order, deliberately: `SimlockClient.onConnectionLost` promises - // that "every in-flight call has already rejected `DAEMON_CONNECTION_LOST` by the time - // this listener runs", so the renewal's rejection is queued first and its handler runs - // after the listener has had its say. - rejectRenew?.(new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {})); - connectionLostListener?.( - new SimlockError("DAEMON_CONNECTION_LOST", "transport", "socket closed", {}), - ); - - expect(await runPromise).toBe(1); - const errorLines = output.stderr - .trim() - .split("\n") - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as { error?: { code: string; message: string } }) - .filter((line) => line.error !== undefined); - expect(errorLines).toHaveLength(1); - expect(errorLines[0]?.error?.code).toBe("DAEMON_CONNECTION_LOST"); - expect(errorLines[0]?.error?.message).toContain("lse_1"); - }); - - it("sends --ttl as the request's own ttlMs, and nothing when it is not given", async () => { - const output = outputCapture(); - const signals = new EventEmitter(); - const requested: Array = []; - const environment = output.environmentWith({ - clock: new FakeClock(0), - connectAdmin: async () => - fakeClient({ - requestLease: (input) => { - requested.push(input.ttlMs); - return Promise.resolve(detachedGrant); - }, - }), - signals: signals as unknown as CliEnvironment["signals"], - }); - await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach", "--ttl", "30m"], - environment, - ); - await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - environment, - ); - - // why: ADR 0004 §4: `--ttl` replaces `lease.defaultTtlMs` for this lease; omitting it sends no - // TTL at all, so the daemon's own default applies rather than a number the CLI invented. - expect(requested).toEqual([30 * 60_000, undefined]); - }); - - it("the CLI --lease-id flag sends leaseId and exits 13 on LEASE_ID_TAKEN", async () => { - const output = outputCapture(); - const requested: unknown[] = []; - let answer: "grant" | "taken" = "grant"; - const environment = output.environmentWith({ - clock: new FakeClock(0), - connectAdmin: async () => - fakeClient({ - requestLease: (input) => { - requested.push("leaseId" in input ? input.leaseId : "absent"); - return answer === "grant" - ? Promise.resolve(detachedGrant) - : Promise.reject( - fromWireError("LEASE_ID_TAKEN", "lease ID ad-7f3a is already in use", { - leaseId: "ad-7f3a", - }), - ); - }, - }), - }); - const lease = ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"]; - - await runCli([...lease, "--lease-id", "ad-7f3a"], environment); - await runCli(lease, environment); - answer = "taken"; - const exitCode = await runCli([...lease, "--lease-id", "ad-7f3a"], environment); - - // why: The flag's value goes out as typed and an omitted flag sends none: the contract decides - // what an ID may look like. - expect(requested).toEqual(["ad-7f3a", "absent", "ad-7f3a"]); - expect(exitCode).toBe(13); - expect(output.stderr).toContain('"code":"LEASE_ID_TAKEN"'); - }); - - it.each(["SIGINT", "SIGTERM"] as const)( - "releases explicitly on %s (ADR 0004 §2's catchable signals)", - async (signal) => { - const output = outputCapture(); - const signals = new EventEmitter(); - let released: string | undefined; - const client = fakeClient({ - releaseLease: (input) => { - released = input.leaseId; - return Promise.resolve({ leaseId: input.leaseId }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - clock: new FakeClock(0), - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - signals.emit(signal); - - expect(await runPromise).toBe(0); - expect(released).toBe("lse_1"); - }, - ); - - it.each([ - { trigger: "SIGINT", act: (signals: EventEmitter) => signals.emit("SIGINT") }, - { trigger: "SIGTERM", act: (signals: EventEmitter) => signals.emit("SIGTERM") }, - { - trigger: "the exit of the --bind-pid process", - act: (_signals: EventEmitter, parentWatch: FakeParentWatch) => parentWatch.exit(9876), - }, - ])("cancels its waiting request and exits 0 on $trigger before a grant", async ({ act }) => { - const output = outputCapture(); - const signals = new EventEmitter(); - const parentWatch = new FakeParentWatch(); - let requestSignal: AbortSignal | undefined; - let requested = false; - const client = fakeClient({ - // why: The request waits in the queue and never resolves on its own; like the client's - // documented abort, it rejects CANCELLED once its signal aborts. - requestLease: (_input, options) => { - requested = true; - requestSignal = options?.signal; - return new Promise((_resolve, reject) => { - options?.signal?.addEventListener("abort", () => { - reject(new SimlockError("CANCELLED", "domain", "Lease request was cancelled", {})); - }); - }); - }, - }); - const run = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--bind-pid", "9876"], - output.environmentWith({ - connectAdmin: async () => client, - parentPid: 4321, - parentWatch, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - const runState = promiseState(run); - await vi.waitFor(() => expect(requested).toBe(true)); - - act(signals, parentWatch); - await settle(); - - expect(requestSignal?.aborted).toBe(true); - expect(runState.state).toBe("fulfilled"); - expect(await run).toBe(0); - expect(output.stdout).toBe(""); - }); - - it("reports a failed renewal on stderr and keeps holding the lease", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - const signals = new EventEmitter(); - let released: string | undefined; - const client = fakeClient({ - renewLease: () => - Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist the lease", {})), - releaseLease: (input) => { - released = input.leaseId; - return Promise.resolve({ leaseId: input.leaseId }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - clock, - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - clock.advance(20_000); - await settle(); - - const errorLines = output.stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record) - .filter((line) => line.error !== undefined); - expect(errorLines).toContainEqual({ - error: { code: "INTERNAL", message: "could not persist the lease" }, - }); - - // why: A failed renewal is not an exit condition: the holder still holds, and still releases. - signals.emit("SIGINT"); - expect(await runPromise).toBe(0); - expect(released).toBe("lse_1"); - }); - - it("still closes the connection when stderr cannot take the failed release's error line", async () => { - const output = outputCapture(); - const signals = new EventEmitter(); - let closed = false; - const client = fakeClient({ - close: () => { - closed = true; - return Promise.resolve(); - }, - releaseLease: () => Promise.reject(new Error("release failed")), - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - clock: new FakeClock(0), - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - stderr: { - write: (value: string) => { - if (value.includes("release failed")) throw new Error("EPIPE"); - return undefined; - }, - }, - }), - ); - await vi.waitFor(() => expect(output.stdout).not.toBe("")); - - signals.emit("SIGINT"); - - // why: a closed stderr pipe must not skip the dispose and the close that follow the release. - expect(await runPromise).toBe(0); - expect(closed).toBe(true); - }); - - it("exits 14 without a farewell release when a renewal is answered UNKNOWN_LEASE", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let released = false; - let renewals = 0; - const client = fakeClient({ - renewLease: () => { - renewals += 1; - return Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lse_1" }), - ); - }, - releaseLease: () => { - released = true; - return Promise.resolve({ leaseId: "lse_1" }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - clock.advance(20_000); - - // why: The daemon's answer is final: this ends the same way the lease-lost push does, rather - // than repeating the rejection until the deadline and then sitting alive. - expect(await runPromise).toBe(14); - expect(released, "asking again would only raise UNKNOWN_LEASE").toBe(false); - expect(renewals).toBe(1); - expect(clock.pendingTimerCount).toBe(0); - - // why: The same stderr line the push path writes, so a script tailing for `push: "lease-lost"` - // sees this way of losing a lease too, with the error line after it as the detail. - const lines = output.stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); - expect(lines).toContainEqual({ - push: "lease-lost", - deviceId: "dev_1", - leaseId: "lse_1", - reason: "renew-rejected", - }); - expect(lines.at(-1)).toEqual({ - error: { code: "UNKNOWN_LEASE", message: "no such lease" }, - }); - }); - - it("exits 14 when renewal gives up, rather than holding a lease it cannot keep", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let released = false; - const client = fakeClient({ - // why: Never terminal, never successful: the ladder retries until the lease's own deadline - // passes, and then there is nothing left to hold. - renewLease: () => - Promise.reject(new SimlockError("INTERNAL", "domain", "could not persist", {})), - releaseLease: () => { - released = true; - return Promise.resolve({ leaseId: "lse_1" }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - clock.advance(20_000); // first attempt, fails, retries - await settle(); - clock.advance(60_000); // the retry lands past the lease's 60_000 deadline - - expect(await runPromise).toBe(14); - expect(released, "a lease that expired is not this process's to release").toBe(false); - const lines = output.stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record); - expect(lines).toContainEqual({ - push: "lease-lost", - deviceId: "dev_1", - leaseId: "lse_1", - reason: "renew-failed", - }); - }); - - it("writes one lease-lost line when a rejected renewal and the daemon's push both arrive", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let leaseLostListener: - | ((push: { leaseId: string; deviceId: string; reason: string }) => void) - | undefined; - const client = fakeClient({ - onLeaseLost: (listener) => { - leaseLostListener = listener; - return () => {}; - }, - renewLease: () => - Promise.reject( - new SimlockError("UNKNOWN_LEASE", "domain", "no such lease", { leaseId: "lse_1" }), - ), - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - clock.advance(20_000); - await settle(); - // why: The daemon's own push for the same lease follows the renewal's answer. One ending, one - // line: a script tailing stderr must not read this as two devices lost. - leaseLostListener?.({ deviceId: "dev_1", leaseId: "lse_1", reason: "expired" }); - - expect(await runPromise).toBe(14); - const leaseLostLines = output.stderr - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record) - .filter((line) => line.push === "lease-lost"); - expect(leaseLostLines).toEqual([ - { push: "lease-lost", deviceId: "dev_1", leaseId: "lse_1", reason: "renew-rejected" }, - ]); - }); - - it("gives up on a farewell release the daemon never answers, instead of hanging", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - const signals = new EventEmitter(); - const client = fakeClient({ releaseLease: () => new Promise(() => {}) }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ - clock, - connectAdmin: async () => client, - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - signals.emit("SIGINT"); - await settle(); - - clock.advance(RELEASE_TIMEOUT_MS); - - expect(await runPromise).toBe(0); - expect(output.stderr).toContain("Timed out releasing lease lse_1"); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("still releases the lease when printing the grant throws (--export-env with a bad key)", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let released: string | undefined; - const client = fakeClient({ - requestLease: () => - Promise.resolve({ - device: { - physical: false, - id: "dev_1", - driverDeviceId: "dev_1", - spec: { physical: false, platform: "ios", model: "x", osVersion: "26.5" }, - mode: "full", - }, - // why: A driver-supplied key that is not a shell identifier fails the command rather than - // being silently dropped -- and that throw must not cost the device. - environment: { "NOT A KEY": "value" }, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }), - releaseLease: (input) => { - released = input.leaseId; - return Promise.resolve({ leaseId: input.leaseId }); - }, - }); - const exitCode = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--export-env"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - - expect(exitCode).toBe(1); - expect(released, "an exit through a throw is still an exit, and still owes a release").toBe( - "lse_1", - ); - expect(clock.pendingTimerCount).toBe(0); - }); - - it("arms no renew timer for --detach: it prints and exits, exactly as before", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let renewed = 0; - const client = fakeClient({ - renewLease: () => { - renewed += 1; - return Promise.reject(new Error("--detach must not renew")); - }, - }); - const exitCode = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - - expect(exitCode).toBe(0); - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - expect(renewed).toBe(0); - }); - - it("stops renewing when the lease is lost, and still does not re-release it", async () => { - const clock = new FakeClock(0); - const output = outputCapture(); - let leaseLostListener: - | ((push: { leaseId: string; deviceId: string; reason: string }) => void) - | undefined; - let renewed = 0; - let released = false; - const client = fakeClient({ - onLeaseLost: (listener) => { - leaseLostListener = listener; - return () => {}; - }, - renewLease: (input) => { - renewed += 1; - return Promise.resolve({ - deviceId: "dev_1", - grantedAt: 0, - id: input.leaseId, - ownerId: "test-requester", - requesterId: "test-requester", - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: clock.now() + 60_000, - }); - }, - releaseLease: () => { - released = true; - return Promise.resolve({ leaseId: "lse_1" }); - }, - }); - const runPromise = runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro"], - output.environmentWith({ clock, connectAdmin: async () => client }), - ); - await settle(); - leaseLostListener?.({ deviceId: "dev_1", leaseId: "lse_1", reason: "ttl-backstop" }); - - expect(await runPromise).toBe(14); - expect(released).toBe(false); - expect(clock.pendingTimerCount).toBe(0); - clock.advance(600_000); - expect(renewed).toBe(0); - }); -}); - -describe("CLI: mcp command (ADR 0003 §11 -- lazy module load)", () => { - it.each([ - [["--gateway", "mcp"], 0, "gateway"], - [["mcp", "--gateway"], 0, "gateway"], - [["mcp"], 0, "none"], - [["--gateway=yes", "mcp"], 2, "--gateway takes no value"], - [["--local", "mcp"], 0, "local"], - [["mcp", "--local"], 0, "local"], - [["--local=yes", "mcp"], 2, "--local takes no value"], - [["--local", "--gateway", "mcp"], 2, "--gateway and --local cannot be used together"], - [["mcp", "--gateway", "--local"], 2, "--gateway and --local cannot be used together"], - ] as const)( - "--gateway and --local are taken from anywhere in the argv and reach the command as context.daemonFlag; = and both together are usage errors (%j)", - async (argv, exitCode, expected) => { - const output = outputCapture(); - const seen: string[] = []; - - const code = await runCli( - argv, - output.environmentWith({ - runMcpStdio: async (options) => { - seen.push(options.daemonFlag ?? "none"); - }, - }), - ); - - expect(code).toBe(exitCode); - if (exitCode === 0) { - expect(seen).toEqual([expected]); - } else { - expect(JSON.parse(output.stderr)).toEqual({ error: { code: "USAGE", message: expected } }); - expect(seen).toEqual([]); - } - }, - ); - - it("does not load the MCP stdio module unless the mcp command runs", async () => { - const output = outputCapture(); - let loaded = false; - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => fakeClient(), - loadMcpStdio: async () => { - loaded = true; - return async () => {}; - }, - }), - ); - expect(loaded).toBe(false); - }); - - it("loads and runs the stdio runner when the mcp command is dispatched", async () => { - const output = outputCapture(); - let ran = false; - const exitCode = await runCli( - ["mcp"], - output.environmentWith({ - loadMcpStdio: async () => async () => { - ran = true; - }, - }), - ); - expect(exitCode).toBe(0); - expect(ran).toBe(true); - }); - - it("reports a startup failure as a structured error instead of throwing raw", async () => { - const output = outputCapture(); - const exitCode = await runCli( - ["mcp"], - output.environmentWith({ - loadMcpStdio: async () => { - throw new Error("cannot bind mcp stdio transport"); - }, - }), - ); - expect(exitCode).toBe(1); - expect(JSON.parse(output.stderr)).toEqual({ - error: { code: "INTERNAL", message: "cannot bind mcp stdio transport" }, - }); - }); -}); - -describe("CLI: events history", () => { - const hour = 3_600_000; - - function envelope(seq: number, timestamp: number, id = `evt_${seq}`) { - return { id, seq, timestamp, event: "lease.granted", payload: { seq }, module: "leases" }; - } - - function printed(stdout: string): unknown[] { - if (stdout.trim() === "") return []; - return stdout - .trimEnd() - .split("\n") - .map((line) => JSON.parse(line) as unknown); - } - - async function historyDirectory( - prepare: (directory: string) => Promise, - ): Promise { - const directory = await mkdtemp(join(tmpdir(), "simlock-cli-history-")); - temporaryDirectories.push(directory); - await prepare(directory); - return { - ...realCliEnvironmentPorts(new NodeFilesystem()), - clock: new FakeClock(2 * hour), - daemonHome: { role: "worker", dataDirectory: directory, gatewayPresent: false }, - }; - } - - const stopped = (seq: number, timestamp: number) => ({ - event: "daemon.stopping", - id: `evt_${seq}`, - module: "daemon", - payload: { reason: `r${seq}` }, - seq, - timestamp, - }); - - it("events --since with nothing listening reads history.db and launches no daemon", async () => { - const ports = await historyDirectory((directory) => seedHistory(directory, [500, hour + 500])); - const launcher = new FakeDaemonLauncher(() => { - throw new Error("a daemon was launched"); - }); - const output = outputCapture({ ...ports, launcher }); - - const exitCode = await runCli(["events", "--since", "1h"], output.environmentWith()); - - expect(launcher.launches).toBe(0); - expect(exitCode).toBe(0); - expect(printed(output.stdout)).toEqual([stopped(2, hour + 500)]); - }); - - it("events --since with nothing listening and no history.db prints nothing, exit 0, even beside an old events.jsonl", async () => { - const bare = outputCapture(await historyDirectory(async () => {})); - const withOldFile = outputCapture( - await historyDirectory((directory) => - writeFile(join(directory, "events.jsonl"), `${JSON.stringify(envelope(1, hour + 500))}\n`), - ), - ); - - const codes = [ - await runCli(["events", "--since", "1h"], bare.environmentWith()), - await runCli(["events", "--since", "1h"], withOldFile.environmentWith()), - ]; - - expect(codes).toEqual([0, 0]); - expect([bare.stdout, withOldFile.stdout]).toEqual(["", ""]); - }); - - it("events --since with nothing listening and an unreadable history.db prints one JSON error and exits 1", async () => { - const output = outputCapture( - await historyDirectory((directory) => - writeFile(join(directory, "history.db"), "this is not a sqlite database ".repeat(20)), - ), - ); - - const exitCode = await runCli(["events", "--since", "1h"], output.environmentWith()); - - expect(exitCode).toBe(1); - expect(output.stdout).toBe(""); - expect(output.stderr.trimEnd().split("\n")).toHaveLength(1); - const printedError = JSON.parse(output.stderr) as { error: { code: string; message: string } }; - expect(printedError.error.code).toBe("INTERNAL"); - expect(printedError.error.message).toMatch(/not a database/); - }); - - it("events --since reports a daemon that refuses the handshake rather than reading the file", async () => { - const output = outputCapture(); - let fileRead = false; - const exitCode = await runCli( - ["events", "--since", "1h"], - output.environmentWith({ - connectExistingAdmin: async () => { - throw new SimlockError("PROTOCOL_VERSION_UNSUPPORTED", "protocol", "no overlap", { - client: { min: 5, max: 5 }, - daemon: { min: 4, max: 4 }, - daemonVersion: "0.9.0", - }); - }, - readEventHistory: async () => { - fileRead = true; - return []; - }, - }), - ); - - expect(exitCode).toBe(1); - expect(fileRead).toBe(false); - expect(output.stderr).toContain("PROTOCOL_VERSION_UNSUPPORTED"); - }); - - it("events --since with a daemon listening replays through the daemon", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - let fileRead = false; - const exitCode = await runCli( - ["events", "--since", "1h"], - output.environmentWith({ - clock: new FakeClock(2 * hour), - connectExistingAdmin: async () => - fakeClient({ - replayEvents: async (input) => { - asked.push(input); - return [envelope(7, hour + 1)]; - }, - }), - readEventHistory: async () => { - fileRead = true; - return []; - }, - }), - ); - - expect(exitCode).toBe(0); - expect(asked).toEqual([{ sinceTs: hour }]); - expect(printed(output.stdout)).toEqual([envelope(7, hour + 1)]); - expect(fileRead).toBe(false); - }); - - it("simlock events --follow prints an event pushed during the replay once", async () => { - const output = outputCapture(); - const signals = new EventEmitter(); - let listener: ((push: EventPush) => void) | undefined; - const push = (seq: number): EventPush => ({ - subscriptionId: "sub_1", - event: envelope(seq, seq), - }); - const runPromise = runCli( - ["events", "--follow"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - subscribeEvents: async (onEvent) => { - listener = onEvent; - return async () => {}; - }, - replayEvents: async () => { - // why: Emitted after the subscribe and before the replay answers: the replay holds - // seq 2, and seq 3 lands just after its snapshot. - listener?.(push(2)); - listener?.(push(3)); - // why: Same seq and timestamp as the replayed event 2, another id: another event. - listener?.({ subscriptionId: "sub_1", event: envelope(2, 2, "evt_other") }); - return [envelope(1, 1), envelope(2, 2)]; - }, - }), - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - listener?.(push(4)); - signals.emit("SIGINT"); - - expect(await runPromise).toBe(0); - expect(printed(output.stdout)).toEqual([ - envelope(1, 1), - envelope(2, 2), - push(3), - { subscriptionId: "sub_1", event: envelope(2, 2, "evt_other") }, - push(4), - ]); - }); - - it("events --since --follow prints the history once and then the live events", async () => { - const output = outputCapture(); - const signals = new EventEmitter(); - const asked: unknown[] = []; - let listener: ((push: EventPush) => void) | undefined; - const push = (seq: number): EventPush => ({ - subscriptionId: "sub_1", - event: envelope(seq, seq), - }); - const runPromise = runCli( - ["events", "--since", "1h", "--follow"], - output.environmentWith({ - clock: new FakeClock(2 * hour), - connectAdmin: async () => - fakeClient({ - subscribeEvents: async (onEvent) => { - listener = onEvent; - return async () => {}; - }, - replayEvents: async (input) => { - asked.push(input); - listener?.(push(2)); - return [envelope(1, 1), envelope(2, 2)]; - }, - }), - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - listener?.(push(3)); - signals.emit("SIGINT"); - - expect(await runPromise).toBe(0); - expect(asked).toEqual([{ sinceTs: hour }]); - expect(printed(output.stdout)).toEqual([envelope(1, 1), envelope(2, 2), push(3)]); - }); - - it("events --follow without --since asks for no sinceTs", async () => { - const output = outputCapture(); - const signals = new EventEmitter(); - const asked: unknown[] = []; - const runPromise = runCli( - ["events", "--follow"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - replayEvents: async (input) => { - asked.push(input); - return []; - }, - }), - signals: signals as unknown as CliEnvironment["signals"], - }), - ); - await settle(); - signals.emit("SIGINT"); - - expect(await runPromise).toBe(0); - expect(asked).toEqual([{}]); - }); -}); - -describe("CLI: daemon logs (ADR 0003 §11 -- must work when the daemon is dead)", () => { - it("reads the log file without connecting to the daemon at all", async () => { - const output = outputCapture(); - let connected = false; - const exitCode = await runCli( - ["daemon", "logs"], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - connectExistingAdmin: async () => { - connected = true; - return fakeClient(); - }, - readLogFile: async () => "line one\nline two\n", - }), - ); - expect(exitCode).toBe(0); - expect(connected).toBe(false); - expect(output.stdout).toBe("line one\nline two\n"); - }); - - it("--json wraps the log tail under a logs key, still with no connection attempted", async () => { - const output = outputCapture(); - let connected = false; - const exitCode = await runCli( - ["daemon", "logs", "--json"], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - connectExistingAdmin: async () => { - connected = true; - return fakeClient(); - }, - readLogFile: async () => "only line\n", - }), - ); - expect(exitCode).toBe(0); - expect(connected).toBe(false); - expect(JSON.parse(output.stdout)).toEqual({ logs: "only line" }); - }); -}); - -describe("CLI: daemon logs --follow", () => { - async function follow(filesystem: MemoryFilesystem) { - const clock = new FakeClock(0); - const signals = new EventEmitter(); - const ports = { - ...realCliEnvironmentPorts(filesystem), - clock, - signals: signals as unknown as NonNullable, - }; - const output = outputCapture(ports); - const exit = runCli(["daemon", "logs", "--follow"], output.environmentWith()); - // why: `followLogFile` loads its follower module on first use, so settling once is no - // longer enough for the tail read to have started; this yields to the event loop until the - // poll timer is armed, which is the point the old single settle proved the follower was - // live. The wait is a real timer: the lazy import resolves through I/O callbacks, which - // `setImmediate` turns alone would not drain. - const settle = async (): Promise => { - for (let attempt = 0; attempt < 200; attempt += 1) { - await new Promise((resolve) => setTimeout(resolve, 0)); - if (clock.pendingTimerCount > 0) return; - } - throw new Error("the follower never armed its poll timer -- lazy load of the module hung"); - }; - await settle(); - /** why: unlike the first settle this cannot require a timer -- after a signal the follower - * is meant to leave none armed -- and the lazy import is resolved by now, so immediate - * turns are enough to drain the poll's reads. */ - const settleTick = async (): Promise => { - for (let attempt = 0; attempt < 20; attempt += 1) - await new Promise((resolve) => setImmediate(resolve)); - }; - return { - clock, - exit, - output, - signals, - settle, - tick: async () => { - clock.advance(250); - await settleTick(); - }, - }; - } - - const LOG = "/simlock/daemon.log"; - - it("daemon logs --follow prints the existing tail, then a line appended later", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(LOG, "old one\nold two\n"); - const run = await follow(filesystem); - - expect(run.output.stdout).toBe("old one\nold two\n"); - await filesystem.writeFileAtomic(LOG, "old one\nold two\nnew three\n"); - await run.tick(); - - expect(run.output.stdout).toBe("old one\nold two\nnew three\n"); - run.signals.emit("SIGINT"); - await expect(run.exit).resolves.toBe(0); - }); - - it.each([ - ["ESC", "\u001b", "\\u001b"], - ["C1 CSI", "\u009b", "\\u009b"], - ])( - "prints %s in a log line escaped, in the tail and in a later line", - async (_name, raw, escaped) => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(LOG, `a${raw}b\n`); - const run = await follow(filesystem); - await filesystem.writeFileAtomic(LOG, `a${raw}b\nc${raw}d\n`); - await run.tick(); - - expect(run.output.stdout).toBe(`a${escaped}b\nc${escaped}d\n`); - run.signals.emit("SIGINT"); - await expect(run.exit).resolves.toBe(0); - }, - ); - - it("it prints lines written after a rotation, when the new file is already larger than the old offset at the next poll", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(LOG, "a\n"); - const run = await follow(filesystem); - - // why: Between two polls: one more line in the old file, a rotation, and a new file already - // longer than everything the follower had read of the old one. - await filesystem.writeFileAtomic(LOG, "a\nb\n"); - await filesystem.rename(LOG, `${LOG}.1`); - await filesystem.writeFileAtomic(LOG, "after rotation 1\nafter rotation 2\n"); - await run.tick(); - - expect(run.output.stdout).toBe("a\nb\nafter rotation 1\nafter rotation 2\n"); - run.signals.emit("SIGTERM"); - await expect(run.exit).resolves.toBe(0); - }); - - it("it waits while the log file is missing and prints lines once it appears", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - const run = await follow(filesystem); - await run.tick(); - expect(run.output.stdout).toBe(""); - expect(run.output.stderr).toBe(""); - - await filesystem.writeFileAtomic(LOG, "first line\n"); - await run.tick(); - - expect(run.output.stdout).toBe("first line\n"); - run.signals.emit("SIGINT"); - await expect(run.exit).resolves.toBe(0); - }); - - it("daemon logs --follow --json fails with USAGE, exit 2", async () => { - const output = outputCapture(); - - await expect( - runCli(["daemon", "logs", "--follow", "--json"], output.environmentWith()), - ).resolves.toBe(2); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("--follow") }, - }); - }); - - it("--follow on any daemon subcommand but logs fails with USAGE, exit 2", async () => { - for (const command of ["start", "stop", "status"]) { - const output = outputCapture(); - await expect(runCli(["daemon", command, "--follow"], output.environmentWith())).resolves.toBe( - 2, - ); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: expect.stringContaining("--follow") }, - }); - } - }); - - it("an interrupt ends the follow with exit 0 and leaves no timer armed", async () => { - const filesystem = new MemoryFilesystem(); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(LOG, "line\n"); - const run = await follow(filesystem); - expect(run.clock.pendingTimerCount).toBe(1); - - run.signals.emit("SIGINT"); - - await expect(run.exit).resolves.toBe(0); - expect(run.clock.pendingTimerCount).toBe(0); - expect(run.signals.listenerCount("SIGINT")).toBe(0); - expect(run.signals.listenerCount("SIGTERM")).toBe(0); - await filesystem.writeFileAtomic(LOG, "line\nafter exit\n"); - await run.tick(); - expect(run.output.stdout).toBe("line\n"); - }); -}); - -describe("CLI smoke test (ADR 0003 §12: one per frontend)", () => { - it("lease --detach, status, release, and token create round-trip through a real daemon", async () => { - const { socketPath } = await startTestDaemon(); - const ipc = new NodeIpcTransport(); - const environment = outputCapture().environmentWith({ - connectAdmin: async (resolveCredential) => { - const credential = await resolveCredential(); - return connectSimlockAdmin({ - ipc, - endpoint: socketPath, - principal: "smoke-test", - ...(credential === undefined ? {} : { credential }), - }); - }, - }); - - const leaseOut = outputCapture(); - const leaseExit = await runCli( - ["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], - leaseOut.environmentWith({ connectAdmin: environment.connectAdmin }), - ); - expect(leaseExit).toBe(0); - const grant = JSON.parse(leaseOut.stdout) as LeaseGrant; - // why: ADR 0004: one kind of lease -- what `--detach` changes is this process, not the grant. - expect(grant.lease).toMatchObject({ ttlMs: expect.any(Number) }); - - const statusOut = outputCapture(); - await runCli( - ["status", "--json"], - statusOut.environmentWith({ connectAdmin: environment.connectAdmin }), - ); - const status = JSON.parse(statusOut.stdout) as StatusGetOutput; - expect(status.leases?.map((lease) => lease.id)).toContain(grant.lease.id); - - const releaseOut = outputCapture(); - const releaseExit = await runCli( - ["release", grant.lease.id], - releaseOut.environmentWith({ connectAdmin: environment.connectAdmin }), - ); - expect(releaseExit).toBe(0); - - const tokenOut = outputCapture(); - const tokenExit = await runCli( - ["token", "create", "--role", "operator", "--label", "ci"], - tokenOut.environmentWith({ connectAdmin: environment.connectAdmin }), - ); - expect(tokenExit).toBe(0); - const created = JSON.parse(tokenOut.stdout) as { secret: string; token: { id: string } }; - expect(typeof created.secret).toBe("string"); - expect((created as unknown as Record).token).not.toHaveProperty("hash"); - }); -}); - -/** - * why: the CLI reference promises exactly one structured error line per failure, and binds `USAGE` to - * exit 2. A `SIMLOCK_HOME` whose socket path exceeds the platform limit had neither: the path - * was resolved while *building* the environment, and `runCli` takes its environment as a - * default parameter -- whose initializer runs before the function body, and therefore before - * the try/catch. The error escaped as an uncaught rejection with a raw stack trace, and took - * `simlock --help` down with it. - */ -describe("CLI: a SIMLOCK_HOME the kernel could not bind", () => { - const tooDeep = `/tmp/${"d".repeat(120)}`; - const ports = (): CliEnvironmentPorts => ({ - ...realCliEnvironmentPorts(), - daemonHome: { role: "worker", dataDirectory: tooDeep, gatewayPresent: false }, - }); - - it("still prints help, which needs no socket at all", async () => { - const output = outputCapture(ports()); - - await expect(runCli(["--help"], output.environmentWith())).resolves.toBe(0); - - expect(output.stdout).toContain("Usage: simlock"); - expect(output.stderr).toBe(""); - }); - - it("reports a command that does need one as a single USAGE line, exit 2", async () => { - const output = outputCapture(ports()); - - await expect(runCli(["status"], output.environmentWith())).resolves.toBe(2); - - const lines = output.stderr.trimEnd().split("\n"); - expect(lines).toHaveLength(1); - const reported = JSON.parse(lines[0] ?? "") as { - readonly error: { readonly code: string; readonly message: string }; - }; - expect(reported.error.code).toBe("USAGE"); - expect(reported.error.message).toContain("SIMLOCK_HOME"); - }); -}); - -describe("CLI: the daemon endpoint check", () => { - it("buildCliEnvironment reports the daemon endpoint present only while its socket file exists", async () => { - const filesystem = new MemoryFilesystem(); - const environment = buildCliEnvironment(realCliEnvironmentPorts(filesystem), {}); - const socketPath = resolveDaemonSocketPath("/simlock", "linux"); - - expect(await environment.daemonEndpointPresent()).toBe(false); - await filesystem.mkdirp("/simlock"); - await filesystem.writeFileAtomic(socketPath, ""); - expect(await environment.daemonEndpointPresent()).toBe(true); - await filesystem.rm(socketPath); - expect(await environment.daemonEndpointPresent()).toBe(false); - }); - - it("on Windows reports the named pipe present while something answers it, and not after it closes", async () => { - const ipc = new MemoryIpcTransport(); - const environment = buildCliEnvironment( - { ...realCliEnvironmentPorts(), ipc, hostPlatform: "win32" }, - {}, - ); - - expect(await environment.daemonEndpointPresent()).toBe(false); - const listener = await ipc.listen( - resolveDaemonSocketPath("/simlock", "win32"), - () => undefined, - ); - expect(await environment.daemonEndpointPresent()).toBe(true); - await listener.close(); - expect(await environment.daemonEndpointPresent()).toBe(false); - }); - - it("on Windows counts a pipe that does not answer in time as present, since only a refused or missing one proves it gone", async () => { - const connector: IpcConnector = { - connect: async () => { - throw new IpcError("connection-timeout", "timed out", undefined); - }, - }; - const environment = buildCliEnvironment( - { ...realCliEnvironmentPorts(), ipc: connector, hostPlatform: "win32" }, - {}, - ); - - expect(await environment.daemonEndpointPresent()).toBe(true); - }); -}); - -describe("CLI: requester id from the agent session", () => { - const sessionEnv = { CLAUDE_CODE_SESSION_ID: "abc" }; - - it("buildCliEnvironment falls back to the pid when no agent id or session id is set", () => { - expect(buildCliEnvironment(realCliEnvironmentPorts(), {}).requesterId).toBe( - String(process.pid), - ); - }); - - it("simlock lease --agent-id overrides a session-derived id", async () => { - const requested: string[] = []; - const client = fakeClient({ - requestLease: (input, _options) => { - requested.push(input.requesterId ?? ""); - return fakeClient().requestLease(input); - }, - }); - const output = outputCapture(); - const environment = { - ...buildCliEnvironment(realCliEnvironmentPorts(), sessionEnv), - connectAdmin: async () => client, - stderr: { write: (value: string) => (output.stderr += value) }, - stdout: { write: (value: string) => (output.stdout += value) }, - }; - - await expect( - runCli(["lease", "--platform", "ios", "--device", "iPhone 17 Pro", "--detach"], environment), - ).resolves.toBe(0); - await expect( - runCli( - [ - "lease", - "--platform", - "ios", - "--device", - "iPhone 17 Pro", - "--detach", - "--agent-id", - "explicit-agent", - ], - environment, - ), - ).resolves.toBe(0); - - expect(requested).toEqual(["claude-code:abc", "explicit-agent"]); - }); -}); - -describe("CLI: pure helpers", () => { - it("parseDuration parses units and rejects garbage", () => { - expect(parseDuration("500")).toBe(500); - expect(parseDuration("500ms")).toBe(500); - expect(parseDuration("2s")).toBe(2_000); - expect(parseDuration("3m")).toBe(180_000); - expect(parseDuration("1h")).toBe(3_600_000); - expect(() => parseDuration("banana")).toThrow("Invalid duration"); - }); - - it("parseDuration accepts the d unit and rejects an unknown one", () => { - expect(parseDuration("2d")).toBe(2 * 24 * 60 * 60 * 1000); - expect(() => parseDuration("2w")).toThrow("Invalid duration"); - }); -}); - -describe("readPipedStdin", () => { - function fakeStdin(chunks: readonly string[], isTTY: boolean): typeof process.stdin { - const stream = Readable.from(chunks); - return Object.assign(stream, { isTTY }) as unknown as typeof process.stdin; - } - - it("reads a terminal stdin to its end, joining its chunks", async () => { - const stdin = fakeStdin(["typed ", "text\n"], true); - const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); - - try { - await expect(readPipedStdin()).resolves.toBe("typed text\n"); - } finally { - stdinGetter.mockRestore(); - } - }); - - it("reads a piped stdin to EOF, concatenating every chunk in order", async () => { - const stdin = fakeStdin(["piped ", "pay", "load\n"], false); - const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); - - try { - await expect(readPipedStdin()).resolves.toBe("piped payload\n"); - } finally { - stdinGetter.mockRestore(); - } - }); - - it("returns an empty string for a piped stdin that carries nothing", async () => { - const stdin = fakeStdin([], false); - const stdinGetter = vi.spyOn(process, "stdin", "get").mockReturnValue(stdin); - - try { - await expect(readPipedStdin()).resolves.toBe(""); - } finally { - stdinGetter.mockRestore(); - } - }); -}); - -function simlockError(code: AnySimlockError["code"]): AnySimlockError { - const messages: Partial> = { - NO_CAPACITY: "No capacity available", - }; - const kind = - code === "DAEMON_CONNECTION_LOST" || - code === "DAEMON_STOPPING" || - code === "DAEMON_STARTUP_FAILED" - ? "transport" - : ( - [ - "BAD_FRAME", - "HANDSHAKE_REQUIRED", - "BAD_REQUEST", - "UNKNOWN_REQUEST", - "PROTOCOL_VERSION_UNSUPPORTED", - "ADMIN_AUTHENTICATION_FAILED", - "FORBIDDEN", - "UNKNOWN_DAEMON_ERROR", - ] as const - ).includes(code as never) - ? "protocol" - : "domain"; - return new SimlockError( - code, - kind, - messages[code] ?? `${code} failed`, - {} as never, - ) as unknown as AnySimlockError; -} - -/** why: Hoisted out of `fakeClient` so a test can hand back the same status with a different - * `mode` -- which is what the passthrough path branches on (ADR 0005 §19c). */ -const EMPTY_STATUS = { - devices: [], - host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, - leases: [], - capacity: { - ios: { - atRamBudget: false, - limit: 1, - running: 0, - maxRunning: 1, - reserved: 0, - overLimit: false, - warm: 0, - used: 0, - }, - android: { - atRamBudget: false, - limit: 1, - running: 0, - maxRunning: 1, - reserved: 0, - overLimit: false, - warm: 0, - used: 0, - }, - global: { running: 0, maxRunning: 2, reserved: 0, overLimit: false, warm: 0 }, - }, - daemon: { - health: "running", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - queueDepth: 0, -} satisfies StatusGetOutput; - -describe("simlock update and simlock status", () => { - type UpdateAnswer = Awaited>; - type Step = UpdateAnswer | SimlockError; - const lost = () => - new SimlockError("DAEMON_CONNECTION_LOST", "transport", "The daemon connection was lost", {}); - const answer = (overrides: Partial = {}): UpdateAnswer => ({ - started: false, - upToDate: false, - resolved: null, - version: "0.3.0", - health: "running", - update: null, - lastUpdateFailure: null, - latestVersion: null, - ...overrides, - }); - const state = ( - stage: "waiting" | "installing" | "restarting", - leases = 0, - ): NonNullable => { - const base = { from: "0.3.0", to: "0.4.0", forced: false, startedAt: 1_000 }; - return stage === "waiting" ? { stage, leases, ...base } : { stage, ...base }; - }; - const failure = (stage: "installing" | "restarting", message: string) => ({ - stage, - from: "0.3.0", - to: "0.4.0", - message, - at: 2_000, - }); - const started = (update = state("installing")) => - answer({ started: true, resolved: "0.4.0", update }); - const running = (version = "0.4.0", overrides: Partial = {}) => - answer({ version, ...overrides }); - - async function runUpdate( - argv: readonly string[], - script: { - readonly first: UpdateAnswer | SimlockError; - readonly polls?: readonly Step[]; - readonly after?: readonly (Step | Error)[]; - readonly onPoll?: (index: number, interrupt: () => void) => void; - readonly onReconnect?: (interrupt: () => void) => void; - }, - ) { - const output = outputCapture(); - const clock = new FakeClock(0); - const calls: unknown[] = []; - const polls = [...(script.polls ?? [])]; - const after = [...(script.after ?? [])]; - let pollCount = 0; - const signals = new EventEmitter(); - let opened = 0; - const closedClients = new Set(); - const clientOver = (steps: Step[], first?: UpdateAnswer | SimlockError) => { - const id = opened++; - return fakeClient({ - close: async () => { - closedClients.add(id); - }, - updateDaemon: async (input) => { - calls.push(input); - const isFirst = first !== undefined && calls.length === 1; - if (!isFirst) script.onPoll?.(pollCount++, () => signals.emit("SIGINT")); - const step = isFirst ? first : steps.length > 1 ? steps.shift() : steps[0]; - if (step === undefined) throw new Error("no scripted answer"); - if (step instanceof Error) throw step; - return step; - }, - }); - }; - const environment = output.environmentWith({ - clock, - connectAdmin: async () => clientOver(polls, script.first), - connectExistingAdmin: async () => { - script.onReconnect?.(() => signals.emit("SIGINT")); - const step = after.length > 1 ? after.shift() : after[0]; - if (step === undefined || step instanceof Error) throw step ?? new Error("refused"); - return clientOver([step]); - }, - sleep: async (milliseconds) => clock.advance(milliseconds), - signals: signals as unknown as CliEnvironment["signals"], - }); - const code = await runCli(argv, environment); - return { - code, - calls, - stdout: output.stdout, - stderr: output.stderr, - opened, - closed: closedClients.size, - }; - } - - it("follows an update to simlock is running, reports a failure and a rollback, says when nothing is needed, stops following on Ctrl-C only, prints --json, refuses bad input, and shows the version, the update and the last failure in simlock status", async () => { - let reconnects = 0; - // why: the follower reconnects every second and gives up when 15 minutes have passed since the last answer, so the last attempt is at 14:59 on the fake clock. - const RECONNECTS_IN_15_MINUTES = 899; - const header = "Updating simlock 0.3.0 → 0.4.0\n"; - const wait = - "Waiting for 2 leases to end. Ctrl-C stops following; simlock update --force stops waiting.\n"; - const cases: readonly { - readonly name: string; - readonly argv: readonly string[]; - readonly script: Parameters[1]; - readonly code: number; - readonly stdout: string; - readonly calls?: readonly unknown[]; - readonly reconnects?: number; - }[] = [ - { - name: "follows to running", - argv: ["update"], - script: { - first: started(state("waiting", 2)), - polls: [ - answer({ update: state("waiting", 2) }), - answer({ update: state("waiting", 1) }), - answer({ update: state("installing") }), - answer({ update: state("restarting") }), - lost(), - ], - after: [new Error("refused"), running()], - }, - code: 0, - stdout: `${header}${wait}Waiting for 1 lease to end.\nInstalling simlock@0.4.0\nRestarting the daemon\nsimlock 0.4.0 is running\n`, - calls: [{}, ...Array.from({ length: 6 }, () => ({ query: true }))], - }, - { - name: "follows an update that already runs, whatever version it names", - argv: ["update", "--version", "0.9.9"], - script: { - first: answer({ update: state("installing") }), - polls: [lost()], - after: [running()], - }, - code: 0, - stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nsimlock 0.4.0 is running\n`, - }, - { - name: "reports a failed install", - argv: ["update"], - script: { - first: started(), - polls: [answer({ lastUpdateFailure: failure("installing", "npm error code EACCES") })], - }, - code: 1, - stdout: `${header}Installing simlock@0.4.0\nUpdate failed (installing): npm error code EACCES\nsimlock 0.3.0 is still running and takes leases again.\n`, - }, - { - name: "reports a version that did not start and the rollback", - argv: ["update"], - script: { - first: started(), - polls: [answer({ update: state("restarting") }), lost()], - after: [ - answer({ - lastUpdateFailure: failure("restarting", "0.4.0 did not start: health failed"), - }), - ], - }, - code: 1, - stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nUpdate failed (restarting): 0.4.0 did not start: health failed. Rolled back to 0.3.0.\n`, - }, - { - name: "waits through a daemon that answers while it is still starting", - argv: ["update"], - script: { - first: started(), - polls: [lost()], - after: [ - answer({ health: "starting" }), - answer({ health: "starting" }), - answer({ health: "starting" }), - answer({ - lastUpdateFailure: failure("restarting", "0.4.0 did not start: health failed"), - }), - ], - }, - code: 1, - stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nUpdate failed (restarting): 0.4.0 did not start: health failed. Rolled back to 0.3.0.\n`, - }, - { - name: "gives up when the daemon does not come back in 15 minutes", - argv: ["update"], - script: { - first: started(), - polls: [answer({ update: state("restarting") }), lost()], - after: [new Error("refused")], - onReconnect: () => { - reconnects += 1; - }, - }, - code: 1, - stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nThe daemon did not come back; run simlock status.\n`, - reconnects: RECONNECTS_IN_15_MINUTES, - }, - { - name: "stops with the daemon's own error when a poll is refused for a reason other than a lost connection", - argv: ["update"], - script: { - first: started(), - polls: [new SimlockError("UPDATE_RESOLVE_FAILED", "domain", "npm is down", {})], - }, - code: 1, - stdout: `${header}Installing simlock@0.4.0\n`, - }, - { - name: "prints its usage for --help and asks the daemon nothing", - argv: ["update", "--help"], - script: { first: answer() }, - code: 0, - stdout: "Usage: simlock update [--version ] [--force] [--json]\n", - calls: [], - }, - { - name: "the latest version is the running one", - argv: ["update"], - script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, - code: 0, - stdout: "simlock 0.3.0 is already the latest\n", - calls: [{}], - }, - { - name: "the named version is the running one", - argv: ["update", "--version", "0.3.0"], - script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, - code: 0, - stdout: "simlock 0.3.0 is already running\n", - calls: [{ version: "0.3.0" }], - }, - { - name: "the running version is newer than the latest", - argv: ["update"], - script: { first: answer({ upToDate: true, resolved: "0.4.0", version: "0.5.0-rc.1" }) }, - code: 0, - stdout: - "simlock 0.5.0-rc.1 is newer than the latest (0.4.0). Use --version to install another version.\n", - }, - { - name: "sends force and the version", - argv: ["update", "--force", "--version", "0.4.0-rc.1"], - script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, - code: 0, - stdout: "simlock 0.3.0 is already running\n", - calls: [{ force: true, version: "0.4.0-rc.1" }], - }, - { - name: "--json prints the final answer only", - argv: ["update", "--json"], - script: { first: started(state("waiting", 1)), polls: [lost()], after: [running()] }, - code: 0, - stdout: `${JSON.stringify(running())}\n`, - }, - { - name: "--json with nothing to do prints the answer at once", - argv: ["update", "--json"], - script: { first: answer({ upToDate: true, resolved: "0.3.0" }) }, - code: 0, - stdout: `${JSON.stringify(answer({ upToDate: true, resolved: "0.3.0" }))}\n`, - calls: [{}], - }, - { - name: "Ctrl-C stops following and sends nothing more", - argv: ["update"], - script: { - first: started(state("waiting", 2)), - polls: [answer({ update: state("waiting", 2) })], - onPoll: (_index, interrupt) => interrupt(), - }, - code: 0, - stdout: `${header}${wait}Stopped following. The update goes on; run simlock status to see it.\n`, - calls: [{}, { query: true }], - }, - { - name: "Ctrl-C while the daemon restarts stops following", - argv: ["update"], - script: { - first: started(), - polls: [lost()], - after: [new Error("refused")], - onReconnect: (interrupt) => interrupt(), - }, - code: 0, - stdout: `${header}Installing simlock@0.4.0\nRestarting the daemon\nStopped following. The update goes on; run simlock status to see it.\n`, - calls: [{}, { query: true }], - }, - ]; - for (const { name, argv, script, code, stdout, calls, reconnects: expected } of cases) { - reconnects = 0; - const result = await runUpdate(argv, script); - if (expected !== undefined) expect(reconnects, name).toBe(expected); - expect({ name, code: result.code, stdout: result.stdout }, name).toEqual({ - name, - code, - stdout, - }); - if (calls !== undefined) expect(result.calls, name).toEqual(calls); - expect(result.closed, `${name}: every client opened is closed`).toBe(result.opened); - } - - const refusals: readonly [readonly string[], number, string][] = [ - [["update", "--version", "latest"], 2, "--version takes a version, e.g. 0.4.0"], - [["update", "--version", "../x"], 2, "--version takes a version, e.g. 0.4.0"], - [["update", "--version", `1.0.0-${"a".repeat(59)}`], 2, "--version takes a version"], - ]; - for (const [argv, code, message] of refusals) { - const result = await runUpdate(argv, { first: answer() }); - expect({ argv, code, stderr: result.stderr.includes(message), calls: result.calls }).toEqual({ - argv, - code, - stderr: true, - calls: [], - }); - } - - const refused = [ - ["UPDATE_RESOLVE_FAILED", 1, "could not resolve simlock@9.9.9: No match found"], - ["UPDATE_UNSUPPORTED_INSTALL", 2, "simlock was not installed with npm i -g"], - ] as const; - for (const [codeName, exit, message] of refused) { - const result = await runUpdate(["update"], { - first: new SimlockError(codeName, "domain", message, {}), - }); - expect({ code: result.code, stdout: result.stdout }).toEqual({ code: exit, stdout: "" }); - expect(JSON.parse(result.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: codeName, message }, - }); - } - - const next = "Host: macOS 15.5 arm64"; - expect(await statusLines({ version: "0.3.0" })).toEqual([ - "Daemon: running (worker)", - "Simlock: 0.3.0", - `CLI: ${PACKAGE_VERSION}`, - next, - expect.stringMatching(/^Running global/), - ]); - for (const [update, line] of [ - [state("waiting", 2), "Updating to 0.4.0: waiting for 2 leases"], - [state("waiting", 1), "Updating to 0.4.0: waiting for 1 lease"], - [state("installing"), "Updating to 0.4.0: installing"], - [state("restarting"), "Updating to 0.4.0: restarting"], - ] as const) { - expect((await statusLines({ version: "0.3.0", update })).slice(0, 4)).toEqual([ - "Daemon: running (worker)", - "Simlock: 0.3.0", - `CLI: ${PACKAGE_VERSION}`, - line, - ]); - } - expect( - ( - await statusLines({ - version: "0.3.0", - lastUpdateFailure: failure("installing", "npm error code EACCES"), - }) - ).slice(0, 4), - ).toEqual([ - "Daemon: running (worker)", - "Simlock: 0.3.0", - `CLI: ${PACKAGE_VERSION}`, - "Last update failed (installing): npm error code EACCES", - ]); - }); - - it("simlock status says when the daemon has seen a newer Simlock version on npm", async () => { - for (const [version, latestVersion, line] of [ - ["0.3.0", "0.4.0", "Simlock: 0.3.0 (0.4.0 available)"], - ["0.9.0", "0.10.0", "Simlock: 0.9.0 (0.10.0 available)"], - ["0.4.0-rc.2", "0.4.0", "Simlock: 0.4.0-rc.2 (0.4.0 available)"], - ["0.4.0", "0.4.0", "Simlock: 0.4.0"], - ["0.5.0-rc.1", "0.4.0", "Simlock: 0.5.0-rc.1"], - ["0.3.0", null, "Simlock: 0.3.0"], - ] as const) { - expect((await statusLines({ version, latestVersion }))[1]).toBe(line); - } - }); -}); - -async function statusLines(daemon: Partial) { - const output = outputCapture(); - await runCli( - ["status"], - output.environmentWith({ - connectAdmin: async () => - fakeClient({ - getStatus: () => - Promise.resolve({ ...EMPTY_STATUS, daemon: { ...EMPTY_STATUS.daemon, ...daemon } }), - }), - }), - ); - return output.stdout.split("\n").slice(0, 5); -} - -function fakeClient(overrides: Partial = {}): SimlockAdminClient { - const emptyCatalog: CatalogGetOutput = { platforms: [] }; - const emptyDoctor: DoctorReport = { findings: [], sources: [] }; - const emptyLeaseList: LeaseListOutput = { leases: [] }; - const emptyListGet: ListGetOutput = []; - const emptyTokenList: TokenListOutput = { tokens: [] }; - const grant: LeaseGrant = { - environment: {}, - device: { - physical: false, - id: "dev_1", - driverDeviceId: "dev_1", - spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, - mode: "full", - }, - lease: { - id: "lse_1", - deviceId: "dev_1", - requesterId: "test-requester", - ownerId: "test-requester", - grantedAt: 0, - lastRenewedAt: 0, - idChosenByRequester: false, - ttlMs: 60_000, - ttlDeadline: 60_000, - }, - timing: { - estimatedProvisionMs: 0, - estimatedBootMs: 0, - estimatedReclaimMs: 0, - estimatedReadyMs: 0, - }, - }; - const base: SimlockAdminClient = { - principal: "test-requester", - role: "admin", - daemonVersion: "test", - getCatalog: () => Promise.resolve(emptyCatalog), - getStatus: () => Promise.resolve(EMPTY_STATUS), - identity: () => - Promise.resolve({ - name: "test-requester", - principal: "test-requester", - role: "admin", - via: "socket", - }), - listComponents: () => Promise.resolve({ components: [] }), - requestLease: (_input, _options) => Promise.resolve(grant), - resolvePassthrough: () => Promise.resolve({ args: [], command: "adb", env: {} }), - exec: () => Promise.resolve({ exitCode: 0 }), - rotate: (leaseId, orientation) => Promise.resolve({ leaseId, orientation }), - cancelLease: () => Promise.resolve({ result: "not-found" }), - renewLease: () => Promise.resolve(grant.lease as LeaseRecord), - releaseLease: (input) => Promise.resolve({ leaseId: input.leaseId }), - listLeases: () => Promise.resolve(emptyLeaseList), - runDoctor: () => Promise.resolve(emptyDoctor), - onLeaseLost: () => () => {}, - onDeviceUnhealthy: () => () => {}, - onDeviceRecovered: () => () => {}, - onConnectionLost: () => () => {}, - close: () => Promise.resolve(), - releaseAllLeases: () => Promise.resolve({ leaseIds: [] }), - list: () => Promise.resolve(emptyListGet), - runCleanup: () => Promise.resolve([]), - runNuke: () => - Promise.resolve({ deletedDevices: [], releasedLeaseIds: [], removedBaselines: [] }), - forgetWarmKind: () => Promise.resolve({ forgotten: [] }), - saveBaseline: () => Promise.reject(new Error("saveBaseline was not expected")), - listBaselines: () => Promise.resolve({ baselines: [] }), - removeBaseline: () => Promise.reject(new Error("removeBaseline was not expected")), - getConfig: () => Promise.resolve({} as SimlockConfig), - stopDaemon: () => Promise.resolve({ stopping: true }), - updateDaemon: () => - Promise.resolve({ - started: false, - upToDate: true, - resolved: null, - version: "test", - health: "running" as const, - update: null, - lastUpdateFailure: null, - latestVersion: null, - }), - replayEvents: () => Promise.resolve([]), - usage: (window) => Promise.resolve(usageAnswer(window)), - subscribeEvents: () => Promise.resolve(() => Promise.resolve()), - createToken: (input) => - Promise.resolve({ - secret: "sec_1", - token: { id: "tok_1", role: input.role, createdAt: 0 }, - }), - listTokens: () => Promise.resolve(emptyTokenList), - revokeToken: () => Promise.resolve({ revoked: true }), - listWorkers: () => Promise.resolve({ workers: [] }), - drainWorker: (input) => Promise.resolve({ drained: true as const, workerId: input.workerId }), - undrainWorker: (input) => - Promise.resolve({ drained: false as const, workerId: input.workerId }), - removeWorker: (input) => Promise.resolve({ removed: true, workerId: input.workerId }), - updateWorker: (input) => - Promise.resolve({ - health: "running" as const, - lastUpdateFailure: null, - latestVersion: null, - resolved: null, - started: false, - update: null, - upToDate: false, - version: "0.3.0", - workerId: input.workerId, - }), - testAlertDestination: (input) => - Promise.resolve({ destination: input.destination, status: 200 }), - installComponent: (input) => - Promise.resolve({ - component: input.version, - outcome: "installed" as const, - platform: input.platform, - version: input.version, - }), - installComponentOnWorkers: () => Promise.resolve({ results: [] }), - removeComponent: (input) => - Promise.resolve({ - outcome: input.dryRun === true ? ("would-remove" as const) : ("removed" as const), - platform: input.platform, - version: input.version, - }), - addDevice: (input) => - Promise.resolve({ - appCount: 2, - class: "phone" as const, - driverDeviceId: input.id, - id: "dev_9", - model: "iPhone 15 Pro", - osVersion: "18.4", - platform: input.platform, - reenrolled: false, - }), - removeDevice: (input) => Promise.resolve({ driverDeviceId: input.id, id: input.id }), - ...overrides, - }; - return base; -} - -interface OutputCapture { - stdout: string; - stderr: string; - environmentWith(overrides?: Partial): CliEnvironment; -} - -function outputCapture(ports?: CliEnvironmentPorts, env: NodeJS.ProcessEnv = {}): OutputCapture { - const capture: OutputCapture = { - stdout: "", - stderr: "", - environmentWith(overrides = {}) { - const stderrOut = { write: (value: string) => (capture.stderr += value) }; - const stdoutOut = { write: (value: string) => (capture.stdout += value) }; - if (ports === undefined) { - return { - clock: new FakeClock(0), - configPath: "/simlock/config.json", - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - requesterId: "test-requester", - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - filesystem: new MemoryFilesystem(), - connectAdmin: async () => fakeClient(), - connectExistingAdmin: async () => fakeClient(), - daemonEndpointPresent: async () => false, - readAdminTokenFile: async () => undefined, - sleep: async () => {}, - readConfigFile: async () => ({}), - writeConfigFile: async () => {}, - validateConfig: async () => {}, - readLogFile: async () => "", - readEventHistory: async () => [], - signals: new EventEmitter() as unknown as CliEnvironment["signals"], - parentWatch: new FakeParentWatch(), - stderr: stderrOut, - stdout: stdoutOut, - confirm: async () => true, - webSocket: new FakeWebSocketClientConnector(), - httpJson: new FakeHttpJsonClient(), - tcpListeners: new FakeTcpListenerFactory(), - tcpStreams: new FakeTcpStreamConnector(), - ...overrides, - }; - } - const base = buildCliEnvironment( - { - ...ports, - signals: - ports.signals ?? - (new EventEmitter() as unknown as NonNullable), - parentWatch: ports.parentWatch ?? new FakeParentWatch(), - confirm: ports.confirm ?? (async () => true), - stderr: stderrOut, - stdout: stdoutOut, - }, - env, - ); - return { ...base, ...overrides }; - }, - }; - return capture; -} - -class HelloThenDeathIpc implements IpcConnector { - helloReplies = 0; - deaths = 0; - readonly frameTypes: string[] = []; - #listening = false; - - rejectedCredentials = 0; - clientCloses = 0; - - constructor( - readonly closeAfter: "hello" | "request" | "silent" | "stopping" | "starting" = "hello", - readonly rejectCredential = false, - ) {} - - startListening(): void { - this.#listening = true; - } - - async connect(): Promise { - if (!this.#listening) { - throw new IpcError("endpoint-not-found", "No IPC endpoint yet", undefined); - } - return new HelloThenDeathConnection(this); - } -} - -class HelloThenDeathConnection implements IpcConnection { - readonly #dataListeners = new Set<(chunk: string) => void>(); - readonly #closeListeners = new Set<() => void>(); - #closed = false; - - constructor(private readonly daemon: HelloThenDeathIpc) {} - - get closed(): boolean { - return this.#closed; - } - - onData(listener: (chunk: string) => void): () => void { - this.#dataListeners.add(listener); - return () => this.#dataListeners.delete(listener); - } - - onClose(listener: () => void): () => void { - this.#closeListeners.add(listener); - return () => this.#closeListeners.delete(listener); - } - - onError(): () => void { - return () => {}; - } - - async write(contents: string): Promise { - if (this.#closed) return; - let answeredHello = false; - for (const line of contents.split("\n")) { - answeredHello = this.#receiveFrame(line) || answeredHello; - if (this.#closed) return; - } - if ( - this.daemon.closeAfter === "hello" || - (!answeredHello && this.daemon.closeAfter === "request") - ) - this.#shutDown(true); - } - - #receiveFrame(line: string): boolean { - if (line.trim() === "") return false; - const frame = JSON.parse(line) as { - id?: unknown; - type?: unknown; - payload?: { credential?: string }; - }; - if (typeof frame.type === "string") this.daemon.frameTypes.push(frame.type); - if (frame.type === "status.get") this.#answerStatus(frame.id); - if (frame.type !== "hello" || typeof frame.id !== "number") return false; - this.#answerHello(frame.id, frame.payload?.credential); - return true; - } - - #answerStatus(id: unknown): void { - const { closeAfter } = this.daemon; - if (closeAfter !== "stopping" && closeAfter !== "starting") return; - const reply = - closeAfter === "stopping" - ? { - id, - ok: false, - error: { code: "DAEMON_STOPPING", message: "Daemon is stopping", details: {} }, - } - : { - id, - ok: true, - payload: { - daemon: { - health: "starting", - mode: "worker", - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, - }, - }; - for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); - } - - #answerHello(id: number, credential: string | undefined): void { - if (this.daemon.rejectCredential && credential !== undefined) { - this.daemon.rejectedCredentials += 1; - const reply = { - id: id, - ok: false, - error: { - code: "ADMIN_AUTHENTICATION_FAILED", - message: "Rejected credential", - details: {}, - }, - }; - for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); - this.#shutDown(false); - return; - } - this.daemon.helloReplies += 1; - const reply = { - id: id, - ok: true, - payload: { - protocolVersion: PROTOCOL_VERSION_RANGE.max, - daemonProtocolRange: PROTOCOL_VERSION_RANGE, - version: "0.0.0-test", - role: "admin", - principal: "test-requester", - }, - }; - for (const listener of this.#dataListeners) listener(`${JSON.stringify(reply)}\n`); - } - - async close(): Promise { - this.daemon.clientCloses += 1; - this.#shutDown(false); - } - - #shutDown(killedByDaemon: boolean): void { - if (this.#closed) return; - this.#closed = true; - if (killedByDaemon) this.daemon.deaths += 1; - for (const listener of this.#closeListeners) listener(); - } -} - -function sequence(): IdGenerator { - let next = 0; - return { generate: () => `id${next++}` }; -} - -async function startTestDaemon(): Promise<{ socketPath: string; daemon: DaemonServer }> { - const directory = await mkdtemp(join(tmpdir(), "simlock-cli-smoke-")); - temporaryDirectories.push(directory); - const socketPath = join(directory, "daemon.sock"); - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const filesystem = new MemoryFilesystem(); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const config = testConfig(); - const wiring = testComponentWiring({ - clock: clock, - drivers: [driver], - eventBus: eventBus, - registry: registry, - }); - const engine = createTestEngine({ - ...wiring, - clock, - config, - drivers: [driver], - eventBus, - idGenerator: sequence(), - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - }); - const reaper = new CleanupReaper({ - clock, - config, - eventBus, - executor: engine.cleanup, - filesystem, - registry, - }); - const auth = createAuth({ - clock, - config: { enabled: false, host: "127.0.0.1", port: 4700 }, - dataDirectory: "/", - database: (await openMemoryStateDatabase([tokensOwner])).database, - filesystem, - idGenerator: sequence(), - logger: new NoopLogger(), - keySetFetcher: new FakeKeySetFetcher(), - secrets: new CryptoTokenSecrets(), - }); - const daemon = new DaemonServer({ - capacity: engine, - warmPool: engine, - deviceModes: engine, - catalog: engine, - instanceId: "instance-test", - clock, - components: wiring.components, - config, - defaultRequesterId: "test-process", - eventBus, - eventHistory: ringEventHistory(eventBus), - host: new DaemonEndpointHost({ - connector: new NodeIpcTransport(), - endpoint: socketPath, - filesystem: new NodeFilesystem(), - listenerFactory: new NodeIpcTransport(), - }), - hostFacts: () => ({ arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }), - leases: engine, - queue: engine, - reaper, - registry, - resolveRole: { resolve: () => "admin" }, - tokens: auth.tokens, - version: "test", - }); - runningDaemons.push(daemon); - await daemon.start(); - return { socketPath, daemon }; -} - -/** why: Bare-minimum ports for exercising `buildCliEnvironment`'s real `validateConfig` (B9) without - * a daemon connection -- `config set` never calls `connectAdmin`, so `ipc`/`launcher` are inert - * placeholders. */ -function realCliEnvironmentPorts( - filesystem: Filesystem = new MemoryFilesystem(), -): CliEnvironmentPorts { - return { - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - ipc: new MemoryIpcTransport(), - launcher: new FakeDaemonLauncher(), - daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }; -} - -/** - * why: A real `DaemonServer` wired against in-memory ports (`MemoryFilesystem` + `MemoryIpcTransport` - * shared with the CLI environment under test), with the genuine credential handshake - * (`createAuth`) instead of `startTestDaemon`'s - * `resolveRole: { resolve: () => "admin" }` shortcut -- B1 and B2 both hinge on that handshake - * actually running. - */ -async function startInMemoryDaemon(options: { - readonly filesystem: MemoryFilesystem; - readonly ipcTransport: MemoryIpcTransport; - readonly socketPath: string; - readonly adminTokenPath: string; -}): Promise<{ daemon: DaemonServer; adminSecret: Auth["adminSecret"] }> { - const { adminTokenPath, filesystem, ipcTransport, socketPath } = options; - const clock = new FakeClock(1_000); - const eventBus = new EventBus(clock); - const registry = await loadTestRegistry({ - clock, - eventBus, - idGenerator: sequence(), - }); - const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const config = testConfig(); - const wiring = testComponentWiring({ - clock: clock, - drivers: [driver], - eventBus: eventBus, - registry: registry, - }); - const engine = createTestEngine({ - ...wiring, - clock, - config, - drivers: [driver], - eventBus, - idGenerator: sequence(), - registry, - state: testStateOf(registry), - systemStats: new FakeSystemStats({ - cpuCount: 8, - totalRamBytes: 32 * gibibyte, - }), - }); - const reaper = new CleanupReaper({ - clock, - config, - eventBus, - executor: engine.cleanup, - filesystem, - registry, - }); - const auth = createAuth({ - clock, - config: { enabled: false, host: "127.0.0.1", port: 4700 }, - dataDirectory: dirname(adminTokenPath), - database: (await openMemoryStateDatabase([tokensOwner])).database, - filesystem, - idGenerator: sequence(), - logger: new NoopLogger(), - keySetFetcher: new FakeKeySetFetcher(), - secrets: new CryptoTokenSecrets(), - }); - const daemon = new DaemonServer({ - adminSecret: auth.adminSecret, - capacity: engine, - warmPool: engine, - deviceModes: engine, - catalog: engine, - instanceId: "instance-test", - clock, - components: wiring.components, - config, - defaultRequesterId: "test-process", - eventBus, - eventHistory: ringEventHistory(eventBus), - host: new DaemonEndpointHost({ - connector: ipcTransport, - endpoint: socketPath, - filesystem, - listenerFactory: ipcTransport, - }), - hostFacts: () => ({ arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }), - leases: engine, - queue: engine, - reaper, - registry, - resolveRole: { resolve: (hello) => auth.resolveHelloRole(hello) }, - tokens: auth.tokens, - version: "test", - }); - runningDaemons.push(daemon); - await daemon.start(); - return { daemon, adminSecret: auth.adminSecret }; -} - -function testConfig(): Config { - return configWithDefaults({ - capacity: { - strategy: "resource", - config: { - limits: { - android: { maxDevices: 1, maxRunning: 1 }, - ios: { maxDevices: 1, maxRunning: 1 }, - maxRunning: 2, - }, - ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, - }, - }, - eventBuffer: { capacity: 100 }, - health: { enabled: false }, - http: { enabled: false, port: 4700 }, - idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, - lease: { defaultTtlMs: 60_000, maxTtlMs: 3_600_000 }, - warmPool: { maxConcurrentBoots: 1 }, - }); -} - -describe("simlock list --requests", () => { - it("prints a starting request without a place in the queue, one on a worker with that worker, and a future timestamp as waiting 0s", async () => { - const output = outputCapture(); - const asked: unknown[] = []; - const client = fakeClient({ - list: async (input) => { - asked.push(input); - return [ - { - createdAt: -12_000, - id: "req_1", - requesterId: "agent-b", - spec: { imageTag: "google_apis", model: "Pixel 8", platform: "android" }, - stage: "starting", - }, - { - createdAt: -3_400, - id: "req_2", - queuePosition: 2, - requesterId: "local-agent", - spec: { model: "iPhone 16", osVersion: "18.4", platform: "ios" }, - stage: "queued", - workerId: "wrk_1", - }, - { - // why: Stamped by a worker whose clock runs ahead of this one. - createdAt: 5_000, - id: "req_3", - queuePosition: 1, - requesterId: "skewed-agent", - spec: { model: "iPhone 16", platform: "ios" }, - stage: "queued", - workerId: "wrk_2", - }, - { - createdAt: -1_000, - id: "req_4", - queuePosition: 3, - requesterId: "baseline-agent", - spec: { baseline: "checkout", platform: "ios" }, - stage: "queued", - }, - ]; - }, - }); - - await expect( - runCli(["list", "--requests"], output.environmentWith({ connectAdmin: async () => client })), - ).resolves.toBe(0); - - expect(asked).toEqual([{ kind: "requests" }]); - expect(output.stdout).toBe( - "Request req_1: agent-b, android Pixel 8 image tag google_apis, starting, waiting 12s\n" + - "Request req_2: local-agent on wrk_1, ios iPhone 16 18.4, queued at 2, waiting 3s\n" + - "Request req_3: skewed-agent on wrk_2, ios iPhone 16, queued at 1, waiting 0s\n" + - "Request req_4: baseline-agent, ios baseline checkout, queued at 3, waiting 1s\n", - ); - }); - - describe("the reason a queued request waits", () => { - const queued = ( - reason: string | undefined, - platform: "ios" | "android" = "ios", - ): Record => ({ - createdAt: 0, - id: "req_1", - queuePosition: 1, - requesterId: "agent", - spec: { model: "Device", platform }, - stage: "queued", - ...(reason === undefined ? {} : { reason }), - }); - - async function printed(entry: Record): Promise { - const output = outputCapture(); - const client = fakeClient({ list: async () => [entry] as never }); - await runCli( - ["list", "--requests"], - output.environmentWith({ connectAdmin: async () => client }), - ); - return output.stdout.trimEnd(); - } - - it.each([ - ["behind-earlier-request", "ios", "behind an earlier request"], - ["device-limit", "ios", "full: device limit"], - ["ram-budget", "ios", "full: RAM budget"], - ["memory-pressure", "ios", "full: host memory pressure"], - ["global-running-limit", "ios", "full: running limit"], - ["platform-running-limit", "ios", "full: running limit for iOS"], - ["platform-running-limit", "android", "full: running limit for Android"], - ["waiting-for-device", "ios", "a matching device is starting"], - ["no-free-physical", "ios", "every matching device is leased or absent"], - ] as const)( - "ends the line of a queued request with %s on %s as ' — %s'", - async (reason, platform, text) => { - expect(await printed(queued(reason, platform))).toBe( - `Request req_1: agent, ${platform} Device, queued at 1, waiting 0s — ${text}`, - ); - }, - ); - - it("ends the line of a queued request with no reason as it ended before", async () => { - expect(await printed(queued(undefined))).toBe( - "Request req_1: agent, ios Device, queued at 1, waiting 0s", - ); - }); - - it("has a text for every reason the contract's enum lists", async () => { - const field = (waitingRequestSchema.shape as Record)["reason"]; - expect(field).toBeDefined(); - const listed = (field as { unwrap(): { options: readonly string[] } }).unwrap().options; - - expect(listed.length).toBeGreaterThan(0); - for (const reason of listed) { - const line = await printed(queued(reason)); - expect(line).toMatch(/ — [a-z]/); - expect(line).not.toContain(reason); - } - }); - }); - - describe("the reason each worker cannot take a request on a gateway", () => { - const gatewayEntry = ( - workers: readonly Record[], - platform: "ios" | "android" = "ios", - ): Record => ({ - createdAt: 0, - id: "req_1", - queuePosition: 1, - requesterId: "ci", - spec: { model: "Device", platform }, - stage: "queued", - workers, - }); - - async function printedGateway( - entries: readonly Record[], - known: readonly { id: string; label?: string }[] = [], - ): Promise<{ stdout: string; listed: number }> { - const output = outputCapture(); - let listed = 0; - const client = fakeClient({ - list: async () => entries as never, - listWorkers: () => { - listed += 1; - return Promise.resolve({ - workers: known.map((worker) => ({ - catalog: [], - connection: "connected" as const, - devices: [], - drained: false, - lastSeenAt: 1, - leases: [], - ...worker, - })), - } as never); - }, - }); - await runCli( - ["list", "--requests"], - output.environmentWith({ connectAdmin: async () => client }), - ); - return { listed, stdout: output.stdout }; - } - - it("prints one indented line per worker under the request, named by its label, else its id", async () => { - const { stdout } = await printedGateway( - [ - gatewayEntry([ - { reason: "drained", workerId: "wrk_a" }, - { capacityReason: "platform-running-limit", reason: "full", workerId: "wrk_b" }, - { reason: "runtime-missing", workerId: "wrk_c" }, - { reason: "busy", workerId: "wrk_d" }, - ]), - ], - [{ id: "wrk_a", label: "mac-mini-1" }, { id: "wrk_b" }, { id: "wrk_c", label: "lab" }], - ); - - expect(stdout).toBe( - "Request req_1: ci, ios Device, queued at 1, waiting 0s\n" + - " mac-mini-1: drained\n" + - " wrk_b: full: running limit for iOS\n" + - " lab: oS not installed\n" + - " wrk_d: busy with its own queue\n", - ); - }); - - it.each([ - ["disconnected", "ios", undefined, "not connected"], - ["incompatible", "ios", undefined, "protocol mismatch"], - ["not-ready", "ios", undefined, "not ready yet"], - ["unhealthy", "ios", undefined, "not running"], - ["updating", "ios", undefined, "updating"], - ["drained", "ios", undefined, "drained"], - ["busy", "ios", undefined, "busy with its own queue"], - ["no-driver", "ios", undefined, "no iOS support"], - ["no-driver", "android", undefined, "no Android support"], - ["unknown-model", "ios", undefined, "model not offered"], - ["runtime-missing", "ios", undefined, "oS not installed"], - ["capability-missing", "ios", undefined, "missing capability"], - ["full", "ios", undefined, "full"], - ["full", "ios", "device-limit", "full: device limit"], - ["full", "ios", "ram-budget", "full: RAM budget"], - ["full", "ios", "memory-pressure", "full: host memory pressure"], - ["full", "ios", "global-running-limit", "full: running limit"], - ["full", "ios", "platform-running-limit", "full: running limit for iOS"], - ["full", "android", "platform-running-limit", "full: running limit for Android"], - ["no-free-physical", "ios", undefined, "every matching device is leased or absent"], - ["not-chosen", "ios", undefined, "not chosen this time"], - ["no-baseline", "ios", undefined, "does not hold the baseline"], - ["baseline-stale", "ios", undefined, "baseline is stale"], - ["baseline-mismatch", "ios", undefined, "baseline is for another device"], - ] as const)( - "prints %s on %s with capacity reason %s as ' wrk_a: %s'", - async (reason, platform, capacityReason, text) => { - const { stdout } = await printedGateway([ - gatewayEntry( - [ - { - ...(capacityReason === undefined ? {} : { capacityReason }), - reason, - workerId: "wrk_a", - }, - ], - platform, - ), - ]); - - expect(stdout.split("\n")[1]).toBe(` wrk_a: ${text}`); - }, - ); - - it("asks worker.list once for names, and not at all when no entry has workers", async () => { - const gateway = await printedGateway([ - gatewayEntry([{ reason: "drained", workerId: "wrk_a" }]), - { ...gatewayEntry([{ reason: "busy", workerId: "wrk_a" }]), id: "req_2" }, - ]); - const plain = await printedGateway([{ ...gatewayEntry([]), workers: undefined }]); - - expect(gateway.listed).toBe(1); - expect(plain.listed).toBe(0); - }); - - it("prints a worker's own entry with its reason after ' — ' and no indented lines", async () => { - const { stdout } = await printedGateway([ - { - ...gatewayEntry([]), - reason: "ram-budget", - workers: undefined, - workerId: "wrk_1", - }, - ]); - - expect(stdout).toBe( - "Request req_1: ci on wrk_1, ios Device, queued at 1, waiting 0s — full: RAM budget\n", - ); - }); - - it("has a text for every gateway reason the contract's enum lists", async () => { - const field = (waitingRequestSchema.shape as Record)["workers"]; - expect(field).toBeDefined(); - const element = ( - field as { unwrap(): { element: { shape: { reason: { options: readonly string[] } } } } } - ).unwrap().element; - const listed = element.shape.reason.options; - - expect(listed.length).toBeGreaterThan(0); - for (const reason of listed) { - const { stdout } = await printedGateway([gatewayEntry([{ reason, workerId: "wrk_a" }])]); - const line = stdout.split("\n")[1] ?? ""; - expect(line).toMatch(/^ {2}wrk_a: [a-zA-Z]/); - expect(line).not.toContain("-"); - } - }); - }); - - it.each([["--devices"], ["--leases"], ["--rules"]])( - "exits 2 with USAGE for --requests beside %s, without asking the daemon", - async (other) => { - const output = outputCapture(); - let listed = 0; - const client = fakeClient({ - list: () => { - listed += 1; - return Promise.resolve([]); - }, - }); - - await expect( - runCli( - ["list", "--requests", other], - output.environmentWith({ connectAdmin: async () => client }), - ), - ).resolves.toBe(2); - expect(listed).toBe(0); - expect(JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "")).toMatchObject({ - error: { code: "USAGE" }, - }); - }, - ); -}); - -describe("simlock lease: a request names a model, a class, or nothing", () => { - const grant = { - device: { - physical: false, - driverDeviceId: "dev_1", - id: "dev_1", - mode: "full" as const, - spec: { physical: false, model: "iPhone 17", osVersion: "26.5", platform: "ios" as const }, - }, - environment: {}, - lease: { - deviceId: "dev_1", - grantedAt: 0, - id: "lse_1", - lastRenewedAt: 0, - idChosenByRequester: false, - ownerId: "test-requester", - requesterId: "test-requester", - ttlDeadline: 60_000, - ttlMs: 60_000, - }, - timing: { - estimatedBootMs: 0, - estimatedProvisionMs: 0, - estimatedReadyMs: 0, - estimatedReclaimMs: 0, - }, - }; - - async function sentFor(args: readonly string[]): Promise | undefined> { - let sent: Record | undefined; - const client = fakeClient({ - requestLease: (input) => { - sent = input as unknown as Record; - return Promise.resolve(grant); - }, - }); - const output = outputCapture(); - await runCli( - ["lease", ...args, "--detach"], - output.environmentWith({ connectAdmin: async () => client }), - ); - return sent; - } - - it.each([ - { - name: "neither model nor class for a lease with only a platform", - args: ["--platform", "ios"], - present: { platform: "ios" }, - absent: ["model", "class"], - }, - { - name: "the class for --class, and no model", - args: ["--platform", "ios", "--class", "phone"], - present: { class: "phone", platform: "ios" }, - absent: ["model"], - }, - { - name: "both fields for --class beside --device, leaving the refusal to the daemon", - args: ["--platform", "ios", "--class", "phone", "--device", "iPhone 16"], - present: { class: "phone", model: "iPhone 16", platform: "ios" }, - absent: [], - }, - ])("sends $name", async ({ args, present, absent }) => { - const sent = await sentFor(args); - - expect(sent).toMatchObject(present); - for (const key of absent) expect(sent).not.toHaveProperty(key); - }); - - it("names --class in the lease usage text, with --device optional", async () => { - const output = outputCapture(); - - await runCli(["lease", "--help"], output.environmentWith({})); - - expect(output.stdout).toBe( - "Usage: simlock lease --platform [--device | --class ]\n" + - " [--os ] [--mode ] [--image-tag ] [--agent-id ]\n" + - " [--except ] [--baseline ]\n" + - " [--timeout ] [--lease-id ] [--physical]\n" + - " [--no-wait] [--detach] [--ttl ] [--allow-download]\n" + - " [--meta ]... [--export-env] [--bind-pid ]\n", - ); - }); - - it.each([ - ["--device", "lease --device must not be empty"], - ["--class", "lease --class must not be empty"], - ])("exits 2 with USAGE for an empty %s, before connecting", async (flag, message) => { - const output = outputCapture(); - let connected = false; - - const exitCode = await runCli( - ["lease", "--platform", "ios", flag, ""], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient({}); - }, - }), - ); - - expect(exitCode).toBe(2); - expect(connected).toBe(false); - expect(output.stderr).toContain(message); - }); - - it("prints the class of a waiting request that names no model, phone when it names none", async () => { - const output = outputCapture(); - const client = fakeClient({ - list: async () => [ - { - createdAt: 0, - id: "req_1", - queuePosition: 1, - requesterId: "a", - spec: { class: "watch", platform: "ios" }, - stage: "queued", - }, - { - createdAt: 0, - id: "req_2", - queuePosition: 2, - requesterId: "b", - spec: { platform: "android" }, - stage: "queued", - }, - ], - }); - - await runCli( - ["list", "--requests"], - output.environmentWith({ connectAdmin: async () => client }), - ); - - expect(output.stdout).toBe( - "Request req_1: a, ios class watch, queued at 1, waiting 0s\n" + - "Request req_2: b, android class phone, queued at 2, waiting 0s\n", - ); - }); - - it("prints physical in a waiting physical request's line, and nothing for a simulator request", async () => { - const output = outputCapture(); - const client = fakeClient({ - list: async () => [ - { - createdAt: 0, - id: "req_1", - queuePosition: 1, - requesterId: "a", - spec: { model: "iPhone 15 Pro", physical: true, platform: "ios" }, - stage: "queued", - }, - { - createdAt: 0, - id: "req_2", - queuePosition: 1, - requesterId: "b", - spec: { model: "iPhone 16", platform: "ios" }, - stage: "queued", - }, - ], - }); - - await runCli( - ["list", "--requests"], - output.environmentWith({ connectAdmin: async () => client }), - ); - - expect(output.stdout).toBe( - "Request req_1: a, ios physical iPhone 15 Pro, queued at 1, waiting 0s\n" + - "Request req_2: b, ios iPhone 16, queued at 1, waiting 0s\n", - ); - }); - - it("prints a waiting request's OS range as typed", async () => { - const output = outputCapture(); - const client = fakeClient({ - list: async () => [ - { - createdAt: 0, - id: "req_1", - queuePosition: 1, - requesterId: "a", - spec: { osVersion: ">=18 <26", platform: "ios" }, - stage: "queued", - }, - ], - }); - - await runCli( - ["list", "--requests"], - output.environmentWith({ connectAdmin: async () => client }), - ); - - expect(output.stdout).toBe( - "Request req_1: a, ios class phone >=18 <26, queued at 1, waiting 0s\n", - ); - }); -}); - -describe("CLI: a worker's capabilities", () => { - const platform = { - atRamBudget: false, - limit: 2, - maxRunning: 2, - overLimit: false, - reserved: 0, - running: 0, - used: 0, - warm: 0, - }; - const worker = (overrides: Record = {}) => ({ - capacity: { - android: platform, - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: platform, - }, - catalog: [], - connection: "connected" as const, - devices: [], - drained: false, - id: "wrk_1", - label: "mac-mini", - lastSeenAt: 1_000, - leases: [], - ...overrides, - }); - const listed = async (workers: unknown[]): Promise => { - const output = outputCapture(); - const environment = output.environmentWith({ - connectAdmin: async () => - fakeClient({ listWorkers: () => Promise.resolve({ workers } as never) }), - }); - await runCli(["worker", "list"], environment); - return output.stdout; - }; - - it("worker list prints ` -- agent-device available`, ` -- agent-device off` and ` -- agent-device unavailable: ` on the worker's line", async () => { - const shown = await listed([ - worker({ capabilities: [{ name: "agent-device", state: "available" }] }), - worker({ capabilities: [{ name: "agent-device", state: "off" }], id: "wrk_2" }), - worker({ - capabilities: [ - { - name: "agent-device", - reason: "found 0.20.1, need 0.21.28 or later", - state: "unavailable", - }, - ], - id: "wrk_3", - }), - worker({ - capabilities: [{ name: "agent-device", reason: "starting", state: "unavailable" }], - id: "wrk_4", - }), - ]); - - expect(shown).toContain("wrk_1 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s)"); - expect(shown).toMatch(/^wrk_1 .* -- agent-device available$/m); - expect(shown).toMatch(/^wrk_2 .* -- agent-device off$/m); - expect(shown).toMatch( - /^wrk_3 .* -- agent-device unavailable: found 0\.20\.1, need 0\.21\.28 or later$/m, - ); - expect(shown).toMatch(/^wrk_4 .* -- agent-device unavailable: starting$/m); - }); - - it("simlock worker list shows each worker's memory step as the end of its first line, and nothing at none, with no field, or for a disconnected worker", async () => { - const withStep = ( - id: string, - step: "none" | "warming-paused" | "new-devices-paused", - extra = {}, - ) => - worker({ - capabilities: [{ name: "agent-device", state: "available" }], - capacity: { ...worker().capacity, memoryPressure: { step } }, - id, - ...extra, - }); - const shown = await listed([ - withStep("wrk_1", "warming-paused"), - withStep("wrk_2", "new-devices-paused"), - withStep("wrk_3", "none"), - worker({ id: "wrk_4" }), - withStep("wrk_5", "new-devices-paused", { connection: "disconnected" }), - ]); - - const lines = shown.trimEnd().split("\n"); - const first = (id: string) => lines.find((line) => line.startsWith(`${id} `)); - expect(first("wrk_1")).toBe( - "wrk_1 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available -- memory: warming paused", - ); - expect(first("wrk_2")).toBe( - "wrk_2 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available -- memory: new devices paused", - ); - expect(first("wrk_3")).toBe( - "wrk_3 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available", - ); - expect(first("wrk_4")).toBe("wrk_4 (mac-mini): connected -- ios 0/2, android 0/2, 0 lease(s)"); - expect(first("wrk_5")).toBe( - "wrk_5 (mac-mini): disconnected -- ios 0/2, android 0/2, 0 lease(s) -- agent-device available", - ); - }); - - it("worker list prints two entries in the order the worker sent them", async () => { - const shown = await listed([ - worker({ - capabilities: [ - { name: "zeta", state: "off" }, - { name: "agent-device", state: "available" }, - ], - }), - ]); - - expect(shown).toMatch(/^wrk_1 .* -- zeta off -- agent-device available$/m); - }); - - it("worker list prints nothing for a worker whose list is absent", async () => { - const shown = await listed([worker()]); - - expect(shown).toMatch(/^wrk_1 .* 0 lease\(s\)$/m); - }); - - it("worker list puts the suffix on the worker's line, before its install lines", async () => { - const shown = await listed([ - worker({ - capabilities: [{ name: "agent-device", state: "available" }], - installs: [ - { component: "26.4", platform: "ios", since: 1_000, state: "downloading", waiters: 2 }, - ], - }), - ]); - - expect(shown).toContain( - "-- agent-device available\n Install ios 26.4: downloading for 0s, 2 waiters", - ); - }); - - it("status against a gateway prints the same suffix in its workers block", async () => { - const output = outputCapture(); - const status = { - capacity: { - android: platform, - global: { maxRunning: 2, overLimit: false, reserved: 0, running: 0, warm: 0 }, - ios: platform, - }, - daemon: { - health: "running" as const, - mode: "gateway" as const, - version: "0.0.0-test", - update: null, - lastUpdateFailure: null, - latestVersion: null, - }, - devices: [], - host: { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] }, - leases: [], - queueDepth: 0, - workers: [ - worker({ - capabilities: [{ name: "agent-device", reason: "starting", state: "unavailable" }], - }), - ], - }; - const environment = output.environmentWith({ - connectAdmin: async () => fakeClient({ getStatus: () => Promise.resolve(status as never) }), - }); - - await runCli(["status"], environment); - - expect(output.stdout).toMatch(/^wrk_1 .* -- agent-device unavailable: starting$/m); - }); -}); - -function usageAnswer( - window: { readonly from: number; readonly to: number }, - overrides: Partial = {}, -): UsageOutput { - const samples = { count: 0, max: null, p50: null, p95: null }; - const figures = { - boot: samples, - bySource: { booted: 0, provisioned: 0, warm: 0 }, - declined: 0, - failures: { byEvent: {} }, - granted: 0, - held: samples, - incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, - provisioning: samples, - queue: { meanDepth: null, peakDepth: null }, - rejected: { byReason: {}, total: 0 }, - requests: 0, - turnaround: samples, - utilisation: { slots: { max: null, mean: null, peak: null } }, - wait: samples, - }; - return { - bucketMs: 60_000, - coversFrom: window.from, - partial: false, - platforms: { android: figures, ios: figures }, - requesters: [], - series: [], - totals: figures, - window, - workers: [], - ...overrides, - }; -} - -describe("CLI: stats", () => { - const NOW = Date.parse("2026-10-05T12:00:00.000Z"); - const HOUR = 3_600_000; - - function statsRun(argv: readonly string[], usage: SimlockAdminClient["usage"]) { - const output = outputCapture(); - const asked: { from: number; to: number; includeLedger?: boolean | undefined }[] = []; - const run = runCli( - ["stats", ...argv], - output.environmentWith({ - adminTokenFromEnv: "admin-token", - clock: new FakeClock(NOW), - connectAdmin: async (resolveCredential) => { - await resolveCredential(); - return fakeClient({ - usage: async (window) => { - asked.push(window); - return usage(window); - }, - }); - }, - }), - ); - return { asked, output, run }; - } - - it("simlock stats words each refusal as the Inputs table says, exits 2, and asks nothing of the daemon", async () => { - for (const argv of [["--since", "1h"], ["--from", "2026-10-05T10:00:00Z"], ["--json"]]) { - const { output, run } = statsRun(argv, async (window) => usageAnswer(window)); - await expect(run, argv.join(" ")).resolves.toBe(0); - expect(output.stderr, argv.join(" ")).not.toContain("USAGE"); - } - for (const [argv, message] of [ - [ - ["--since", "1h", "--from", "2026-10-05T10:00:00Z"], - "stats takes --since or --from, not both", - ], - [["--to", "2026-10-04T10:00:00Z"], "--to needs --from"], - [ - ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-03T10:00:00Z"], - "--from must be earlier than --to, or than now when --to is left out", - ], - [ - ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T10:00:00Z"], - "--from must be earlier than --to, or than now when --to is left out", - ], - [ - ["--from", "2026-10-05T13:00:00Z"], - "--from must be earlier than --to, or than now when --to is left out", - ], - [["--since", "soon"], "Invalid duration: soon"], - [["--from", "2026-10-15"], "Invalid time: 2026-10-15 (use ISO 8601 with Z or an offset)"], - [ - ["--from", "2026-10-04T10:00:00Z", "--to", "10/15/2026"], - "Invalid time: 10/15/2026 (use ISO 8601 with Z or an offset)", - ], - ] as const) { - const { asked, output, run } = statsRun(argv, async (window) => usageAnswer(window)); - - await expect(run, argv.join(" ")).resolves.toBe(2); - expect(output.stderr, argv.join(" ")).toContain('"code":"USAGE"'); - expect(output.stderr, argv.join(" ")).toContain(`"message":"${message}"`); - expect(asked, argv.join(" ")).toEqual([]); - expect(output.stdout).toBe(""); - } - }); - - it("simlock stats --json prints the operation's output unchanged", async () => { - const answer = usageAnswer( - { from: NOW - HOUR, to: NOW }, - { - partial: true, - requesters: [ - { granted: 1, heldTotalMs: 5, id: "tok_a", label: "ci", rejected: 0, requests: 1 }, - ], - }, - ); - const { output, run } = statsRun(["--since", "1h", "--json"], async () => answer); - - await expect(run).resolves.toBe(0); - - expect(JSON.parse(output.stdout)).toEqual(answer); - expect(output.stdout.endsWith("\n")).toBe(true); - }); - - it("asks for the last 24 hours by default, and for --since up to now", async () => { - const byDefault = statsRun([], async (window) => usageAnswer(window)); - const since = statsRun(["--since", "90m"], async (window) => usageAnswer(window)); - - await byDefault.run; - await since.run; - - expect(byDefault.asked).toEqual([{ from: NOW - 24 * HOUR, to: NOW }]); - expect(since.asked).toEqual([{ from: NOW - 90 * 60_000, to: NOW }]); - }); - - it("asks for the ISO window --from and --to name, and for --from up to now", async () => { - const both = statsRun( - ["--from", "2026-10-04T10:00:00Z", "--to", "2026-10-04T12:00:00Z"], - async (window) => usageAnswer(window), - ); - const open = statsRun(["--from", "2026-10-05T10:00:00Z"], async (window) => - usageAnswer(window), - ); - - await both.run; - await open.run; - - expect(both.asked).toEqual([ - { from: Date.parse("2026-10-04T10:00:00Z"), to: Date.parse("2026-10-04T12:00:00Z") }, - ]); - expect(open.asked).toEqual([{ from: Date.parse("2026-10-05T10:00:00Z"), to: NOW }]); - }); - - it("prints the figures as a table, with the partial note above them", async () => { - const { output, run } = statsRun(["--since", "30d"], async (window) => - usageAnswer(window, { coversFrom: NOW - 60_000, partial: true }), - ); - - await expect(run).resolves.toBe(0); - - const lines = output.stdout.split("\n"); - expect(lines[0]).toBe("Usage from 2026-09-05T12:00:00.000Z to 2026-10-05T12:00:00.000Z"); - expect(lines[1]).toContain("Figures cover from 2026-10-05T11:59:00.000Z"); - expect(lines).toContain("Totals"); - }); - - it.each([ - ["", ["--since", "30d"]], - [" with --csv", ["--since", "30d", "--csv"]], - ])( - "prints the HISTORY_NOT_KEPT message on stderr, nothing on stdout, and exits with its code when the window ends before the history%s", - async (_name, args) => { - const { output, run } = statsRun(args, async () => { - throw fromWireError( - "HISTORY_NOT_KEPT", - "The history does not reach back to the end of that window; its figures start at 2026-10-05T11:59:00.000Z.", - { oldestTs: NOW - 60_000 }, - ); - }); - - await expect(run).resolves.toBe(12); - - expect(output.stdout).toBe(""); - expect(output.stderr).toContain('"code":"HISTORY_NOT_KEPT"'); - expect(output.stderr).toContain("its figures start at 2026-10-05T11:59:00.000Z"); - }, - ); - - const LEDGER_HEADER = - "requestId,requester,label,platform,model,workerId,source,requestedAt,grantedAt,rejectedAt,endedAt,waitMs,heldMs,turnaroundMs,outcome,probe,reason,endReason"; - const TRUNCATION_NOTICE = - "The ledger holds the newest 100000 requests of the window; ask for a shorter window for the rest."; - - it("simlock stats --csv prints the header row and one row per ledger entry with times as ISO 8601, durations as whole milliseconds (400, not 400.0), probe as true or false, and asks usage.get with includeLedger: true", async () => { - const requests = [ - { - endReason: "explicit", - endedAt: Date.parse("2026-10-05T11:09:30.000Z"), - grantedAt: Date.parse("2026-10-05T11:01:02.400Z"), - heldMs: 507_600.4, - label: "ci-runner", - model: "iPhone 16", - outcome: "granted" as const, - platform: "ios", - probe: false, - requestId: "req_8f2", - requestedAt: Date.parse("2026-10-05T11:01:02.000Z"), - requester: "tok_ci", - source: "warm", - turnaroundMs: 508_000, - waitMs: 400, - }, - { - outcome: "open" as const, - platform: "android", - probe: true, - requestId: "req_9", - requestedAt: Date.parse("2026-10-05T11:02:00.000Z"), - requester: "gw:gw-1:tok_a", - }, - ]; - const { asked, output, run } = statsRun(["--since", "1h", "--csv"], async (window) => - usageAnswer(window, { requests }), - ); - - await expect(run).resolves.toBe(0); - - expect(asked[0]?.includeLedger).toBe(true); - expect(output.stdout.split("\r\n")).toEqual([ - LEDGER_HEADER, - "req_8f2,tok_ci,ci-runner,ios,iPhone 16,,warm,2026-10-05T11:01:02.000Z,2026-10-05T11:01:02.400Z,,2026-10-05T11:09:30.000Z,400,507600,508000,granted,false,,explicit", - "req_9,gw:gw-1:tok_a,,android,,,,2026-10-05T11:02:00.000Z,,,,,,,open,true,,", - "", - ]); - }); - - it("simlock stats --csv writes the truncation notice on stderr and exits 0 when ledgerTruncated, and writes no notice otherwise", async () => { - const truncated = statsRun(["--csv"], async (window) => - usageAnswer(window, { ledgerTruncated: true, requests: [] }), - ); - const whole = statsRun(["--csv"], async (window) => usageAnswer(window, { requests: [] })); - - await expect(truncated.run).resolves.toBe(0); - await expect(whole.run).resolves.toBe(0); - - expect(truncated.output.stderr).toBe(`${JSON.stringify({ notice: TRUNCATION_NOTICE })}\n`); - expect(truncated.output.stdout.startsWith(LEDGER_HEADER)).toBe(true); - expect(whole.output.stderr).toBe(""); - expect(whole.output.stdout.startsWith(LEDGER_HEADER)).toBe(true); - }); - - it("simlock stats refuses --csv with --json as a usage error, exit 2", async () => { - const accepted = statsRun(["--csv"], async (window) => usageAnswer(window, { requests: [] })); - const both = statsRun(["--csv", "--json"], async (window) => usageAnswer(window)); - - await expect(accepted.run).resolves.toBe(0); - await expect(both.run).resolves.toBe(2); - - expect(both.output.stderr).toContain('"code":"USAGE"'); - expect(both.output.stderr).toContain('"message":"stats takes --csv or --json, not both"'); - expect(both.asked).toEqual([]); - expect(both.output.stdout).toBe(""); - }); - - it("closes the connection after the answer, and after a failure", async () => { - let closed = 0; - const output = outputCapture(); - const connect = (usage: SimlockAdminClient["usage"]) => async () => - fakeClient({ - close: async () => { - closed += 1; - }, - usage, - }); - - await runCli( - ["stats"], - output.environmentWith({ connectAdmin: connect(async (window) => usageAnswer(window)) }), - ); - await runCli( - ["stats"], - output.environmentWith({ - connectAdmin: connect(async () => { - throw new Error("boom"); - }), - }), - ); - - expect(closed).toBe(2); - }); - - it("prints its usage on --help without connecting", async () => { - const output = outputCapture(); - let connected = false; - - const exitCode = await runCli( - ["stats", "--help"], - output.environmentWith({ - connectAdmin: async () => { - connected = true; - return fakeClient(); - }, - }), - ); - - expect(exitCode).toBe(0); - expect(connected).toBe(false); - expect(output.stdout).toBe( - "Usage: simlock stats [--since | --from [--to ]] [--json | --csv]\n", - ); - }); -}); - -describe("CLI: daemon install (#830)", () => { - const home = "/Users/me"; - const nodePath = "/opt/node/bin/node"; - const mainPath = "/opt/simlock/dist/daemon/main.js"; - - async function install(options: { - readonly argv: readonly string[]; - readonly role: "worker" | "gateway"; - readonly root: string; - readonly env?: NodeJS.ProcessEnv; - readonly linux?: boolean; - readonly user?: () => string; - }): Promise<{ - readonly exitCode: number; - readonly output: ReturnType; - readonly serviceManager: FakeServiceManager; - }> { - const { root } = options; - const dataDirectory = options.role === "gateway" ? join(root, "gateway") : root; - const filesystem = new MemoryFilesystem(); - const ipcTransport = new MemoryIpcTransport(); - const serviceManager = new FakeServiceManager({ - ...(options.linux === true ? { manager: "systemd" as const } : {}), - onInstall: async () => { - await startInMemoryDaemon({ - adminTokenPath: join(dataDirectory, "admin.token"), - filesystem, - ipcTransport, - socketPath: join(dataDirectory, "daemon.sock"), - }); - }, - }); - const output = outputCapture( - { - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: ipcTransport, - launcher: new FakeDaemonLauncher(), - daemonHome: { role: options.role, dataDirectory, gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - service: { - manager: serviceManager, - platform: options.linux === true ? "linux" : "darwin", - home: options.linux === true ? "/home/me" : home, - user: options.user ?? (() => "me"), - root, - nodePath, - mainPath, - }, - }, - options.env, - ); - const exitCode = await runCli(options.argv, output.environmentWith()); - return { exitCode, output, serviceManager }; - } - - it("an operator runs daemon install on macOS: the service is installed, the daemon answers, install prints the name, Daemon running and the hint, and --json prints the status and service", async () => { - const cases = [ - { - label: "worker, default home", - argv: ["daemon", "install"], - role: "worker" as const, - root: join(home, ".simlock"), - name: "com.callstack.simlock.worker", - args: [mainPath], - logPath: "/Users/me/.simlock/daemon.log", - }, - { - label: "gateway on a host with no gateway folder yet", - argv: ["--gateway", "daemon", "install"], - role: "gateway" as const, - root: join(home, ".simlock"), - name: "com.callstack.simlock.gateway", - args: [mainPath, "--gateway"], - logPath: "/Users/me/.simlock/gateway/daemon.log", - }, - { - label: "another SIMLOCK_HOME", - argv: ["daemon", "install"], - role: "worker" as const, - root: "/tmp/x", - name: "com.callstack.simlock.worker.2e56aa", - args: [mainPath], - logPath: "/tmp/x/daemon.log", - }, - ]; - - for (const testCase of cases) { - const { exitCode, output, serviceManager } = await install(testCase); - const label = testCase.label; - expect(output.stderr, label).toBe(""); - expect(exitCode, label).toBe(0); - expect(serviceManager.installs, label).toHaveLength(1); - expect(serviceManager.installs[0], label).toMatchObject({ - name: testCase.name, - path: `/Users/me/Library/LaunchAgents/${testCase.name}.plist`, - command: nodePath, - args: testCase.args, - logPath: testCase.logPath, - }); - expect(output.stdout.split("\n").slice(0, 2), label).toEqual([ - `Service installed: ${testCase.name} (launchd)`, - "Daemon running", - ]); - expect(output.stdout.trimEnd().split("\n").at(-1), label).toBe( - "To start after a reboot, turn on automatic login for this user.", - ); - } - - const withShell = await install({ - argv: ["daemon", "install"], - role: "worker", - root: join(home, ".simlock"), - env: { PATH: "/opt/bin:/usr/bin", JAVA_HOME: "/jdk", SIMLOCK_ADMIN_TOKEN: "secret" }, - }); - expect(withShell.exitCode, withShell.output.stderr).toBe(0); - expect(withShell.serviceManager.installs[0]?.environment).toEqual({ - SIMLOCK_HOME: join(home, ".simlock"), - PATH: "/opt/bin:/usr/bin", - JAVA_HOME: "/jdk", - }); - - const json = await install({ - argv: ["daemon", "install", "--json"], - role: "worker", - root: join(home, ".simlock"), - }); - expect(json.exitCode).toBe(0); - expect(JSON.parse(json.output.stdout)).toEqual({ - status: "installed", - service: { - state: "installed", - manager: "launchd", - name: "com.callstack.simlock.worker", - path: "/Users/me/Library/LaunchAgents/com.callstack.simlock.worker.plist", - }, - }); - }); - - it("an operator on Linux runs daemon install: the unit lands in the user config directory, and install prints systemd and the linger hint", async () => { - const cases = [ - { - label: "default config directory", - env: {}, - unit: "/home/me/.config/systemd/user/com.callstack.simlock.worker.service", - }, - { - label: "absolute XDG_CONFIG_HOME", - env: { XDG_CONFIG_HOME: "/xdg" }, - unit: "/xdg/systemd/user/com.callstack.simlock.worker.service", - }, - ]; - for (const { label, env, unit } of cases) { - const { exitCode, output, serviceManager } = await install({ - argv: ["daemon", "install"], - role: "worker", - root: "/home/me/.simlock", - env, - linux: true, - }); - - expect(output.stderr, label).toBe(""); - expect(exitCode, label).toBe(0); - expect(serviceManager.installs[0]?.path, label).toBe(unit); - expect(output.stdout.split("\n").slice(0, 2), label).toEqual([ - "Service installed: com.callstack.simlock.worker (systemd)", - "Daemon running", - ]); - expect(output.stdout.trimEnd().split("\n").at(-1), label).toBe( - "To start after a reboot without logging in, run: loginctl enable-linger me", - ); - } - - // why: a uid with no passwd entry makes the user lookup throw; install must still succeed. - const unknownUser = (): string => { - throw new Error("ENOENT: no passwd entry for uid 1234"); - }; - const lookupFails = await install({ - argv: ["daemon", "install"], - role: "worker", - root: "/home/me/.simlock", - linux: true, - user: unknownUser, - }); - expect(lookupFails.output.stderr).toBe(""); - expect(lookupFails.exitCode).toBe(0); - expect(lookupFails.output.stdout.trimEnd().split("\n").at(-1)).toBe( - "To start after a reboot without logging in, run: loginctl enable-linger ", - ); - }); - - async function runWithService(options: { - readonly argv: readonly string[]; - readonly running: boolean; - readonly socketHeldPolls: number; - readonly state?: ServiceState; - readonly refuse?: boolean; - }): Promise<{ - readonly exitCode: number; - readonly output: ReturnType; - readonly stops: number; - readonly serviceManager: FakeServiceManager; - readonly clock: FakeClock; - }> { - const root = join(home, ".simlock"); - const clock = new FakeClock(0); - const serviceManager = new FakeServiceManager( - options.state === undefined ? {} : { state: options.state }, - ); - let stops = 0; - let polls = 0; - const connectExistingAdmin: CliEnvironment["connectExistingAdmin"] = async ( - resolveCredential, - ) => { - await resolveCredential(); - if (!options.running) - throw new IpcError("endpoint-not-found", "No IPC endpoint yet", undefined); - if (options.refuse === true) { - throw new SimlockError("FORBIDDEN", "protocol", "The admin token was rejected", {}); - } - // why: a stopping daemon refuses new connections while it still holds its socket. - if (stops > 0) throw new IpcError("connection-refused", "Connection refused", undefined); - return fakeClient({ - stopDaemon: async () => { - stops += 1; - return { stopping: true }; - }, - }); - }; - const output = outputCapture(); - const environment = output.environmentWith({ - clock, - daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, - connectExistingAdmin, - daemonEndpointPresent: async () => { - if (stops === 0) return options.running; - polls += 1; - return polls <= options.socketHeldPolls; - }, - readAdminTokenFile: async () => "admin-token", - sleep: async (milliseconds) => { - clock.advance(milliseconds); - }, - service: { - manager: serviceManager, - platform: "darwin", - user: () => "me", - definition: async () => - serviceDefinition({ - role: "worker", - root, - dataDirectory: root, - home, - nodePath, - mainPath, - env: {}, - }), - }, - }); - const exitCode = await runCli(options.argv, environment); - return { exitCode, output, stops, serviceManager, clock }; - } - - it('daemon stop with no socket exits 0 and prints Daemon not running; with --json it prints {"status":"not-running"}', async () => { - const text = await runWithService({ - argv: ["daemon", "stop"], - running: false, - socketHeldPolls: 0, - }); - expect(text.output.stderr).toBe(""); - expect(text.exitCode).toBe(0); - expect(text.output.stdout).toBe("Daemon not running\n"); - - const json = await runWithService({ - argv: ["daemon", "stop", "--json"], - running: false, - socketHeldPolls: 0, - }); - expect(json.output.stderr).toBe(""); - expect(json.exitCode).toBe(0); - expect(JSON.parse(json.output.stdout)).toEqual({ status: "not-running" }); - }); - - it("daemon uninstall removes the service", async () => { - const installedState: ServiceState = { - state: "installed", - path: "/Users/me/Library/LaunchAgents/com.callstack.simlock.worker.plist", - }; - const name = "com.callstack.simlock.worker"; - const cases = [ - { - label: "installed", - argv: ["daemon", "uninstall"], - state: installedState, - stdout: `Service removed: ${name} (launchd)\n`, - removed: 1, - }, - { - label: "installed, --json", - argv: ["daemon", "uninstall", "--json"], - state: installedState, - json: { status: "removed", service: { state: "not-installed", manager: "launchd", name } }, - removed: 1, - }, - { - label: "installed, a daemon running, --json", - argv: ["daemon", "uninstall", "--json"], - state: installedState, - running: true, - json: { - status: "removed", - stopped: true, - service: { state: "not-installed", manager: "launchd", name }, - }, - removed: 1, - }, - { - label: "installed, gateway flag, no gateway folder", - argv: ["--gateway", "daemon", "uninstall"], - state: installedState, - stdout: `Service removed: ${name} (launchd)\n`, - removed: 1, - }, - { - label: "not installed", - argv: ["daemon", "uninstall"], - state: { state: "not-installed" } as ServiceState, - stdout: `Service not installed: ${name}\n`, - removed: 0, - }, - { - label: "not installed, --json", - argv: ["daemon", "uninstall", "--json"], - state: { state: "not-installed" } as ServiceState, - json: { - status: "not-installed", - service: { state: "not-installed", manager: "launchd", name }, - }, - removed: 0, - }, - ]; - for (const testCase of cases) { - const result = await runWithService({ - argv: testCase.argv, - running: testCase.running ?? false, - socketHeldPolls: 0, - state: testCase.state, - }); - expect(result.output.stderr, testCase.label).toBe(""); - expect(result.exitCode, testCase.label).toBe(0); - expect(result.serviceManager.uninstalls, testCase.label).toHaveLength(testCase.removed); - if (testCase.json === undefined) - expect(result.output.stdout, testCase.label).toBe(testCase.stdout); - else expect(JSON.parse(result.output.stdout), testCase.label).toEqual(testCase.json); - } - }); - - it("daemon install stops a running daemon first and waits until its socket is released, even while it refuses connections", async () => { - const none = await runWithService({ - argv: ["daemon", "install"], - running: false, - socketHeldPolls: 0, - }); - expect(none.exitCode).toBe(0); - expect(none.stops).toBe(0); - expect(none.output.stdout).not.toContain("Stopped"); - - const running = await runWithService({ - argv: ["daemon", "install"], - running: true, - socketHeldPolls: 3, - }); - expect(running.exitCode).toBe(0); - expect(running.stops).toBe(1); - expect(running.output.stdout.split("\n").slice(0, 2)).toEqual([ - "Stopped the running daemon; leases are kept.", - "Service installed: com.callstack.simlock.worker (launchd)", - ]); - - const json = await runWithService({ - argv: ["daemon", "install", "--json"], - running: true, - socketHeldPolls: 0, - }); - expect(JSON.parse(json.output.stdout)).toMatchObject({ status: "installed", stopped: true }); - - const stubborn = await runWithService({ - argv: ["daemon", "install"], - running: true, - socketHeldPolls: Number.POSITIVE_INFINITY, - }); - expect(stubborn.exitCode).toBe(1); - expect(JSON.parse(stubborn.output.stderr)).toEqual({ - error: { - code: "SERVICE_FAILED", - message: "The running daemon did not stop within 300 s.", - }, - }); - expect(stubborn.serviceManager.installs).toEqual([]); - expect(stubborn.clock.now()).toBeGreaterThanOrEqual(300_000); - expect(stubborn.clock.now()).toBeLessThan(301_000); - - const refused = await runWithService({ - argv: ["daemon", "install"], - running: true, - socketHeldPolls: 0, - refuse: true, - }); - expect(refused.exitCode).toBe(1); - expect(JSON.parse(refused.output.stderr)).toEqual({ - error: { code: "FORBIDDEN", message: "The admin token was rejected" }, - }); - expect(refused.serviceManager.installs).toEqual([]); - - const uninstall = await runWithService({ - argv: ["daemon", "uninstall"], - running: true, - socketHeldPolls: 3, - state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, - }); - expect(uninstall.exitCode).toBe(0); - expect(uninstall.stops).toBe(1); - expect(uninstall.output.stdout.split("\n").slice(0, 2)).toEqual([ - "Daemon stopping", - "Service removed: com.callstack.simlock.worker (launchd)", - ]); - }); -}); - -describe("CLI: a start goes through the service (#830)", () => { - it("daemon start and a command's auto-start ask a loaded service to start the daemon and spawn nothing", async () => { - const home = "/Users/me"; - const root = join(home, ".simlock"); - const cases = [ - { label: "daemon start", argv: ["daemon", "start", "--json"] }, - { label: "a command's auto-start", argv: ["status", "--json"] }, - ]; - for (const testCase of cases) { - const filesystem = new MemoryFilesystem(); - const ipcTransport = new MemoryIpcTransport(); - const serviceManager = new FakeServiceManager({ - state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, - onStart: async () => { - await startInMemoryDaemon({ - adminTokenPath: join(root, "admin.token"), - filesystem, - ipcTransport, - socketPath: join(root, "daemon.sock"), - }); - }, - }); - const spawner = new FakeDaemonLauncher(() => undefined, { - exit: { code: 1, signal: null }, - }); - const notices: string[] = []; - const output = outputCapture({ - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: ipcTransport, - launcher: launcherFor({ - role: "worker", - root, - dataDirectory: root, - mainPath: "/opt/simlock/dist/daemon/main.js", - nodePath: "/opt/node/bin/node", - home, - env: {}, - serviceManager, - notify: (line) => notices.push(line), - spawner, - }), - daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - - const exitCode = await runCli(testCase.argv, output.environmentWith()); - - expect(exitCode, testCase.label).toBe(0); - expect(serviceManager.starts, testCase.label).toHaveLength(1); - expect(serviceManager.starts[0]?.name, testCase.label).toBe("com.callstack.simlock.worker"); - expect(spawner.launches, testCase.label).toBe(0); - expect(notices, testCase.label).toEqual([]); - } - }); -}); - -describe("CLI: a start through a service that fails (#830)", () => { - it("fails at the 20 s deadline with DAEMON_STARTUP_FAILED and the daemon.log lines written since the start", async () => { - const startError = - '{"message":"Daemon failed to start","fields":{"message":"EADDRINUSE: socket taken"}}\n'; - const cases = [ - { label: "the log holds the start error", written: startError, expected: "EADDRINUSE" }, - { label: "empty log", written: "", expected: undefined }, - { - label: "25 lines: the last 20 are kept", - written: Array.from({ length: 25 }, (_, index) => `line ${String(index + 1)}\n`).join(""), - expected: "line 6\n", - notExpected: "line 5\n", - }, - ]; - for (const testCase of cases) { - const directory = await mkdtemp(join(tmpdir(), "simlock-service-start-")); - try { - const logPath = join(directory, "daemon.log"); - await writeFile(logPath, "older line\n"); - const clock = new FakeClock(0); - const serviceManager = new FakeServiceManager({ - state: { state: "installed", path: "/Users/me/Library/LaunchAgents/x.plist" }, - onStart: async () => { - await appendFile(logPath, testCase.written); - }, - }); - const output = outputCapture({ - filesystem: new MemoryFilesystem(), - clock, - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc: new MemoryIpcTransport(), - launcher: launcherFor({ - role: "worker", - root: directory, - dataDirectory: directory, - mainPath: "/opt/simlock/dist/daemon/main.js", - nodePath: "/opt/node/bin/node", - home: "/Users/me", - env: {}, - serviceManager, - notify: () => undefined, - spawner: new FakeDaemonLauncher(), - }), - daemonHome: { role: "worker", dataDirectory: directory, gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - }); - let settled = false; - const exitCode = runCli(["daemon", "start", "--json"], output.environmentWith()).then( - (code) => { - settled = true; - return code; - }, - ); - for (let tick = 0; tick < 500 && !settled; tick += 1) { - clock.advance(50); - await settle(); - await settle(); - } - expect(settled, `${testCase.label}: not settled at ${String(clock.now())}ms`).toBe(true); - expect(await exitCode, testCase.label).toBe(1); - const error = ( - JSON.parse(output.stderr.trim().split("\n").at(-1) ?? "") as { - error: { code: string; message: string }; - } - ).error; - expect(error.code, testCase.label).toBe("DAEMON_STARTUP_FAILED"); - expect(error.message, testCase.label).toContain("Timed out starting simlock daemon"); - expect(error.message, testCase.label).not.toContain("older line"); - if (testCase.expected === undefined) { - expect(error.message, testCase.label).not.toContain("\n"); - } else { - expect(error.message, testCase.label).toContain(testCase.expected); - } - if (testCase.notExpected !== undefined) { - expect(error.message, testCase.label).not.toContain(testCase.notExpected); - } - } finally { - await rm(directory, { force: true, recursive: true }); - } - } - }); -}); - -describe("CLI: a service manager call fails, or the platform has none (#830)", () => { - const home = "/Users/me"; - const root = join(home, ".simlock"); - const name = "com.callstack.simlock.worker"; - const plist = `${home}/Library/LaunchAgents/${name}.plist`; - const nodePath = "/opt/node/bin/node"; - const mainPath = "/opt/simlock/dist/daemon/main.js"; - const done = (code: number | null, stderr = "") => ({ code, stdout: "", stderr }); - - async function run(options: { - readonly argv: readonly string[]; - readonly platform?: NodeJS.Platform; - readonly script?: ConstructorParameters[0]; - readonly present?: readonly string[]; - }): Promise<{ - readonly exitCode: number; - readonly stderr: string; - readonly filesystem: MemoryFilesystem; - }> { - const filesystem = new MemoryFilesystem(); - for (const path of options.present ?? []) { - await filesystem.mkdirp(dirname(path)); - await filesystem.writeFileAtomic(path, ""); - } - const manager = - options.script === undefined - ? undefined - : new LaunchdServiceManager({ - runner: new ScriptedProcessRunner(options.script), - filesystem, - uid: 501, - }); - const ipc = new MemoryIpcTransport(); - const ports: CliEnvironmentPorts = { - filesystem, - clock: new FakeClock(0), - systemStats: new FakeSystemStats({ cpuCount: 8, totalRamBytes: 32 * gibibyte }), - ipc, - launcher: launcherFor({ - role: "worker", - root, - dataDirectory: root, - mainPath, - nodePath, - home, - env: {}, - serviceManager: manager, - notify: () => undefined, - spawner: new FakeDaemonLauncher(() => undefined, { exit: { code: 1, signal: null } }), - }), - daemonHome: { role: "worker", dataDirectory: root, gatewayPresent: false }, - homeDirectory: "/home/operator", - workingDirectory: "/work/project", - hostPlatform: "linux", - runTool: async () => ({ outcome: "not-found" }), - service: { - manager, - platform: options.platform ?? "darwin", - user: () => "me", - home, - root, - nodePath, - mainPath, - }, - }; - const output = outputCapture(ports); - const exitCode = await runCli(options.argv, output.environmentWith()); - return { exitCode, stderr: output.stderr, filesystem }; - } - - const errorOf = (stderr: string): { code: string; message: string } => - ( - JSON.parse(stderr.trim().split("\n").at(-1) ?? "") as { - error: { code: string; message: string }; - } - ).error; - - it("a failed or stuck launchctl call is SERVICE_FAILED, a failed load removes the plist, and a platform without a service manager is a usage error", async () => { - for (const command of ["install", "uninstall"]) { - const result = await run({ argv: ["daemon", command], platform: "win32" }); - expect(result.exitCode, `win32 ${command}`).toBe(2); - expect(errorOf(result.stderr), `win32 ${command}`).toEqual({ - code: "USAGE", - message: `daemon ${command} is supported on macOS and Linux only`, - }); - } - - const bootout = { - match: { command: "launchctl", args: ["bootout", `gui/501/${name}`] }, - result: done(113), - }; - const bootstrap = { command: "launchctl", args: ["bootstrap", "gui/501", plist] }; - const failedLoad = await run({ - argv: ["daemon", "install"], - script: [ - bootout, - { - match: bootstrap, - result: done(5, "Bootstrap failed: 5: Input/output error\nsecond line"), - }, - ], - }); - expect(failedLoad.exitCode).toBe(1); - expect(errorOf(failedLoad.stderr)).toEqual({ - code: "SERVICE_FAILED", - message: `launchctl bootstrap gui/501 ${plist} failed: Bootstrap failed: 5: Input/output error`, - }); - expect(await failedLoad.filesystem.exists(plist)).toBe(false); - - const stuckLoad = await run({ - argv: ["daemon", "install"], - script: [bootout, { match: bootstrap, result: done(null) }], - }); - expect(stuckLoad.exitCode).toBe(1); - expect(errorOf(stuckLoad.stderr)).toEqual({ - code: "SERVICE_FAILED", - message: `launchctl bootstrap gui/501 ${plist} did not finish within 10 s`, - }); - expect(await stuckLoad.filesystem.exists(plist)).toBe(false); - - const failedKickstart = await run({ - argv: ["daemon", "start"], - present: [plist, nodePath, mainPath], - script: [ - { - match: { command: "plutil", args: ["-convert", "json", "-o", "-", plist] }, - result: { - code: 0, - stdout: JSON.stringify({ ProgramArguments: [nodePath, mainPath] }), - stderr: "", - }, - }, - { match: { command: "launchctl", args: ["print", `gui/501/${name}`] }, result: done(0) }, - { - match: { command: "launchctl", args: ["kickstart", `gui/501/${name}`] }, - result: done(1, "Could not find service"), - }, - ], - }); - expect(failedKickstart.exitCode).toBe(1); - expect(errorOf(failedKickstart.stderr)).toEqual({ - code: "SERVICE_FAILED", - message: `launchctl kickstart gui/501/${name} failed: Could not find service`, - }); - }); -}); - -describe("simlock worker update", () => { - type UpdateAnswer = Awaited>; - type View = Awaited>["workers"][number]; - const view = (overrides: Partial = {}): View => ({ - connection: "connected", - drained: false, - id: "3f81a2c4", - label: "mac-studio-2", - lastSeenAt: 1_000, - version: "0.3.0", - ...overrides, - }); - const run = { forced: false, from: "0.3.0", startedAt: 1_000, to: "0.4.0" } as const; - const answer = (overrides: Partial = {}): UpdateAnswer => ({ - health: "running", - lastUpdateFailure: null, - latestVersion: "0.4.0", - resolved: "0.4.0", - started: true, - update: { ...run, stage: "installing" }, - upToDate: false, - version: "0.3.0", - workerId: "3f81a2c4", - ...overrides, - }); - - it("follows the update to is running, reports a failure with exit 1, says why the worker is up to date, detaches, and refuses on a worker daemon", async () => { - const failure = { - at: 2_000, - from: "0.3.0", - message: "npm exited 1", - stage: "installing", - to: "0.4.0", - } as const; - const cases: readonly { - readonly name: string; - readonly argv: readonly string[]; - readonly answer: UpdateAnswer | SimlockError; - readonly lists?: readonly View[]; - readonly code: number; - readonly stdout?: string; - readonly stderr?: string; - readonly polls?: number; - }[] = [ - { - name: "follows to running", - argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0", "--force"], - answer: answer(), - lists: [ - view({ update: { ...run, leases: 2, stage: "waiting" } }), - view({ update: { ...run, stage: "installing" } }), - view({ connection: "disconnected", update: { ...run, stage: "restarting" } }), - view({ update: { ...run, stage: "restarting" }, version: "0.4.0" }), - view({ update: null, version: "0.4.0" }), - ], - code: 0, - stdout: "mac-studio-2 (3f81a2c4) is running 0.4.0\n", - polls: 5, - }, - { - name: "fails with the stage and message", - argv: ["worker", "update", "3f81a2c4"], - answer: answer(), - lists: [view({ lastUpdateFailure: failure, update: null })], - code: 1, - stdout: "Update failed (installing): npm exited 1\n", - polls: 1, - }, - { - name: "already on the latest", - argv: ["worker", "update", "3f81a2c4"], - answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), - code: 0, - stdout: "mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)\n", - polls: 1, - }, - { - name: "already running the requested version", - argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0"], - answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), - code: 0, - stdout: "mac-studio-2 (3f81a2c4) is already running 0.4.0\n", - polls: 1, - }, - { - name: "newer than the latest", - argv: ["worker", "update", "3f81a2c4"], - answer: answer({ - resolved: "0.3.0", - started: false, - update: null, - upToDate: true, - version: "0.4.0", - }), - code: 0, - stdout: - "mac-studio-2 (3f81a2c4) is newer than the latest (0.3.0). Use --version to install another version.\n", - polls: 1, - }, - { - name: "detach", - argv: ["worker", "update", "3f81a2c4", "--detach"], - answer: answer(), - code: 0, - stdout: - "Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.\n", - polls: 1, - }, - { - name: "worker daemon", - argv: ["worker", "update", "3f81a2c4"], - answer: new SimlockError( - "UNSUPPORTED_IN_WORKER_MODE", - "domain", - "worker.update is a gateway operation", - { operation: "worker.update" }, - ), - code: 2, - stderr: "worker.update is a gateway operation", - polls: 0, - }, - ]; - for (const spec of cases) { - const output = outputCapture(); - const clock = new FakeClock(0); - const sent: unknown[] = []; - const lists = [...(spec.lists ?? [view()])]; - let polls = 0; - const environment = output.environmentWith({ - clock, - connectAdmin: async () => - fakeClient({ - listWorkers: async () => { - polls += 1; - const next = lists.length > 1 ? lists.shift() : lists[0]; - return { workers: next === undefined ? [] : [next] }; - }, - updateWorker: async (input) => { - sent.push(input); - if (spec.answer instanceof Error) throw spec.answer; - return spec.answer; - }, - }), - // why: a follow that never reaches an outcome fails on this line, not by timing out. - sleep: async (milliseconds) => { - if (polls > (spec.lists ?? []).length + 2) - throw new Error(`${spec.name}: the follow never ended`); - clock.advance(milliseconds); - }, - }); - - const code = await runCli(spec.argv, environment); - - expect({ code, name: spec.name, stdout: output.stdout }).toEqual({ - code: spec.code, - name: spec.name, - stdout: spec.stdout ?? "", - }); - if (spec.stderr !== undefined) expect(output.stderr).toContain(spec.stderr); - expect({ name: spec.name, polls }).toEqual({ name: spec.name, polls: spec.polls ?? 0 }); - expect(sent).toEqual([ - { - workerId: "3f81a2c4", - ...(spec.argv.includes("--version") ? { version: "0.4.0" } : {}), - ...(spec.argv.includes("--force") ? { force: true } : {}), - }, - ]); - } - }); -}); - -describe("simlock rotate", () => { - const heldLease = { - deviceId: "dev_1", - grantedAt: 0, - id: "lse_mine", - lastRenewedAt: 0, - idChosenByRequester: false, - ownerId: "some-other-principal", - requesterId: "test-requester", - ttlDeadline: 60_000, - ttlMs: 60_000, - }; - - const two = [heldLease, { ...heldLease, deviceId: "dev_2", id: "lse_other" }]; - - async function rotateWith( - argv: readonly string[], - leases: readonly (typeof heldLease)[], - role: "admin" | "agent" = "admin", - daemonHome?: { role: "gateway"; dataDirectory: string; gatewayPresent: true }, - ) { - const output = outputCapture(); - const rotations: unknown[] = []; - let closes = 0; - const exitCode = await runCli( - argv, - output.environmentWith({ - ...(daemonHome === undefined ? {} : { daemonHome }), - connectAdmin: async () => - fakeClient({ - role, - close: () => { - closes += 1; - return Promise.resolve(); - }, - listLeases: () => Promise.resolve({ leases: [...leases] }), - rotate: (leaseId, orientation, options) => { - rotations.push({ leaseId, orientation, options }); - return Promise.resolve({ leaseId: `${leaseId}_answered`, orientation }); - }, - }), - }), - ); - return { closes: () => closes, exitCode, output, rotations }; - } - - it("a user runs simlock rotate [--lease ], on a worker or with --gateway, and sees Rotated lease to , or a usage error", async () => { - const only = await rotateWith(["rotate", "landscape-left"], [heldLease]); - expect(only.exitCode).toBe(0); - expect(only.rotations).toEqual([ - { - leaseId: "lse_mine", - orientation: "landscape_left", - options: { requesterId: "test-requester" }, - }, - ]); - expect(only.output.stdout).toBe("Rotated lease lse_mine_answered to landscape_left\n"); - expect(only.closes()).toBe(1); - - const asAgent = await rotateWith(["rotate", "landscape-left"], [heldLease], "agent"); - expect(asAgent.exitCode).toBe(0); - expect(asAgent.rotations).toEqual([ - { leaseId: "lse_mine", orientation: "landscape_left", options: undefined }, - ]); - - const gateway = { - role: "gateway", - dataDirectory: "/simlock/gateway", - gatewayPresent: true, - } as const; - for (const argv of [ - ["--gateway", "rotate", "landscape-left"], - ["rotate", "landscape-left", "--gateway"], - ]) { - const throughGateway = await rotateWith(argv, [heldLease], "agent", gateway); - expect(throughGateway.exitCode, argv.join(" ")).toBe(0); - expect(throughGateway.rotations, argv.join(" ")).toEqual([ - { leaseId: "lse_mine", orientation: "landscape_left", options: undefined }, - ]); - expect(throughGateway.output.stdout, argv.join(" ")).toBe( - "Rotated lease lse_mine_answered to landscape_left\n", - ); - } - - for (const argv of [ - ["rotate", "--lease", "lse_other", "left"], - ["rotate", "left", "--lease", "lse_other"], - ["rotate", "left", "--lease=lse_other"], - ]) { - const named = await rotateWith(argv, two); - expect(named.exitCode).toBe(0); - expect(named.rotations).toEqual([ - { - leaseId: "lse_other", - orientation: "landscape_left", - options: { requesterId: "test-requester" }, - }, - ]); - } - - for (const [spelling, orientation] of [ - ["LEFT", "landscape_left"], - [" left ", "landscape_left"], - ["upside-down", "portrait_upside_down"], - ]) { - const spelled = await rotateWith(["rotate", spelling as string], [heldLease]); - expect(spelled.exitCode).toBe(0); - expect(spelled.output.stdout).toBe(`Rotated lease lse_mine_answered to ${orientation}\n`); - } - - const refusals: [readonly string[], readonly (typeof heldLease)[], RegExp][] = [ - [["rotate", "sideways"], [heldLease], /^Unknown orientation sideways; use portrait,/], - [["rotate", "auto"], [heldLease], /^Unknown orientation auto;/], - [["rotate", "landscapeLeft"], [heldLease], /^Unknown orientation landscapeLeft;/], - [["rotate"], [heldLease], /^Expected orientation/], - [["rotate", "left"], [], /test-requester/], - [["rotate", "left"], two, /lse_mine.*lse_other.*--lease /s], - ]; - for (const [argv, leases, message] of refusals) { - const refused = await rotateWith(argv, leases); - expect(refused.exitCode).toBe(2); - expect(refused.rotations).toEqual([]); - expect(JSON.parse(refused.output.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { code: "USAGE", message: expect.stringMatching(message) }, - }); - } - - const unsupported = outputCapture(); - let unsupportedCloses = 0; - await expect( - runCli( - ["rotate", "left", "--lease", "lse_android"], - unsupported.environmentWith({ - connectAdmin: async () => - fakeClient({ - close: () => { - unsupportedCloses += 1; - return Promise.resolve(); - }, - rotate: () => - Promise.reject( - new SimlockError( - "ROTATION_UNSUPPORTED", - "domain", - "Lease lse_android is on an Android emulator; only an iOS simulator can be rotated", - { - class: "emulator", - leaseId: "lse_android", - physical: false, - platform: "android", - }, - ), - ), - }), - }), - ), - ).resolves.toBe(12); - expect(JSON.parse(unsupported.stderr.trim().split("\n").at(-1) ?? "")).toEqual({ - error: { - code: "ROTATION_UNSUPPORTED", - message: - "Lease lse_android is on an Android emulator; only an iOS simulator can be rotated", - }, - }); - expect(unsupported.stdout).toBe(""); - expect(unsupportedCloses).toBe(1); - - const help = await rotateWith(["rotate", "--help"], [heldLease]); - expect(help.exitCode).toBe(0); - expect(help.output.stdout).toBe( - "Usage: simlock rotate [--lease ]\n", - ); - expect(help.rotations).toEqual([]); - const listing = outputCapture(); - await runCli(["--help"], listing.environmentWith()); - expect(listing.stdout).toContain(" rotate [--lease ]"); - }); -}); - -describe("CLI text output escapes control characters in remote text", () => { - const NOW = Date.parse("2026-10-05T12:00:00.000Z"); - type View = Awaited>["workers"][number]; - const worker = (overrides: Partial = {}): View => ({ - connection: "connected", - drained: false, - id: "3f81a2c4", - lastSeenAt: 1_000, - version: "0.3.0", - ...overrides, - }); - const failure = (message: string) => - ({ at: 2_000, from: "0.3.0", message, stage: "installing", to: "0.4.0" }) as const; - - it.each([ - ["ESC", "\u001b", "\\u001b"], - ["BEL", "\u0007", "\\u0007"], - ["CR", "\r", "\\u000d"], - ["NUL", "\u0000", "\\u0000"], - ["C1 CSI", "\u009b", "\\u009b"], - ])( - "simlock stats, worker list, worker update, device add, update, warm forget and daemon logs print %s in remote text escaped", - async (_name, raw, escaped) => { - const text = `a${raw}b`; - const shown = `a${escaped}b`; - const run = async (argv: readonly string[], client: Partial) => { - const output = outputCapture(); - const code = await runCli( - [...argv], - output.environmentWith({ - readLogFile: async () => `ok\n${text}\n`, - clock: new FakeClock(NOW), - connectAdmin: async () => fakeClient(client), - sleep: async () => undefined, - }), - ); - return { code, stdout: output.stdout }; - }; - const updateAnswer = (version: string) => - ({ - health: "running", - lastUpdateFailure: null, - latestVersion: "0.4.0", - resolved: "0.4.0", - started: true, - update: { - forced: false, - from: version, - stage: "installing", - startedAt: 1_000, - to: "0.4.0", - }, - upToDate: false, - version, - workerId: "3f81a2c4", - }) as Awaited>; - - const stats = await run(["stats"], { - usage: async (window) => - usageAnswer(window, { - requesters: [{ id: text, granted: 0, heldTotalMs: 0, rejected: 0, requests: 1 }], - }), - }); - const list = await run(["worker", "list"], { - listWorkers: async () => ({ workers: [worker({ version: text })] }), - }); - const detached = await run(["worker", "update", "3f81a2c4", "--detach"], { - listWorkers: async () => ({ workers: [worker()] }), - updateWorker: async () => updateAnswer(text), - }); - const failed = await run(["worker", "update", "3f81a2c4"], { - listWorkers: async () => ({ - workers: [worker({ lastUpdateFailure: failure(text), update: null })], - }), - updateWorker: async () => updateAnswer("0.3.0"), - }); - const enrolled = await run(["device", "add", "--platform", "android", "SER-1"], { - addDevice: async () => ({ - appCount: 0, - class: "phone", - driverDeviceId: "SER-1", - id: "dev_1", - model: text, - osVersion: text, - platform: "android", - reenrolled: false, - }), - }); - - let polled = 0; - const daemonFollowed = await run(["update"], { - updateDaemon: async () => { - polled += 1; - const answer = updateAnswer(text); - return polled === 1 ? answer : { ...answer, update: null }; - }, - }); - const daemonFailed = await run(["update"], { - updateDaemon: async () => ({ - ...updateAnswer("0.3.0"), - update: null, - lastUpdateFailure: failure(text), - }), - }); - const forgotten = await run(["warm", "forget", "android", "Pixel"], { - forgetWarmKind: async () => ({ - forgotten: [{ platform: "android", model: text, osVersion: "36", mode: "full" }], - }), - }); - const logs = await run(["daemon", "logs"], {}); - const results = { - logs, - stats, - list, - detached, - failed, - enrolled, - daemonFollowed, - daemonFailed, - forgotten, - }; - for (const [name, result] of Object.entries(results)) { - expect({ name, raw: result.stdout.includes(raw) }).toEqual({ name, raw: false }); - expect({ name, shown: result.stdout.includes(shown) }).toEqual({ name, shown: true }); - } - }, - ); -}); diff --git a/src/cli/test-index.ts b/src/cli/test-index.ts new file mode 100644 index 000000000..2c4893dea --- /dev/null +++ b/src/cli/test-index.ts @@ -0,0 +1,573 @@ +import { + testConfig as configWithDefaults, + testComponentWiring, + testStateOf, + FakeDriver, + loadTestRegistry, +} from "../core/testing.js"; +import { EventEmitter } from "node:events"; +import { readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { EventBus } from "../bus/index.js"; +import { ringEventHistory } from "../history/testing.js"; +import { CleanupReaper, type Config } from "../core/index.js"; +import { + CryptoTokenSecrets, + FakeClock, + FakeDaemonLauncher, + FakeParentWatch, + MemoryFilesystem, + MemoryIpcTransport, + FakeKeySetFetcher, + NoopLogger, + type Filesystem, + type IdGenerator, + FakeHttpJsonClient, + FakeSystemStats, + FakeTcpListenerFactory, + FakeTcpStreamConnector, + FakeWebSocketClientConnector, +} from "../ports/index.js"; +import { type Auth, createAuth, tokensOwner } from "../auth/index.js"; +import { openMemoryStateDatabase } from "../auth/testing.js"; +import { DaemonEndpointHost } from "../daemon/connection-host.js"; +import { DaemonServer } from "../daemon/server.js"; +import { SimlockError, type AnySimlockError, type UsageOutput } from "../contract/index.js"; +import type { + CatalogGetOutput, + DoctorReport, + LeaseGrant, + LeaseListOutput, + LeaseRecord, + ListGetOutput, + SimlockAdminClient, + SimlockConfig, + StatusGetOutput, + TokenListOutput, +} from "../simlock-client/client.js"; +import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; +import { buildCliEnvironment } from "./environment.js"; +import { createTestEngine } from "../leasing/testing.js"; + +export const gibibyte = 1024 ** 3; + +/** + * why: Yields to the macrotask queue, so `runLease` gets past `requestLease` and reaches its wait + * point -- its push listeners registered and its renew timer armed -- before a test fires a + * push or advances the clock. + */ +export const settle = (): Promise => new Promise((resolve) => setTimeout(resolve, 0)); + +export const detachedGrant: LeaseGrant = { + device: { + physical: false, + id: "device-1", + driverDeviceId: "ABCD", + spec: { physical: false, model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + mode: "full", + }, + environment: {}, + lease: { + id: "lse_env", + deviceId: "device-1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 61_000, + }, + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, +}; + +export const runningDaemons: DaemonServer[] = []; + +export const temporaryDirectories: string[] = []; + +// why: read straight from the manifest so the expected version is not computed by the code under test. +export const PACKAGE_VERSION = ( + JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8")) as { + version: string; + } +).version; + +export function simlockError(code: AnySimlockError["code"]): AnySimlockError { + const messages: Partial> = { + NO_CAPACITY: "No capacity available", + }; + const kind = + code === "DAEMON_CONNECTION_LOST" || + code === "DAEMON_STOPPING" || + code === "DAEMON_STARTUP_FAILED" + ? "transport" + : ( + [ + "BAD_FRAME", + "HANDSHAKE_REQUIRED", + "BAD_REQUEST", + "UNKNOWN_REQUEST", + "PROTOCOL_VERSION_UNSUPPORTED", + "ADMIN_AUTHENTICATION_FAILED", + "FORBIDDEN", + "UNKNOWN_DAEMON_ERROR", + ] as const + ).includes(code as never) + ? "protocol" + : "domain"; + return new SimlockError( + code, + kind, + messages[code] ?? `${code} failed`, + {} as never, + ) as unknown as AnySimlockError; +} + +/** why: Hoisted out of `fakeClient` so a test can hand back the same status with a different + * `mode` -- which is what the passthrough path branches on (ADR 0005 §19c). */ + +export const EMPTY_STATUS = { + devices: [], + host: { arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }, + leases: [], + capacity: { + ios: { + atRamBudget: false, + limit: 1, + running: 0, + maxRunning: 1, + reserved: 0, + overLimit: false, + warm: 0, + used: 0, + }, + android: { + atRamBudget: false, + limit: 1, + running: 0, + maxRunning: 1, + reserved: 0, + overLimit: false, + warm: 0, + used: 0, + }, + global: { running: 0, maxRunning: 2, reserved: 0, overLimit: false, warm: 0 }, + }, + daemon: { + health: "running", + mode: "worker", + version: "0.0.0-test", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }, + queueDepth: 0, +} satisfies StatusGetOutput; + +export async function statusLines(daemon: Partial) { + const output = outputCapture(); + await runCli( + ["status"], + output.environmentWith({ + connectAdmin: async () => + fakeClient({ + getStatus: () => + Promise.resolve({ ...EMPTY_STATUS, daemon: { ...EMPTY_STATUS.daemon, ...daemon } }), + }), + }), + ); + return output.stdout.split("\n").slice(0, 5); +} + +export function fakeClient(overrides: Partial = {}): SimlockAdminClient { + const emptyCatalog: CatalogGetOutput = { platforms: [] }; + const emptyDoctor: DoctorReport = { findings: [], sources: [] }; + const emptyLeaseList: LeaseListOutput = { leases: [] }; + const emptyListGet: ListGetOutput = []; + const emptyTokenList: TokenListOutput = { tokens: [] }; + const grant: LeaseGrant = { + environment: {}, + device: { + physical: false, + id: "dev_1", + driverDeviceId: "dev_1", + spec: { physical: false, platform: "ios", model: "iPhone 17 Pro", osVersion: "26.5" }, + mode: "full", + }, + lease: { + id: "lse_1", + deviceId: "dev_1", + requesterId: "test-requester", + ownerId: "test-requester", + grantedAt: 0, + lastRenewedAt: 0, + idChosenByRequester: false, + ttlMs: 60_000, + ttlDeadline: 60_000, + }, + timing: { + estimatedProvisionMs: 0, + estimatedBootMs: 0, + estimatedReclaimMs: 0, + estimatedReadyMs: 0, + }, + }; + const base: SimlockAdminClient = { + principal: "test-requester", + role: "admin", + daemonVersion: "test", + getCatalog: () => Promise.resolve(emptyCatalog), + getStatus: () => Promise.resolve(EMPTY_STATUS), + identity: () => + Promise.resolve({ + name: "test-requester", + principal: "test-requester", + role: "admin", + via: "socket", + }), + listComponents: () => Promise.resolve({ components: [] }), + requestLease: (_input, _options) => Promise.resolve(grant), + resolvePassthrough: () => Promise.resolve({ args: [], command: "adb", env: {} }), + exec: () => Promise.resolve({ exitCode: 0 }), + rotate: (leaseId, orientation) => Promise.resolve({ leaseId, orientation }), + cancelLease: () => Promise.resolve({ result: "not-found" }), + renewLease: () => Promise.resolve(grant.lease as LeaseRecord), + releaseLease: (input) => Promise.resolve({ leaseId: input.leaseId }), + listLeases: () => Promise.resolve(emptyLeaseList), + runDoctor: () => Promise.resolve(emptyDoctor), + onLeaseLost: () => () => {}, + onDeviceUnhealthy: () => () => {}, + onDeviceRecovered: () => () => {}, + onConnectionLost: () => () => {}, + close: () => Promise.resolve(), + releaseAllLeases: () => Promise.resolve({ leaseIds: [] }), + list: () => Promise.resolve(emptyListGet), + runCleanup: () => Promise.resolve([]), + runNuke: () => + Promise.resolve({ deletedDevices: [], releasedLeaseIds: [], removedBaselines: [] }), + forgetWarmKind: () => Promise.resolve({ forgotten: [] }), + saveBaseline: () => Promise.reject(new Error("saveBaseline was not expected")), + listBaselines: () => Promise.resolve({ baselines: [] }), + removeBaseline: () => Promise.reject(new Error("removeBaseline was not expected")), + getConfig: () => Promise.resolve({} as SimlockConfig), + stopDaemon: () => Promise.resolve({ stopping: true }), + updateDaemon: () => + Promise.resolve({ + started: false, + upToDate: true, + resolved: null, + version: "test", + health: "running" as const, + update: null, + lastUpdateFailure: null, + latestVersion: null, + }), + replayEvents: () => Promise.resolve([]), + usage: (window) => Promise.resolve(usageAnswer(window)), + subscribeEvents: () => Promise.resolve(() => Promise.resolve()), + createToken: (input) => + Promise.resolve({ + secret: "sec_1", + token: { id: "tok_1", role: input.role, createdAt: 0 }, + }), + listTokens: () => Promise.resolve(emptyTokenList), + revokeToken: () => Promise.resolve({ revoked: true }), + listWorkers: () => Promise.resolve({ workers: [] }), + drainWorker: (input) => Promise.resolve({ drained: true as const, workerId: input.workerId }), + undrainWorker: (input) => + Promise.resolve({ drained: false as const, workerId: input.workerId }), + removeWorker: (input) => Promise.resolve({ removed: true, workerId: input.workerId }), + updateWorker: (input) => + Promise.resolve({ + health: "running" as const, + lastUpdateFailure: null, + latestVersion: null, + resolved: null, + started: false, + update: null, + upToDate: false, + version: "0.3.0", + workerId: input.workerId, + }), + testAlertDestination: (input) => + Promise.resolve({ destination: input.destination, status: 200 }), + installComponent: (input) => + Promise.resolve({ + component: input.version, + outcome: "installed" as const, + platform: input.platform, + version: input.version, + }), + installComponentOnWorkers: () => Promise.resolve({ results: [] }), + removeComponent: (input) => + Promise.resolve({ + outcome: input.dryRun === true ? ("would-remove" as const) : ("removed" as const), + platform: input.platform, + version: input.version, + }), + addDevice: (input) => + Promise.resolve({ + appCount: 2, + class: "phone" as const, + driverDeviceId: input.id, + id: "dev_9", + model: "iPhone 15 Pro", + osVersion: "18.4", + platform: input.platform, + reenrolled: false, + }), + removeDevice: (input) => Promise.resolve({ driverDeviceId: input.id, id: input.id }), + ...overrides, + }; + return base; +} + +export interface OutputCapture { + stdout: string; + stderr: string; + environmentWith(overrides?: Partial): CliEnvironment; +} + +export function outputCapture( + ports?: CliEnvironmentPorts, + env: NodeJS.ProcessEnv = {}, +): OutputCapture { + const capture: OutputCapture = { + stdout: "", + stderr: "", + environmentWith(overrides = {}) { + const stderrOut = { write: (value: string) => (capture.stderr += value) }; + const stdoutOut = { write: (value: string) => (capture.stdout += value) }; + if (ports === undefined) { + return { + clock: new FakeClock(0), + configPath: "/simlock/config.json", + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + requesterId: "test-requester", + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + filesystem: new MemoryFilesystem(), + connectAdmin: async () => fakeClient(), + connectExistingAdmin: async () => fakeClient(), + daemonEndpointPresent: async () => false, + readAdminTokenFile: async () => undefined, + sleep: async () => {}, + readConfigFile: async () => ({}), + writeConfigFile: async () => {}, + validateConfig: async () => {}, + readLogFile: async () => "", + readEventHistory: async () => [], + signals: new EventEmitter() as unknown as CliEnvironment["signals"], + parentWatch: new FakeParentWatch(), + stderr: stderrOut, + stdout: stdoutOut, + confirm: async () => true, + webSocket: new FakeWebSocketClientConnector(), + httpJson: new FakeHttpJsonClient(), + tcpListeners: new FakeTcpListenerFactory(), + tcpStreams: new FakeTcpStreamConnector(), + ...overrides, + }; + } + const base = buildCliEnvironment( + { + ...ports, + signals: + ports.signals ?? + (new EventEmitter() as unknown as NonNullable), + parentWatch: ports.parentWatch ?? new FakeParentWatch(), + confirm: ports.confirm ?? (async () => true), + stderr: stderrOut, + stdout: stdoutOut, + }, + env, + ); + return { ...base, ...overrides }; + }, + }; + return capture; +} + +export function sequence(): IdGenerator { + let next = 0; + return { generate: () => `id${next++}` }; +} + +export function realCliEnvironmentPorts( + filesystem: Filesystem = new MemoryFilesystem(), +): CliEnvironmentPorts { + return { + filesystem, + clock: new FakeClock(0), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + ipc: new MemoryIpcTransport(), + launcher: new FakeDaemonLauncher(), + daemonHome: { role: "worker", dataDirectory: "/simlock", gatewayPresent: false }, + homeDirectory: "/home/operator", + workingDirectory: "/work/project", + hostPlatform: "linux", + runTool: async () => ({ outcome: "not-found" }), + }; +} + +/** + * why: A real `DaemonServer` wired against in-memory ports (`MemoryFilesystem` + `MemoryIpcTransport` + * shared with the CLI environment under test), with the genuine credential handshake + * (`createAuth`) instead of `startTestDaemon`'s + * `resolveRole: { resolve: () => "admin" }` shortcut -- B1 and B2 both hinge on that handshake + * actually running. + */ +export async function startInMemoryDaemon(options: { + readonly filesystem: MemoryFilesystem; + readonly ipcTransport: MemoryIpcTransport; + readonly socketPath: string; + readonly adminTokenPath: string; +}): Promise<{ daemon: DaemonServer; adminSecret: Auth["adminSecret"] }> { + const { adminTokenPath, filesystem, ipcTransport, socketPath } = options; + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const registry = await loadTestRegistry({ + clock, + eventBus, + idGenerator: sequence(), + }); + const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); + const config = testConfig(); + const wiring = testComponentWiring({ + clock: clock, + drivers: [driver], + eventBus: eventBus, + registry: registry, + }); + const engine = createTestEngine({ + ...wiring, + clock, + config, + drivers: [driver], + eventBus, + idGenerator: sequence(), + registry, + state: testStateOf(registry), + systemStats: new FakeSystemStats({ + cpuCount: 8, + totalRamBytes: 32 * gibibyte, + }), + }); + const reaper = new CleanupReaper({ + clock, + config, + eventBus, + executor: engine.cleanup, + filesystem, + registry, + }); + const auth = createAuth({ + clock, + config: { enabled: false, host: "127.0.0.1", port: 4700 }, + dataDirectory: dirname(adminTokenPath), + database: (await openMemoryStateDatabase([tokensOwner])).database, + filesystem, + idGenerator: sequence(), + logger: new NoopLogger(), + keySetFetcher: new FakeKeySetFetcher(), + secrets: new CryptoTokenSecrets(), + }); + const daemon = new DaemonServer({ + adminSecret: auth.adminSecret, + capacity: engine, + warmPool: engine, + deviceModes: engine, + catalog: engine, + instanceId: "instance-test", + clock, + components: wiring.components, + config, + defaultRequesterId: "test-process", + eventBus, + eventHistory: ringEventHistory(eventBus), + host: new DaemonEndpointHost({ + connector: ipcTransport, + endpoint: socketPath, + filesystem, + listenerFactory: ipcTransport, + }), + hostFacts: () => ({ arch: "arm64", os: "macOS", osVersion: "15.5", tools: [] }), + leases: engine, + queue: engine, + reaper, + registry, + resolveRole: { resolve: (hello) => auth.resolveHelloRole(hello) }, + tokens: auth.tokens, + version: "test", + }); + runningDaemons.push(daemon); + await daemon.start(); + return { daemon, adminSecret: auth.adminSecret }; +} + +export function testConfig(): Config { + return configWithDefaults({ + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 1, maxRunning: 1 }, + ios: { maxDevices: 1, maxRunning: 1 }, + maxRunning: 2, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + eventBuffer: { capacity: 100 }, + health: { enabled: false }, + http: { enabled: false, port: 4700 }, + idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, + lease: { defaultTtlMs: 60_000, maxTtlMs: 3_600_000 }, + warmPool: { maxConcurrentBoots: 1 }, + }); +} + +export function usageAnswer( + window: { readonly from: number; readonly to: number }, + overrides: Partial = {}, +): UsageOutput { + const samples = { count: 0, max: null, p50: null, p95: null }; + const figures = { + boot: samples, + bySource: { booted: 0, provisioned: 0, warm: 0 }, + declined: 0, + failures: { byEvent: {} }, + granted: 0, + held: samples, + incidents: { crashRecovered: 0, lost: 0, quarantineRecovered: 0, quarantined: 0 }, + provisioning: samples, + queue: { meanDepth: null, peakDepth: null }, + rejected: { byReason: {}, total: 0 }, + requests: 0, + turnaround: samples, + utilisation: { slots: { max: null, mean: null, peak: null } }, + wait: samples, + }; + return { + bucketMs: 60_000, + coversFrom: window.from, + partial: false, + platforms: { android: figures, ios: figures }, + requesters: [], + series: [], + totals: figures, + window, + workers: [], + ...overrides, + }; +} diff --git a/src/source-hygiene.test.ts b/src/source-hygiene.test.ts index 7968754b9..f7c21661b 100644 --- a/src/source-hygiene.test.ts +++ b/src/source-hygiene.test.ts @@ -26,9 +26,7 @@ type SizedFile = { path: string; lines: number }; * why: ADR 0063 section 3. These files were over their limit when the size test landed; each may * only shrink. A PR that has to grow one raises its number here, as a visible line of its diff. */ -const SHRINK_ONLY: Record = { - "src/cli/index.test.ts": 11674, -}; +const SHRINK_ONLY: Record = {}; const CODE_LIMIT = 1400; const TEST_LIMIT = 2500; From 0f1a8c3d44e5901a6c33a8ca11c580108567f9a4 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 20:45:26 +0100 Subject: [PATCH 76/83] chore(fallow): suppressions and exports follow the split code; the client class stays private behind a factory --- .agents/scripts/comments.mjs | 6 ++- .agents/scripts/test-lint.mjs | 17 +++++--- e2e/test-physical-enrollment.ts | 4 +- .../host/test-agent-device-supervisor.ts | 8 ++-- src/agent-device/test-edge-front.ts | 2 +- src/comment-lint.test.ts | 22 ++++++++++ src/contract/operations-admin.ts | 18 -------- src/contract/operations-fleet.ts | 15 ------- src/contract/operations-lease-request.ts | 1 - src/contract/operations-leases.ts | 7 ---- src/contract/operations.ts | 41 +++++++++++++++++++ src/core/config.ts | 2 - src/core/doctor.ts | 4 -- src/core/registry.ts | 1 + src/core/test-quarantine-coordinator.ts | 1 + src/core/warm-pool/test-policy.ts | 10 ++--- src/daemon/server.ts | 6 +-- src/daemon/test-optional-modules.ts | 2 +- src/drivers/android/screen/test-session.ts | 15 +++---- src/drivers/android/test-physical-handler.ts | 8 ++-- src/drivers/android/testing.ts | 4 +- src/drivers/ios/test-physical.ts | 8 ++-- src/gateway/fleet-dispatch.ts | 1 + src/gateway/test-relay.ts | 1 + src/gateway/test-service.ts | 2 +- src/gateway/test-worker-registry.ts | 2 +- src/history/projections/usage/test-usage.ts | 2 +- src/history/test-history.ts | 2 +- src/http/app.ts | 12 +----- src/leasing/acquisition-gates.ts | 1 + src/leasing/lease-acquisition-coordinator.ts | 11 +---- src/leasing/waiter-outcomes.ts | 3 ++ src/ports/process-runner.ts | 8 +--- src/simlock-client/client-impl.ts | 21 +++++++++- src/simlock-client/client.ts | 4 +- src/test-spec-lint.ts | 6 +-- 36 files changed, 149 insertions(+), 129 deletions(-) diff --git a/.agents/scripts/comments.mjs b/.agents/scripts/comments.mjs index b2f034ee7..0d3f9763b 100755 --- a/.agents/scripts/comments.mjs +++ b/.agents/scripts/comments.mjs @@ -106,12 +106,14 @@ function blocksIn(path, lines, kinds, added, all) { function blockAt(path, lines, added, start, end, trailing) { const block = lines.slice(start - 1, end); if (block.every((l) => DIRECTIVE.test(l.trim()) || l.trim() === "")) return null; - const changed = block.filter((_, i) => added.has(start + i)).length; + // why: a tool directive added beside a comment edits nothing that comment says. + const said = block.map((l, i) => i).filter((i) => !DIRECTIVE.test(block[i].trim())); + const changed = said.filter((i) => added.has(start + i)).length; return { path, start, end, - state: changed === block.length ? "added" : changed > 0 ? "edited" : "unchanged", + state: changed === said.length ? "added" : changed > 0 ? "edited" : "unchanged", trailing, lines: block.map((l, i) => ({ n: start + i, text: l })), next: trailing ? null : nextCode(lines, end), diff --git a/.agents/scripts/test-lint.mjs b/.agents/scripts/test-lint.mjs index a01b7628e..570cc565d 100644 --- a/.agents/scripts/test-lint.mjs +++ b/.agents/scripts/test-lint.mjs @@ -190,6 +190,10 @@ function takeMovedLine(removed, line) { function splitTests(base, head) { const from = gitText(["merge-base", base, head ?? "HEAD"]).trim(); const range = head === undefined ? [from] : [from, head]; + return { removed: removedTestLines(range), imported: internalImportsBefore(from, range) }; +} + +function removedTestLines(range) { const removed = new Map(); const diff = gitText([ "diff", @@ -201,11 +205,14 @@ function splitTests(base, head) { "*.test.ts", "*.test.tsx", ]); - for (const line of diff.split("\n")) { - if (!line.startsWith("-") || line.startsWith("--- ")) continue; - const text = line.slice(1).trim(); - if (text !== "") removed.set(text, (removed.get(text) ?? 0) + 1); + const lines = diff.split("\n").filter((line) => line.startsWith("-") && !line.startsWith("--- ")); + for (const text of lines.map((line) => line.slice(1).trim()).filter((text) => text !== "")) { + removed.set(text, (removed.get(text) ?? 0) + 1); } + return removed; +} + +function internalImportsBefore(from, range) { const imported = new Map(); const shrunk = gitText(["diff", "--name-only", "--no-renames", "--diff-filter=MD", ...range]); for (const file of shrunk.split("\n").filter((f) => TEST_FILE.test(f))) { @@ -216,7 +223,7 @@ function splitTests(base, head) { for (const m of text.matchAll(IMPORT_RE)) if (isInternalValueImport(m, code)) names.add(m[2]); imported.set(dir, names); } - return { removed, imported }; + return imported; } function contractFindings(changed, moved) { diff --git a/e2e/test-physical-enrollment.ts b/e2e/test-physical-enrollment.ts index b27f18555..c967da69e 100644 --- a/e2e/test-physical-enrollment.ts +++ b/e2e/test-physical-enrollment.ts @@ -7,7 +7,7 @@ export const UDID = "00008110-001A2B3C4D5E"; export const ALIAS = "7A1C0E52-6B7D-4F1E-9C1A-2D3E4F5A6B7C"; -export const APPS = Array.from({ length: 12 }, (_, index) => `com.example.app${String(index)}`); +const APPS = Array.from({ length: 12 }, (_, index) => `com.example.app${String(index)}`); export const IPHONE: ScriptedPhysicalDevice = { apps: APPS, @@ -32,7 +32,7 @@ export const IOS_BASE = { modelClasses: { "iPhone 15 Pro": "phone", "iPhone 16": "phone" }, } as const; -export const ANDROID_BASE = { +const ANDROID_BASE = { availableOsVersions: ["35"], defaultModels: { phone: ["Pixel 9"] }, knownModels: ["Pixel 9"], diff --git a/src/agent-device/host/test-agent-device-supervisor.ts b/src/agent-device/host/test-agent-device-supervisor.ts index f0abbf5a2..9da4473a1 100644 --- a/src/agent-device/host/test-agent-device-supervisor.ts +++ b/src/agent-device/host/test-agent-device-supervisor.ts @@ -51,7 +51,7 @@ export type Behaviour = "dies" | "ignores-sigterm" | "immortal"; export type DaemonJson = "ready" | "none" | "other-pid"; -export class ProcessTable extends FakeProcessSupervisor { +class ProcessTable extends FakeProcessSupervisor { readonly behaviours = new Map(); readonly onDeath = new Map void>(); @@ -69,7 +69,7 @@ export class ProcessTable extends FakeProcessSupervisor { } } -export interface Child { +interface Child { readonly pid: number; readonly args: readonly string[]; exit(code: number | null): void; @@ -102,7 +102,7 @@ export function stemOf(role: AgentDeviceRole | undefined): string { return role === "phones" ? "agent-device-phones" : "agent-device"; } -export function supervisorRole(options: Options): { +function supervisorRole(options: Options): { readonly ios?: AgentDeviceIosSigning; readonly role: AgentDeviceRole; } { @@ -248,7 +248,7 @@ export function isDaemonEntry(arg: string | undefined): boolean { return arg === internalEntry || arg === fallbackEntry; } -export async function installAgentDevice( +async function installAgentDevice( filesystem: MemoryFilesystem, simlockHome: string, options: Options, diff --git a/src/agent-device/test-edge-front.ts b/src/agent-device/test-edge-front.ts index e268d731e..2787a9fa9 100644 --- a/src/agent-device/test-edge-front.ts +++ b/src/agent-device/test-edge-front.ts @@ -12,7 +12,7 @@ export const phoneModels = { modelClasses: { "iPhone 16": "phone" }, } as const; -export const endedMessage = "lease ended by simlock; run disconnect, then open"; +const endedMessage = "lease ended by simlock; run disconnect, then open"; export function expectEndedBySimlock(answer: Envelope): void { expect(answer.result).toBeUndefined(); diff --git a/src/comment-lint.test.ts b/src/comment-lint.test.ts index a414161fc..e3c48637b 100644 --- a/src/comment-lint.test.ts +++ b/src/comment-lint.test.ts @@ -89,6 +89,28 @@ describe("comment-lint.mjs", () => { expect(result.lines).toEqual([`scripts/run.mjs:4: ${REASON}`, `src/a.ts:5: ${REASON}`]); }); + it("accepts a tool directive added under an unchanged comment, and still judges a comment added with one", () => { + const dir = repo( + { "src/a.ts": "/** Says what a is. */\nconst a = 1;\n" }, + { + "src/a.ts": [ + "/** Says what a is. */", + "// fallow-ignore-next-line unused-export -- read through a registry.", + "const a = 1;", + "// Says what b is.", + "// fallow-ignore-next-line unused-export -- read through a registry.", + "const b = 2;", + "", + ].join("\n"), + }, + ); + + const result = lint(dir); + + expect(result.status).toBe(1); + expect(result.lines).toEqual([`src/a.ts:4: ${REASON}`]); + }); + it("accepts a block moved unchanged from another file, and rejects one whose moved text was edited", () => { const dir = repo( { diff --git a/src/contract/operations-admin.ts b/src/contract/operations-admin.ts index 3b722a53c..59bd7a0c0 100644 --- a/src/contract/operations-admin.ts +++ b/src/contract/operations-admin.ts @@ -46,7 +46,6 @@ const doctorRunInputSchema = z.object({ purgeOrphans: z.boolean().optional(), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const doctorRun = defineOperation({ name: "doctor.run", /** @@ -70,7 +69,6 @@ export const doctorRun = defineOperation({ }); /** Arm order is load-bearing: a full record matches the first arm, so a worker's answer is unchanged. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const listGet = defineOperation({ name: "list.get", role: "admin", @@ -85,7 +83,6 @@ export const listGet = defineOperation({ ]), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const cleanupRun = defineOperation({ name: "cleanup.run", role: "admin", @@ -94,7 +91,6 @@ export const cleanupRun = defineOperation({ output: z.array(proposalSchema), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const nukeRun = defineOperation({ name: "nuke.run", role: "admin", @@ -112,7 +108,6 @@ const warmKindSchema = z.object({ baseline: baselineNameSchema.optional(), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const warmForget = defineOperation({ name: "warm.forget", role: "admin", @@ -127,7 +122,6 @@ export const warmForget = defineOperation({ output: z.object({ forgotten: z.array(warmKindSchema).max(WARM_POOL_LEARNED_KINDS_MAX) }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const baselineSave = defineOperation({ name: "baseline.save", role: "admin", @@ -142,7 +136,6 @@ export const baselineSave = defineOperation({ output: z.object({ baseline: baselineEntrySchema, leaseId: z.string().max(256) }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const baselineList = defineOperation({ name: "baseline.list", role: "agent", @@ -151,7 +144,6 @@ export const baselineList = defineOperation({ output: z.object({ baselines: z.array(baselineEntrySchema).max(1000) }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const baselineRemove = defineOperation({ name: "baseline.remove", role: "admin", @@ -170,7 +162,6 @@ export const baselineRemove = defineOperation({ }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const configGet = defineOperation({ name: "config.get", role: "admin", @@ -179,7 +170,6 @@ export const configGet = defineOperation({ output: configSchema, }); -// fallow-ignore-next-line unused-export -- see the comment above. export const daemonStop = defineOperation({ name: "daemon.stop", role: "admin", @@ -189,7 +179,6 @@ export const daemonStop = defineOperation({ }); // why: `DaemonServer` dispatches it on a connection whose protocol range does not overlap (like `daemon.stop`), so its input and answer never change shape. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const daemonUpdate = defineOperation({ name: "daemon.update", role: "admin", @@ -218,7 +207,6 @@ export const daemonUpdate = defineOperation({ }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const eventsReplay = defineOperation({ name: "events.replay", role: "admin", @@ -234,7 +222,6 @@ export const eventsReplay = defineOperation({ output: z.array(eventEnvelopeSchema), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const usageGet = defineOperation({ name: "usage.get", role: "admin", @@ -243,7 +230,6 @@ export const usageGet = defineOperation({ output: usageOutputSchema, }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const eventsSubscribe = defineOperation({ name: "events.subscribe", role: "admin", @@ -252,7 +238,6 @@ export const eventsSubscribe = defineOperation({ output: z.object({ subscribed: z.literal(true), subscriptionId: z.string() }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const eventsUnsubscribe = defineOperation({ name: "events.unsubscribe", role: "admin", @@ -266,7 +251,6 @@ export const eventsUnsubscribe = defineOperation({ // `src/cli/commands/token.ts`, and no CLI code reads the token store itself. The declarations live // here because the `OPERATIONS` map every dispatcher validates against is built from this file. -// fallow-ignore-next-line unused-export -- reachable from outside this file only through OPERATIONS. export const tokenCreate = defineOperation({ name: "token.create", role: "admin", @@ -275,7 +259,6 @@ export const tokenCreate = defineOperation({ output: z.object({ secret: z.string(), token: tokenRecordSchema }), }); -// fallow-ignore-next-line unused-export -- same as tokenCreate above. export const tokenList = defineOperation({ name: "token.list", role: "admin", @@ -284,7 +267,6 @@ export const tokenList = defineOperation({ output: z.object({ tokens: z.array(tokenRecordSchema) }), }); -// fallow-ignore-next-line unused-export -- same as tokenCreate above. export const tokenRevoke = defineOperation({ name: "token.revoke", role: "admin", diff --git a/src/contract/operations-fleet.ts b/src/contract/operations-fleet.ts index ab40c6351..938de6647 100644 --- a/src/contract/operations-fleet.ts +++ b/src/contract/operations-fleet.ts @@ -11,7 +11,6 @@ import { workerViewSchema, } from "./schemas.js"; -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerList = defineOperation({ name: "worker.list", role: "admin", @@ -31,7 +30,6 @@ export const workerList = defineOperation({ * a typo, and answering "done" would leave an operator believing a machine was taken out of * service when nothing was. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerDrain = defineOperation({ name: "worker.drain", role: "admin", @@ -40,7 +38,6 @@ export const workerDrain = defineOperation({ output: z.object({ workerId: z.string(), drained: z.literal(true) }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerUndrain = defineOperation({ name: "worker.undrain", role: "admin", @@ -59,7 +56,6 @@ export const workerUndrain = defineOperation({ * already true of something the gateway has never heard of or has already forgotten, so it is * an outcome to report rather than an error to raise (the same shape `token.revoke` uses). */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerRemove = defineOperation({ name: "worker.remove", role: "admin", @@ -69,7 +65,6 @@ export const workerRemove = defineOperation({ }); // why: ADR 0042 §2: the gateway sends `daemon.update` over that worker's uplink and answers with the worker's own answer plus the id. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerUpdate = defineOperation({ name: "worker.update", role: "admin", @@ -89,7 +84,6 @@ export const workerUpdate = defineOperation({ * and running it again is the recovery. `component` echoes the version asked for; `version` is * the exact one installed or found. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const componentInstall = defineOperation({ name: "component.install", role: "admin", @@ -112,7 +106,6 @@ export const componentInstall = defineOperation({ * counts the devices outside Simlock that use it. `variant` tells two components of one * version apart, in the platform's own words; `sizeBytes` is absent when it could not be read. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const componentList = defineOperation({ name: "component.list", role: "agent", @@ -169,7 +162,6 @@ const workerInstallResultSchema = z.object({ * ascending worker id. Progress arrives as `component-progress` pushes carrying `workerId`. * A worker answers `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012 §2). */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const workerInstallComponent = defineOperation({ name: "worker.install-component", role: "admin", @@ -182,7 +174,6 @@ export const workerInstallComponent = defineOperation({ output: z.object({ results: z.array(workerInstallResultSchema) }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const componentRemove = defineOperation({ name: "component.remove", role: "admin", @@ -218,7 +209,6 @@ function isControlCharacter(character: string): boolean { return code <= 0x1f || (code >= 0x7f && code <= 0x9f); } -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const deviceAdd = defineOperation({ name: "device.add", role: "admin", @@ -236,7 +226,6 @@ export const deviceAdd = defineOperation({ }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const deviceRemove = defineOperation({ name: "device.remove", role: "admin", @@ -267,7 +256,6 @@ const relayUrlSchema = z * effect. With `twoWay` (GET, no body) both directions of the channel stay open: the answer comes once * both have ended. Admin only, and on a worker only the gateway's own uplink session may call it. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const relayRequest = defineOperation({ name: "relay.request", role: "admin", @@ -295,7 +283,6 @@ export const relayRequest = defineOperation({ output: z.object({ cancelled: z.literal(true).optional() }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const relayCancel = defineOperation({ name: "relay.cancel", role: "admin", @@ -309,7 +296,6 @@ export const relayCancel = defineOperation({ * once, outside the destination's queue. `destination` is only looked up by name among the * configured ones, never used in a URL. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const alertsTest = defineOperation({ name: "alerts.test", role: "admin", @@ -318,7 +304,6 @@ export const alertsTest = defineOperation({ output: z.object({ destination: z.string(), status: z.number() }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const inboxGet = defineOperation({ name: "inbox.get", role: "admin", diff --git a/src/contract/operations-lease-request.ts b/src/contract/operations-lease-request.ts index a79feb151..aff819325 100644 --- a/src/contract/operations-lease-request.ts +++ b/src/contract/operations-lease-request.ts @@ -273,7 +273,6 @@ const leaseRequestInputSchema = leaseRequestBaseSchema .superRefine(refusePhysicalWithImageFields()) .superRefine(refuseRequiresWithIdempotencyKey()); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseRequest = defineOperation({ name: "lease.request", role: "agent", diff --git a/src/contract/operations-leases.ts b/src/contract/operations-leases.ts index 15b645ac5..8390b4b2f 100644 --- a/src/contract/operations-leases.ts +++ b/src/contract/operations-leases.ts @@ -10,7 +10,6 @@ import { statusLeaseSchema, } from "./schemas.js"; -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseCancel = defineOperation({ name: "lease.cancel", role: "agent", @@ -52,7 +51,6 @@ export const leaseCancel = defineOperation({ * `ttlMs` above `lease.maxTtlMs` is `BAD_REQUEST`, checked in the dispatcher's handler where * that config value is visible. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseRenew = defineOperation({ name: "lease.renew", role: "agent", @@ -83,7 +81,6 @@ export const leaseRelease = defineOperation({ * what comes back is a command the caller could have written had it known the root path, which * ADR 0001 is explicit is an accident boundary rather than a security one. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const driverPassthrough = defineOperation({ name: "driver.passthrough", role: "agent", @@ -140,7 +137,6 @@ export const EXEC_STDIN_MAX_LENGTH = 8_388_608; * which would say the request was malformed when it was merely aimed at a tool this host does * not have. `BAD_REQUEST` on this operation means the body's shape is wrong, nothing more. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const deviceExec = defineOperation({ name: "device.exec", role: "agent", @@ -207,7 +203,6 @@ export const deviceExec = defineOperation({ * answer, set by a gateway, which lists the leases its worker views report. Additive, so a * client that never talks to a gateway sees exactly what it saw before. */ -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseList = defineOperation({ name: "lease.list", role: "agent", @@ -217,7 +212,6 @@ export const leaseList = defineOperation({ }); // why: no `authorize` hook -- the handler answers `UNKNOWN_LEASE` for a lease the caller does not own, as `lease.list` hides it (ADR 0044 §3), so "not yours" and "not there" look the same. -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseGet = defineOperation({ name: "lease.get", role: "agent", @@ -226,7 +220,6 @@ export const leaseGet = defineOperation({ output: z.object({ lease: statusLeaseSchema }), }); -// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. export const leaseReleaseAll = defineOperation({ name: "lease.release-all", role: "admin", diff --git a/src/contract/operations.ts b/src/contract/operations.ts index 3059806e9..12aa2fd0b 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -76,58 +76,99 @@ export { leaseIdSchema, refuseModelWithClass, leaseRequestFields, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseRequest, requestedDevice, requestedClass, } from "./operations-lease-request.js"; export { + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseCancel, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseRenew, leaseRelease, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. driverPassthrough, LEASE_FILE_NAME_PATTERN, EXEC_STDIN_MAX_LENGTH, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. deviceExec, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseList, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseGet, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. leaseReleaseAll, } from "./operations-leases.js"; export { + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. doctorRun, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. listGet, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. cleanupRun, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. nukeRun, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. warmForget, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. baselineSave, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. baselineList, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. baselineRemove, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. configGet, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. daemonStop, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. daemonUpdate, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. eventsReplay, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. usageGet, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. eventsSubscribe, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. eventsUnsubscribe, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. tokenCreate, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. tokenList, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. tokenRevoke, } from "./operations-admin.js"; export { + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerList, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerDrain, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerUndrain, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerRemove, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerUpdate, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. componentInstall, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. componentList, WORKER_RESULT_MESSAGE_MAX_LENGTH, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. workerInstallComponent, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. componentRemove, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. deviceAdd, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. deviceRemove, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. relayRequest, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. relayCancel, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. alertsTest, + // fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. inboxGet, } from "./operations-fleet.js"; diff --git a/src/core/config.ts b/src/core/config.ts index 4757e6383..106512edd 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -42,10 +42,8 @@ export type { Config, ConfigOverrides, DaemonMode, - DefaultModels, DownloadPolicy, DriverConfigEntry, - LegacyCapacityOverrides, LoadConfigOptions, MaxPerRequester, ModuleConfigEntry, diff --git a/src/core/doctor.ts b/src/core/doctor.ts index 13ae49d51..c7102ce07 100644 --- a/src/core/doctor.ts +++ b/src/core/doctor.ts @@ -34,16 +34,12 @@ import { reservedRunning, type WarmTargetFigures } from "./warm-pool/index.js"; import type { StartupRead } from "./startup-read.js"; export type { - DoctorBaselines, DoctorConfig, DoctorFinding, DoctorOptions, - DoctorQuarantine, DoctorReconcileOptions, DoctorReport, - DoctorSource, OptionalModuleFinding, - ProvenanceDrift, ServiceCheck, } from "./doctor-types.js"; export { isStalledTransition } from "./doctor-findings.js"; diff --git a/src/core/registry.ts b/src/core/registry.ts index 4e4e42322..cb8e21f7e 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -156,6 +156,7 @@ export class Registry implements LeaseRequestStore { * `ComponentInstaller` calls it once the driver removed the component, before it emits * `component.removed`. Deleting a record that is not there commits nothing. */ + // fallow-ignore-next-line unused-class-member -- called by ComponentInstaller through its Pick port and by the registry tests. async deleteComponent(platform: Platform, version: string): Promise { const components = this.#components.filter( (existing) => existing.platform !== platform || existing.version !== version, diff --git a/src/core/test-quarantine-coordinator.ts b/src/core/test-quarantine-coordinator.ts index c4e6b540b..8eecd1338 100644 --- a/src/core/test-quarantine-coordinator.ts +++ b/src/core/test-quarantine-coordinator.ts @@ -88,6 +88,7 @@ export class FakeRegistry implements QuarantineRegistry { }); } /** Test-only escape hatch to simulate a device leaving quarantine out of band. */ + // fallow-ignore-next-line unused-class-member -- the quarantine tests call it on the harness this file builds. forceState(deviceId: string, state: VirtualDevice["state"]): void { this.#update(deviceId, (device) => ({ ...device, state })); } diff --git a/src/core/warm-pool/test-policy.ts b/src/core/warm-pool/test-policy.ts index d0c122f5b..27c9ce1ca 100644 --- a/src/core/warm-pool/test-policy.ts +++ b/src/core/warm-pool/test-policy.ts @@ -15,8 +15,8 @@ export const minute = 60_000; export const now = 1_000 * minute; export const shutdownAfterMs = 10 * minute; -export const phones = new Set(["iPhone 15", "iPhone 17"]); -export const classOf = (model: string): DeviceClass | undefined => +const phones = new Set(["iPhone 15", "iPhone 17"]); +const classOf = (model: string): DeviceClass | undefined => phones.has(model) ? "phone" : "tablet"; export function spec(model: string, overrides: Partial = {}): VirtualSpec { @@ -32,9 +32,7 @@ export interface DeviceOptions { spent?: boolean; } -export function readyStamp( - options: DeviceOptions, -): Pick | Record { +function readyStamp(options: DeviceOptions): Pick | Record { return options.readyAt === undefined && options.endedAgo === undefined ? {} : { readyAt: options.readyAt ?? now }; @@ -43,7 +41,7 @@ export function readyStamp( // why: a virtual device that ended a lease is spent (ADR 0061 §2) and never grantable; `endedAgo` // only orders idle devices (least recently used), so it moves createdAt, and only a spent device // carries the end stamp. Such a device counts as freshly ready, so the never-leased idle rule leaves it be. -export function endStamp( +function endStamp( options: DeviceOptions, ): Pick | Record { return options.spent === true && options.endedAgo !== undefined diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 131804499..983253fb2 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -38,11 +38,7 @@ import { type DaemonServerOptions, } from "./server-options.js"; import { acceptRelayChannel, ConnectionRequests } from "./server-requests.js"; -export type { - DaemonServerCommonOptions, - DaemonServerEngineOptions, - DaemonServerOptions, -} from "./server-options.js"; +export type { DaemonServerOptions } from "./server-options.js"; export class DaemonServer { readonly #connections = new Set(); diff --git a/src/daemon/test-optional-modules.ts b/src/daemon/test-optional-modules.ts index 5d7d17f8a..d0f02f35b 100644 --- a/src/daemon/test-optional-modules.ts +++ b/src/daemon/test-optional-modules.ts @@ -37,7 +37,7 @@ export const directories: string[] = []; export const servers: Server[] = []; -export class NotingDriver extends FakeDriver { +class NotingDriver extends FakeDriver { constructor( options: ConstructorParameters[0], readonly notes: string[], diff --git a/src/drivers/android/screen/test-session.ts b/src/drivers/android/screen/test-session.ts index 29e9a2398..b8575a60f 100644 --- a/src/drivers/android/screen/test-session.ts +++ b/src/drivers/android/screen/test-session.ts @@ -15,11 +15,11 @@ import { import { createAndroidScreenCapture, type ScreenTarget } from "./session.js"; import { codecId, concatBytes, sessionPacket } from "./test-scrcpy.js"; -export const ADB = "/sdk/platform-tools/adb"; +const ADB = "/sdk/platform-tools/adb"; export const SERVER = "/simlock/scrcpy-server-v5.0.1"; -export const SERIAL = "emulator-5586"; -export const PHONE_SERIAL = "R58M123ABC"; -export const SCID = 0x1234abcd; +const SERIAL = "emulator-5586"; +const PHONE_SERIAL = "R58M123ABC"; +const SCID = 0x1234abcd; export const DEVICE_JAR = "/data/local/tmp/simlock-scrcpy-1234abcd.jar"; export const SETTINGS = { bitrateKbps: 1500, fps: 30, maxDimension: 720 }; export const SCREEN_READ_MS = 10_000; @@ -45,11 +45,11 @@ export const GATE_REFUSAL = "No command goes to physical Android devices until they match, and Simlock will not restart that server."; export const SCREEN_OFF = "The phone's screen is off and did not turn on. Turn it on and ask for the stream again."; -export const WAKE_ARGS = "shell input keyevent KEYCODE_WAKEUP"; +const WAKE_ARGS = "shell input keyevent KEYCODE_WAKEUP"; export type Behaviour = "accept" | "refuse" | "no-transport" | "connection-refused" | "silent"; -export class Peer { +class Peer { readonly written: string[] = []; closed = false; pulls = 0; @@ -185,6 +185,7 @@ export class FakeTarget implements ScreenTarget { this.server = server ?? new AdbServer(); } + // fallow-ignore-next-line unused-class-member -- the session tests read it on the harness this file builds. get steps(): string[] { return this.log.map((entry) => (entry.includes("app_process") ? "server" : entry)); } @@ -241,7 +242,7 @@ export class FakeTarget implements ScreenTarget { } } -export function isScreenRead(args: readonly string[]): boolean { +function isScreenRead(args: readonly string[]): boolean { return args.join(" ") === "shell dumpsys power"; } diff --git a/src/drivers/android/test-physical-handler.ts b/src/drivers/android/test-physical-handler.ts index a76462ff1..33dcff93d 100644 --- a/src/drivers/android/test-physical-handler.ts +++ b/src/drivers/android/test-physical-handler.ts @@ -18,11 +18,11 @@ import { memoryAndroidSourceDatabase } from "./testing.js"; export const sdk = "/android-sdk"; -export const home = "/home/simlock"; +const home = "/home/simlock"; -export const simlockHome = "/home/simlock/.simlock"; +const simlockHome = "/home/simlock/.simlock"; -export const instanceId = "instance-1"; +const instanceId = "instance-1"; export const adb = `${sdk}/platform-tools/adb`; @@ -174,7 +174,7 @@ export class PhoneAdb implements ProcessRunner { } } -export class LoggingTcpProbe extends FakeTcpProbe { +class LoggingTcpProbe extends FakeTcpProbe { readonly #log: string[]; constructor(log: string[], listening: readonly number[]) { diff --git a/src/drivers/android/testing.ts b/src/drivers/android/testing.ts index 335c75204..589769570 100644 --- a/src/drivers/android/testing.ts +++ b/src/drivers/android/testing.ts @@ -23,7 +23,7 @@ export { WorkingDirectoryFilesystem, } from "./test-sdk-fixtures.js"; -export type { FitImage, Refusal, CustomFitProfile } from "./test-sdk-fixtures.js"; +export type { FitImage, Refusal } from "./test-sdk-fixtures.js"; export { EmulatorExitingRunner, @@ -43,7 +43,7 @@ export { taggedHarness, } from "./test-seed-harness.js"; -export type { SeedProvisioner, BuildProvisioner } from "./test-seed-harness.js"; +export type { BuildProvisioner } from "./test-seed-harness.js"; export { bootProbes, diff --git a/src/drivers/ios/test-physical.ts b/src/drivers/ios/test-physical.ts index 9db57fa3a..6cf177be0 100644 --- a/src/drivers/ios/test-physical.ts +++ b/src/drivers/ios/test-physical.ts @@ -15,11 +15,11 @@ export type Json = string | number | boolean | null | undefined | Json[] | { [ke export type Fields = Record; -export const lockFixture = JSON.parse( +const lockFixture = JSON.parse( readFileSync(new URL("./fixtures/devicectl-lock-state.json", import.meta.url), "utf8"), ) as Fields; -export const appsFixture = JSON.parse( +const appsFixture = JSON.parse( readFileSync(new URL("./fixtures/devicectl-info-apps.json", import.meta.url), "utf8"), ) as { readonly result: { readonly apps: readonly Fields[] } }; @@ -57,7 +57,7 @@ export const PHONE: Fields = { udid: PHONE_UDID, }; -export const DEPRECATED_GROUP: Readonly> = { +const DEPRECATED_GROUP: Readonly> = { deviceType: "hardwareProperties", developerModeStatus: "deviceProperties", ecid: "hardwareProperties", @@ -166,7 +166,7 @@ export function lockArgv(udid: string): string[] { ]; } -export class UmaskFilesystem extends MemoryFilesystem { +class UmaskFilesystem extends MemoryFilesystem { override async mkdirp(path: string): Promise { const existed = await this.exists(path); await super.mkdirp(path); diff --git a/src/gateway/fleet-dispatch.ts b/src/gateway/fleet-dispatch.ts index 9301f124f..5e3645921 100644 --- a/src/gateway/fleet-dispatch.ts +++ b/src/gateway/fleet-dispatch.ts @@ -157,6 +157,7 @@ export class FleetDispatch { } } + // fallow-ignore-next-line unused-class-member -- FleetAdmission calls it through its options' Pick. noteCreated(waiter: FleetWaiter): void { this.#createdAt.set(waiter, this.options.clock.now()); } diff --git a/src/gateway/test-relay.ts b/src/gateway/test-relay.ts index 93ceb1164..f0cbdca1d 100644 --- a/src/gateway/test-relay.ts +++ b/src/gateway/test-relay.ts @@ -83,6 +83,7 @@ export class FakeChannel implements LinkChannel { } // why: the real channel errors its writable at once on a peer's STOP (src/link/channel.ts onStop → #failOutgoing). + // fallow-ignore-next-line unused-class-member -- the relay tests call it on the fake channel this file builds. stopWriting(): void { this.#writeController.error(new LinkChannelError("stopped")); } diff --git a/src/gateway/test-service.ts b/src/gateway/test-service.ts index ebad49479..0ff79d422 100644 --- a/src/gateway/test-service.ts +++ b/src/gateway/test-service.ts @@ -14,7 +14,7 @@ export const RETENTION_MS = 24 * 60 * 60_000; export const REFRESH_MS = 30_000; -export const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; +const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; export class RecordingLogger implements Logger { readonly warnings: Array<{ message: string; fields?: Record }> = []; diff --git a/src/gateway/test-worker-registry.ts b/src/gateway/test-worker-registry.ts index b44b685db..3b14df9b2 100644 --- a/src/gateway/test-worker-registry.ts +++ b/src/gateway/test-worker-registry.ts @@ -7,7 +7,7 @@ export const RETENTION_MS = 24 * 60 * 60_000; /** Matches `core/config.ts`'s own default, so a test that does not care about ADR 0005 §15's * warning gets a cap no fixture's `ttlMs` ever comes near. */ -export const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; +const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; /** Matches `service.test.ts`'s `principal: "gw:instance-1"` -- ADR 0005 §14/§27's own-lease * prefix is that principal plus a trailing `:`. */ diff --git a/src/history/projections/usage/test-usage.ts b/src/history/projections/usage/test-usage.ts index 87440d52a..225cc0a6f 100644 --- a/src/history/projections/usage/test-usage.ts +++ b/src/history/projections/usage/test-usage.ts @@ -197,7 +197,7 @@ export const rejected = ( requester: "alice", ...over, }); -export const figures = (running: number, reserved: number, maxRunning: number) => ({ +const figures = (running: number, reserved: number, maxRunning: number) => ({ maxRunning, reserved, running, diff --git a/src/history/test-history.ts b/src/history/test-history.ts index f61512891..df1c408e1 100644 --- a/src/history/test-history.ts +++ b/src/history/test-history.ts @@ -24,7 +24,7 @@ export const HOUR = 3_600_000; export const JUNK = "this is not a sqlite database ".repeat(20); export const directories: string[] = []; -export class RecordingFilesystem extends NodeFilesystem { +class RecordingFilesystem extends NodeFilesystem { readonly renames: Array<{ readonly from: string; readonly to: string }> = []; beforeRename: (() => Promise) | undefined; diff --git a/src/http/app.ts b/src/http/app.ts index 937fc4556..7b91050b1 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -64,17 +64,7 @@ import { validationHook } from "./routes.js"; import { pipeSse } from "./sse.js"; import { LeaseRequestTracker } from "./tracker.js"; -export type { - HttpAppDisposable, - HttpGatewayDeps, - HttpLeaseFiles, - HttpLeaseRoute, - HttpLeaseSocketAccept, - HttpLeaseSocketOutcome, - HttpMount, - HttpRegistryDevice, - HttpRegistryReader, -} from "./app-types.js"; +export type { HttpAppDisposable, HttpGatewayDeps, HttpLeaseFiles, HttpMount } from "./app-types.js"; // fallow-ignore-next-line complexity -- route wiring for one focused resource surface; splitting it would scatter the shared closures (tracker, notices) across files for no clarity gain. export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpAppDisposable { diff --git a/src/leasing/acquisition-gates.ts b/src/leasing/acquisition-gates.ts index 347bbbfd4..d0dd3066a 100644 --- a/src/leasing/acquisition-gates.ts +++ b/src/leasing/acquisition-gates.ts @@ -15,6 +15,7 @@ export class AcquisitionGates { return this.#updateHold; } + // fallow-ignore-next-line unused-class-member -- AcquisitionWorkflow reads it through its options' gates. get stopPlanning(): boolean { return this.#stopPlanning; } diff --git a/src/leasing/lease-acquisition-coordinator.ts b/src/leasing/lease-acquisition-coordinator.ts index c6dd072e0..8fd6c7d0e 100644 --- a/src/leasing/lease-acquisition-coordinator.ts +++ b/src/leasing/lease-acquisition-coordinator.ts @@ -30,16 +30,7 @@ export { NoCapacityError, ProvisionFailedError, } from "./acquisition-errors.js"; -export type { - AcquisitionClaims, - AcquisitionDecision, - AcquisitionDrivers, - AcquisitionPlannerPort, - AcquisitionQueue, - AcquisitionQuarantine, - LeaseAcquisitionCoordinatorOptions, - LeaseAcquisitionRegistry, -} from "./acquisition-ports.js"; +export type { LeaseAcquisitionCoordinatorOptions } from "./acquisition-ports.js"; export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { readonly #logger: Logger; diff --git a/src/leasing/waiter-outcomes.ts b/src/leasing/waiter-outcomes.ts index 04cef9dbc..c5db28cb7 100644 --- a/src/leasing/waiter-outcomes.ts +++ b/src/leasing/waiter-outcomes.ts @@ -22,6 +22,7 @@ export type WaiterOutcomesOptions = Pick< export class WaiterOutcomes { constructor(private readonly options: WaiterOutcomesOptions) {} + // fallow-ignore-next-line unused-class-member -- AcquisitionWorkflow calls it through its options' outcomes. async grant( waiter: AcquisitionWaiter, device: DeviceRecord, @@ -50,6 +51,7 @@ export class WaiterOutcomes { /** One `warn` line for a device step that failed in the background and is handled here by * deferring the waiter or destroying the device. The waiter only ever sees a generic outcome. */ + // fallow-ignore-next-line unused-class-member -- AcquisitionWorkflow calls it through its options' outcomes. logFailure( message: string, waiter: AcquisitionWaiter, @@ -65,6 +67,7 @@ export class WaiterOutcomes { }); } + // fallow-ignore-next-line unused-class-member -- AcquisitionWorkflow calls it through its options' outcomes. defer(waiter: AcquisitionWaiter, reason: WaitingReason): void { if (waiter.options.noWait) { this.reject(waiter, new NoCapacityError(reason), "no-wait"); diff --git a/src/ports/process-runner.ts b/src/ports/process-runner.ts index d9020632a..f98d24634 100644 --- a/src/ports/process-runner.ts +++ b/src/ports/process-runner.ts @@ -3,8 +3,6 @@ import { constants } from "node:os"; import type { StreamingProcessResult } from "./process-runner-types.js"; export type { - LineEnd, - NodeProcessRunnerOptions, ProcessHandle, ProcessResult, ProcessRunner, @@ -15,11 +13,7 @@ export type { } from "./process-runner-types.js"; export { InputStreamFailedError, ProcessSpawnError } from "./process-runner-types.js"; export { ExecOutputDeliveryStalledError, NodeProcessRunner } from "./node-process-runner.js"; -export type { - ProcessInvocation, - ProcessMatcher, - ScriptedProcessExpectation, -} from "./scripted-process-runner.js"; +export type { ScriptedProcessExpectation } from "./scripted-process-runner.js"; export { ScriptedProcessRunner } from "./scripted-process-runner.js"; /** diff --git a/src/simlock-client/client-impl.ts b/src/simlock-client/client-impl.ts index 10c2a4119..1c3f0249c 100644 --- a/src/simlock-client/client-impl.ts +++ b/src/simlock-client/client-impl.ts @@ -11,7 +11,12 @@ import { } from "../contract/index.js"; import type { LinkChannel, LinkConnection } from "../link/index.js"; import type { CallHooks, LeaseScopedPush, SimlockWire } from "./wire.js"; -import type { OperationInput, OperationOutput, SimlockAdminClient } from "./client-api.js"; +import type { + OperationInput, + OperationOutput, + SimlockAdminClient, + SimlockClient, +} from "./client-api.js"; import { cancelledError, toSimlockError } from "./client-errors.js"; import type { AlertsTestInput, @@ -94,7 +99,19 @@ import type { WorkerUpdateOutput, } from "./types.js"; -export class SimlockClientImpl { +// why: the class stays private to this file, as it was in client.ts: callers hold it only as the +// client interface, so they reach its members through that interface. +export function createSimlockClient( + wire: SimlockWire, + role: Role, + principal: string, + daemonVersion: string, + link?: LinkConnection, +): SimlockClient & { asAdmin(): SimlockAdminClient } { + return new SimlockClientImpl(wire, role, principal, daemonVersion, link); +} + +class SimlockClientImpl { readonly #wire: SimlockWire; readonly #link: LinkConnection | undefined; readonly #leaseLostListeners = new Set<(push: LeaseLostPush) => void>(); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index 00d4e2797..d23ad5a42 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -18,7 +18,7 @@ import type { IpcConnection, IpcConnector } from "../ports/index.js"; import { SimlockWire } from "./wire.js"; import type { SimlockAdminClient, SimlockClient } from "./client-api.js"; import { buildDegradedClient } from "./client-degraded.js"; -import { SimlockClientImpl } from "./client-impl.js"; +import { createSimlockClient } from "./client-impl.js"; export type { BaselineListOutput, @@ -169,7 +169,7 @@ export async function connectSimlockClient( await connection.close(); throw error; } - const impl = new SimlockClientImpl( + const impl = createSimlockClient( wire, hello.role, hello.principal, diff --git a/src/test-spec-lint.ts b/src/test-spec-lint.ts index 8e94cdb35..c8c78b5e3 100644 --- a/src/test-spec-lint.ts +++ b/src/test-spec-lint.ts @@ -6,9 +6,9 @@ import { fileURLToPath } from "node:url"; import { expect } from "vitest"; -export const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); -export const SCRIPT = join(ROOT, ".agents", "scripts", "spec-lint.mjs"); +const SCRIPT = join(ROOT, ".agents", "scripts", "spec-lint.mjs"); export const SHA = execFileSync("git", ["rev-parse", "HEAD"], { cwd: ROOT, @@ -19,7 +19,7 @@ export const FOOTER = "*Written by an agent.*"; export const FENCE = "```"; -export const PARENT = `Request: none +const PARENT = `Request: none ## Behaviours From 4562000dfcad65d8a6919db32cf7d9876263c8fb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 20:46:12 +0100 Subject: [PATCH 77/83] chore(fallow): core's index takes DownloadPolicy from the config types part --- src/core/config.ts | 1 - src/core/index.ts | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/config.ts b/src/core/config.ts index 106512edd..89deea67f 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -42,7 +42,6 @@ export type { Config, ConfigOverrides, DaemonMode, - DownloadPolicy, DriverConfigEntry, LoadConfigOptions, MaxPerRequester, diff --git a/src/core/index.ts b/src/core/index.ts index afd614d11..42fbe2c85 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -7,8 +7,6 @@ export { type Config, type ConfigOverrides, type DaemonMode, - // fallow-ignore-next-line unused-type -- public Config surface (config.downloads.policy); no in-tree consumer names it directly yet - type DownloadPolicy, driverConfig, type DriverConfigEntry, effectiveAllowDownload, @@ -19,6 +17,10 @@ export { REDACTED_VALUE, redactConfig, } from "./config.js"; +export { + // fallow-ignore-next-line unused-type -- public Config surface (config.downloads.policy); no in-tree consumer names it directly yet + type DownloadPolicy, +} from "./config-types.js"; export { type AbsentReason, DEVICE_CLASSES, From f1e5c2d838d963492a0b2ae92e06401b4f6d345c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 20:46:52 +0100 Subject: [PATCH 78/83] chore: record that the decomposition changes no public surface The config, contract and client files were split into parts that the same entry files re-export; no key, default, operation, wire shape or client method changed. Docs: none (structure-only split; every public name and behaviour is unchanged) From a9187173600c947c774d7242f787a7b160f2c96d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:00:25 +0100 Subject: [PATCH 79/83] chore(fallow): one canonical location per test helper name, and suppressions for members only tests reach through a harness --- .agents/scripts/comments.mjs | 4 +- e2e/fake-driver/fake-driver.ts | 1 + .../host/agent-device-supervisor.ts | 1 + src/agent-device/testing.ts | 1 + src/cli/index-events-mcp-smoke.test.ts | 4 +- src/cli/test-index.ts | 4 +- src/core/component-installer.ts | 2 + src/core/fake-device-tcp.ts | 1 + src/core/fake-driver.ts | 1 + src/core/quarantine-coordinator.ts | 1 + .../warm-pool/policy-learned-kinds.test.ts | 42 ++--- .../warm-pool/policy-memory-pressure.test.ts | 6 +- src/core/warm-pool/policy-targets.test.ts | 86 +++++----- src/core/warm-pool/policy.test.ts | 158 ++++++++++-------- src/core/warm-pool/test-policy.ts | 2 +- src/daemon/optional-modules-start.test.ts | 2 +- src/daemon/optional-modules-stop.test.ts | 2 +- src/daemon/test-optional-modules.ts | 2 - src/drivers/android/index.ts | 1 + src/gateway/worker-registry.ts | 2 + src/history/history-prune.test.ts | 2 +- src/history/history-start-recovery.test.ts | 2 +- src/history/test-history.ts | 2 +- src/http/notices.ts | 1 + src/http/tracker.ts | 1 + 25 files changed, 182 insertions(+), 149 deletions(-) diff --git a/.agents/scripts/comments.mjs b/.agents/scripts/comments.mjs index 0d3f9763b..2cba5aff8 100755 --- a/.agents/scripts/comments.mjs +++ b/.agents/scripts/comments.mjs @@ -39,7 +39,9 @@ export function listComments({ base, head, all = false, files = null }) { function movedVerbatim(block, removed) { if (block.state !== "added") return false; const wanted = new Map(); - for (const { text } of block.lines) wanted.set(text.trim(), (wanted.get(text.trim()) ?? 0) + 1); + for (const { text } of block.lines.filter((l) => !DIRECTIVE.test(l.text.trim()))) { + wanted.set(text.trim(), (wanted.get(text.trim()) ?? 0) + 1); + } if ([...wanted].some(([text, n]) => (removed.get(text) ?? 0) < n)) return false; for (const [text, n] of wanted) removed.set(text, removed.get(text) - n); return true; diff --git a/e2e/fake-driver/fake-driver.ts b/e2e/fake-driver/fake-driver.ts index 70479e30a..712f0731f 100644 --- a/e2e/fake-driver/fake-driver.ts +++ b/e2e/fake-driver/fake-driver.ts @@ -650,6 +650,7 @@ export class OutOfProcessFakeDriver implements Driver { } /** Not through `#beforeCall`, so a test's call log holds only what the test caused. */ + // fallow-ignore-next-line unused-class-member -- called through the Driver interface, and by its tests through a test-* harness. async toolVersions(): Promise { return (await this.#readScript()).toolVersions ?? []; } diff --git a/src/agent-device/host/agent-device-supervisor.ts b/src/agent-device/host/agent-device-supervisor.ts index 98bb96456..907684898 100644 --- a/src/agent-device/host/agent-device-supervisor.ts +++ b/src/agent-device/host/agent-device-supervisor.ts @@ -213,6 +213,7 @@ export class AgentDeviceSupervisor { await this.#begin(); } + // fallow-ignore-next-line unused-class-member -- called through the module host's port, and by its tests through the test-* harness. async stop(): Promise { this.#stopped = true; for (const wake of this.#stopWaiters) wake(); diff --git a/src/agent-device/testing.ts b/src/agent-device/testing.ts index e3ac8d9a5..5bf05fd45 100644 --- a/src/agent-device/testing.ts +++ b/src/agent-device/testing.ts @@ -14,6 +14,7 @@ export class FakeAgentDeviceHealth implements AgentDeviceHealthProbe { this.#answer = { body }; } + // fallow-ignore-next-line unused-class-member -- the agent-device tests call it on fakes their test-* helpers build. rejectWith(error: Error): void { this.#answer = { error }; } diff --git a/src/cli/index-events-mcp-smoke.test.ts b/src/cli/index-events-mcp-smoke.test.ts index e7fd0a3b9..360372083 100644 --- a/src/cli/index-events-mcp-smoke.test.ts +++ b/src/cli/index-events-mcp-smoke.test.ts @@ -36,7 +36,7 @@ import { sequence, settle, temporaryDirectories, - testConfig, + cliTestConfig, } from "./test-index.js"; afterEach(async () => { @@ -681,7 +681,7 @@ async function startTestDaemon(): Promise<{ socketPath: string; daemon: DaemonSe idGenerator: sequence(), }); const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const config = testConfig(); + const config = cliTestConfig(); const wiring = testComponentWiring({ clock: clock, drivers: [driver], diff --git a/src/cli/test-index.ts b/src/cli/test-index.ts index 2c4893dea..4d2769213 100644 --- a/src/cli/test-index.ts +++ b/src/cli/test-index.ts @@ -442,7 +442,7 @@ export async function startInMemoryDaemon(options: { idGenerator: sequence(), }); const driver = new FakeDriver({ availableOsVersions: ["26.5"], clock, platform: "ios" }); - const config = testConfig(); + const config = cliTestConfig(); const wiring = testComponentWiring({ clock: clock, drivers: [driver], @@ -515,7 +515,7 @@ export async function startInMemoryDaemon(options: { return { daemon, adminSecret: auth.adminSecret }; } -export function testConfig(): Config { +export function cliTestConfig(): Config { return configWithDefaults({ capacity: { strategy: "resource", diff --git a/src/core/component-installer.ts b/src/core/component-installer.ts index b7dda2662..bcf06d6a4 100644 --- a/src/core/component-installer.ts +++ b/src/core/component-installer.ts @@ -275,6 +275,7 @@ export class ComponentInstaller { * covers an `ended` install too, which never has a call: settling it empties its list, and * nothing joins it after. */ + // fallow-ignore-next-line unused-class-member -- called through core's ports, and by its tests through the test-* harness. inProgress(): readonly ComponentInstallInProgress[] { const listed: ComponentInstallInProgress[] = []; for (const queue of this.#queues.values()) { @@ -465,6 +466,7 @@ export class ComponentInstaller { * Simlock device in `list` and in a removal's checks, so a removal cannot start under a * device the registry does not hold yet (ADR 0010 §8). */ + // fallow-ignore-next-line unused-class-member -- called through core's ports, and by its tests through the test-* harness. claimProvision(spec: Pick): () => void { const key = componentKey(spec.platform, spec.osVersion); const removal = this.#removals.get(spec.platform); diff --git a/src/core/fake-device-tcp.ts b/src/core/fake-device-tcp.ts index 01568af41..66d191e88 100644 --- a/src/core/fake-device-tcp.ts +++ b/src/core/fake-device-tcp.ts @@ -127,6 +127,7 @@ export class FakeDeviceTcp implements DeviceTcp { return peer.stream; } + // fallow-ignore-next-line unused-class-member -- tests call it on fakes their test-* helpers build. openFromDevice(devicePort: number): FakeTcpPeer { const peer = new FakeTcpPeer(); this.#reverses.get(devicePort)?.accept(peer.stream); diff --git a/src/core/fake-driver.ts b/src/core/fake-driver.ts index 051e98552..7a43ea573 100644 --- a/src/core/fake-driver.ts +++ b/src/core/fake-driver.ts @@ -473,6 +473,7 @@ export class FakeDriver implements Driver { } /** why: what a saved source holds, for a test to compare with what the device held. */ + // fallow-ignore-next-line unused-class-member -- tests read it on fakes their test-* helpers build. sourceFiles(sourceId: string): ReadonlyMap | undefined { return this.#sourceFiles.get(sourceId); } diff --git a/src/core/quarantine-coordinator.ts b/src/core/quarantine-coordinator.ts index ae05a7d2b..b28b837b3 100644 --- a/src/core/quarantine-coordinator.ts +++ b/src/core/quarantine-coordinator.ts @@ -76,6 +76,7 @@ export class QuarantineCoordinator { } // No `notifyAvailability`: a quarantined virtual device still counts as running, and a physical device counts toward no capacity, so no waiter can be satisfied by this. + // fallow-ignore-next-line unused-class-member -- called through core's quarantine port, and by its tests through the test-* harness. async enter(failure: QuarantinePurgeFailure): Promise { const nextRetryAt = this.options.clock.now() + this.options.config.retryBackoffMs; await this.options.decisions.run(async () => { diff --git a/src/core/warm-pool/policy-learned-kinds.test.ts b/src/core/warm-pool/policy-learned-kinds.test.ts index 2b8ee7f4b..21386b8af 100644 --- a/src/core/warm-pool/policy-learned-kinds.test.ts +++ b/src/core/warm-pool/policy-learned-kinds.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { VirtualDevice, VirtualSpec } from "../domain.js"; import { evaluate as plan } from "./policy.js"; import { - evaluate, + proposalsOf, minute, now, shutdownAfterMs, @@ -43,11 +43,11 @@ describe("warm pool policy", () => { const configured = spec("iPhone 15"); expect( - evaluate(view([], { learned, limit: 1, targets: [{ count: 1, spec: configured }] })), + proposalsOf(view([], { learned, limit: 1, targets: [{ count: 1, spec: configured }] })), ).toEqual([provision(configured)]); const shut = ofKind("shut", "shutdown"); - expect(evaluate(view([shut], { learned: [learnedOf(kind, 1, now)] }))).toEqual([ + expect(proposalsOf(view([shut], { learned: [learnedOf(kind, 1, now)] }))).toEqual([ { action: "boot", deviceId: "shut", reason: "target", target: kind }, ]); @@ -158,7 +158,7 @@ describe("warm pool policy", () => { for (const [name, devices, peak, wanted, claimed, expected, inFlight = []] of cases) { expect( - evaluate( + proposalsOf( view(devices, { claimed, inFlight, @@ -180,7 +180,7 @@ describe("warm pool policy", () => { for (const [name, configuredCount, peak, created] of cases) { expect( - evaluate( + proposalsOf( view([ofKind("a", "leased")], { learned: [learnedOf(kind, peak, now)], maxConcurrentBoots: 3, @@ -199,12 +199,14 @@ describe("warm pool policy", () => { const young = ofKind("young", "ready", { createdAt: 3, readyAt: now - 1_000 }); const learned = [learnedOf(kind, 1, now)]; - expect(evaluate(view([wanted], { learned }))).toEqual([]); - expect(evaluate(view([wanted, extra], { learned }))).toEqual([ + expect(proposalsOf(view([wanted], { learned }))).toEqual([]); + expect(proposalsOf(view([wanted, extra], { learned }))).toEqual([ { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, ]); - expect(evaluate(view([wanted, young], { learned }))).toEqual([]); - expect(evaluate(view([wanted, extra], { learned: [learnedOf(kind, 2, now)] }))).toEqual([]); + expect(proposalsOf(view([wanted, young], { learned }))).toEqual([]); + expect(proposalsOf(view([wanted, extra], { learned: [learnedOf(kind, 2, now)] }))).toEqual( + [], + ); }); it("keeps a learned kind and its devices but plans nothing for it while it is unplanned, and still plans a configured target of its spec", () => { @@ -212,11 +214,13 @@ describe("warm pool policy", () => { const old = now - shutdownAfterMs - 1; const spare = ofKind("spare", "ready", { createdAt: 1, readyAt: old }); - expect(evaluate(view([], { learned, unplanned: [kind] }))).toEqual([]); - expect(evaluate(view([spare], { learned, unplanned: [kind] }))).toEqual([]); - expect(evaluate(view([], { learned }))).toEqual([provision(kind)]); + expect(proposalsOf(view([], { learned, unplanned: [kind] }))).toEqual([]); + expect(proposalsOf(view([spare], { learned, unplanned: [kind] }))).toEqual([]); + expect(proposalsOf(view([], { learned }))).toEqual([provision(kind)]); expect( - evaluate(view([], { learned, targets: [{ count: 1, spec: kind }], unplanned: [kind] })), + proposalsOf( + view([], { learned, targets: [{ count: 1, spec: kind }], unplanned: [kind] }), + ), ).toEqual([provision(kind)]); }); @@ -234,18 +238,18 @@ describe("warm pool policy", () => { ]); const refused = (options: Parameters[1]) => - evaluate(view([recent, stale], { learned, maxConcurrentBoots: 2, ...options })); + proposalsOf(view([recent, stale], { learned, maxConcurrentBoots: 2, ...options })); expect(refused({ limit: 2 })).toEqual([displace, provision(kind)]); expect(refused({ limit: 2, targets: [{ count: 1, spec: older }] })).toEqual([]); expect(refused({ limit: 2, waiting: [modelDemand("iPad Pro")] })).toEqual([]); - expect(evaluate(view([recent], { learned, limit: 1, maxConcurrentBoots: 2 }))).toEqual([ + expect(proposalsOf(view([recent], { learned, limit: 1, maxConcurrentBoots: 2 }))).toEqual([ { ...displace, deviceId: "recent" }, provision(kind), ]); expect(refused({ handoffInFlight: true, limit: 2 })).toEqual([]); const holder = device("holder", "leased", { spec: spec("iPad Air") }); expect( - evaluate( + proposalsOf( view([holder, stale], { learned: [learnedOf(kind, 2, now - minute), learnedOf(older, 1, now - 2 * minute)], limit: 1, @@ -258,7 +262,7 @@ describe("warm pool policy", () => { spec: spec("Pixel 8", { platform: "android" }), }); expect( - evaluate( + proposalsOf( view([droid], { learned: [learnedOf(kind, 1, now), learnedOf(droid.spec, 1, now - minute)], limit: 2, @@ -280,7 +284,7 @@ describe("warm pool policy", () => { const newer = device("newer", "ready", { endedAgo: 30 * minute, spec: kind }); expect( - evaluate( + proposalsOf( view([newer], { learned, limit: 1, @@ -290,7 +294,7 @@ describe("warm pool policy", () => { ).toEqual([]); const phone = spec("iPhone 15"); expect( - evaluate( + proposalsOf( view([recent], { learned: [learnedOf(older, 1, now)], limit: 1, diff --git a/src/core/warm-pool/policy-memory-pressure.test.ts b/src/core/warm-pool/policy-memory-pressure.test.ts index 5f3f3063c..27b279cbb 100644 --- a/src/core/warm-pool/policy-memory-pressure.test.ts +++ b/src/core/warm-pool/policy-memory-pressure.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { VirtualDevice } from "../domain.js"; import { evaluate as plan } from "./policy.js"; import { - evaluate, + proposalsOf, minute, spec, device, @@ -88,7 +88,7 @@ describe("warm pool policy", () => { options: Parameters[1] = {}, pressure: NonNullable[1]>["pressure"]> = {}, ): string[] => - evaluate( + proposalsOf( view(devices, { targets, ...options, pressure: { step: "warming-paused", ...pressure } }), ).flatMap((proposal) => proposal.action === "shutdown" && proposal.reason === "memory-pressure" @@ -127,7 +127,7 @@ describe("warm pool policy", () => { expect(sheds([keptOldest, olderOther], {}, { step: "none" }), "step none").toEqual([]); expect(sheds([device("busy", "leased")]), "nothing idle").toEqual([]); - const overBudget = evaluate( + const overBudget = proposalsOf( view([keptOldest, olderOther, newerOther], { limit: 2, pressure: { step: "warming-paused" }, diff --git a/src/core/warm-pool/policy-targets.test.ts b/src/core/warm-pool/policy-targets.test.ts index 5cf7af22e..4cd341e14 100644 --- a/src/core/warm-pool/policy-targets.test.ts +++ b/src/core/warm-pool/policy-targets.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { VirtualDevice, VirtualSpec } from "../domain.js"; import { evaluate as plan, type ResolvedTarget, targetedDevices } from "./policy.js"; import { - evaluate, + proposalsOf, minute, now, shutdownAfterMs, @@ -30,7 +30,7 @@ describe("warm pool policy", () => { const ready = ofKind("ready", "ready"); const shut = ofKind("shut", "shutdown"); - const proposals = evaluate(view([ready, shut], { targets: [target(2)] })); + const proposals = proposalsOf(view([ready, shut], { targets: [target(2)] })); expect(proposals).toEqual([ { action: "boot", deviceId: "shut", reason: "target", target: kind }, @@ -40,7 +40,7 @@ describe("warm pool policy", () => { it("with a target and no device of its kind, proposes one provision with the resolved spec", () => { const other = device("other", "ready", { endedAgo: minute, spec: spec("iPad Pro") }); - const proposals = evaluate(view([other], { targets: [target(1)] })); + const proposals = proposalsOf(view([other], { targets: [target(1)] })); expect(proposals).toEqual([ { action: "provision", reason: "target", spec: kind, target: kind }, @@ -51,15 +51,15 @@ describe("warm pool policy", () => { const first = ofKind("first", "ready"); const second = ofKind("second", "ready"); - expect(evaluate(view([first, second], { targets: [target(2)] }))).toEqual([]); + expect(proposalsOf(view([first, second], { targets: [target(2)] }))).toEqual([]); }); it("counts a boot or creation of its kind still in flight toward the target's count", () => { expect( - evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(1)] })), + proposalsOf(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(1)] })), ).toEqual([]); expect( - evaluate(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(2)] })), + proposalsOf(view([], { inFlight: [kind], maxConcurrentBoots: 2, targets: [target(2)] })), ).toHaveLength(1); }); @@ -72,7 +72,7 @@ describe("warm pool policy", () => { }); const reset = ofKind("reset", "shutdown"); - const proposals = evaluate( + const proposals = proposalsOf( view([claimed, spent, reset], { claimed: ["claimed"], reset: ["reset"], @@ -94,7 +94,7 @@ describe("warm pool policy", () => { spent: true, }); - const proposals = evaluate( + const proposals = proposalsOf( view([leased, claimed, spent], { claimed: ["claimed"], limit: 10, targets: [target(1)] }), ); @@ -107,12 +107,12 @@ describe("warm pool policy", () => { const shut = ofKind("shut", "shutdown"); expect( - evaluate(view([shut], { inFlight: [spec("iPad Pro")], targets: [target(1)] })), + proposalsOf(view([shut], { inFlight: [spec("iPad Pro")], targets: [target(1)] })), ).toEqual([]); - expect(evaluate(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] }))).toEqual( + expect(proposalsOf(view([], { inFlight: [spec("iPad Pro")], targets: [target(1)] }))).toEqual( [], ); - expect(evaluate(view([shut], { targets: [target(1)] }))).toHaveLength(1); + expect(proposalsOf(view([shut], { targets: [target(1)] }))).toHaveLength(1); }); it("with maxConcurrentBoots 2 proposes two of a target of three and counts a waiter's boot against the cap", () => { @@ -120,14 +120,14 @@ describe("warm pool policy", () => { const two = ofKind("two", "shutdown"); const three = ofKind("three", "shutdown"); - const proposals = evaluate( + const proposals = proposalsOf( view([one, two, three], { maxConcurrentBoots: 2, targets: [target(3)] }), ); expect(proposals).toHaveLength(2); expect(proposals.every((proposal) => proposal.reason === "target")).toBe(true); const tablet = device("tablet", "shutdown", { endedAgo: minute, spec: spec("iPad Pro") }); - const withWaiter = evaluate( + const withWaiter = proposalsOf( view([tablet, one, two], { maxConcurrentBoots: 2, targets: [target(2)], @@ -141,9 +141,9 @@ describe("warm pool policy", () => { const shut = ofKind("shut", "shutdown"); const queued = modelDemand("iPad Pro"); - expect(evaluate(view([shut], { targets: [target(1)], waiting: [queued] }))).toEqual([]); + expect(proposalsOf(view([shut], { targets: [target(1)], waiting: [queued] }))).toEqual([]); expect( - evaluate( + proposalsOf( view([shut], { maxConcurrentBoots: 2, targets: [target(1)], @@ -152,7 +152,7 @@ describe("warm pool policy", () => { ), ).toEqual([{ action: "boot", deviceId: "shut", reason: "target", target: kind }]); expect( - evaluate( + proposalsOf( view([shut], { maxConcurrentBoots: 2, targets: [target(1)], @@ -176,7 +176,7 @@ describe("warm pool policy", () => { expect(capped.targets[0]).not.toHaveProperty("short"); expect( - evaluate( + proposalsOf( view([phone, shut], { maxConcurrentBoots: 2, targets: [target(1)], @@ -205,7 +205,7 @@ describe("warm pool policy", () => { readyAt: now - shutdownAfterMs * 3, }); - const proposals = evaluate(view([stale, boundary, fresh, leasedBefore])); + const proposals = proposalsOf(view([stale, boundary, fresh, leasedBefore])); expect(proposals).toEqual([ { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, @@ -216,7 +216,7 @@ describe("warm pool policy", () => { const old = device("old", "ready", { createdAt: now - shutdownAfterMs - 1 }); const young = device("young", "ready", { createdAt: now - 1_000 }); - expect(evaluate(view([old, young]))).toEqual([ + expect(proposalsOf(view([old, young]))).toEqual([ { action: "shutdown", deviceId: "old", reason: "never-leased-idle" }, ]); }); @@ -226,12 +226,12 @@ describe("warm pool policy", () => { const extra = ofKind("extra", "ready", { readyAt: 2, createdAt: 2 }); // A target of one counts the older device only; the other one is not targeted and goes. - expect(evaluate(view([counted, extra], { targets: [target(1)] }))).toEqual([ + expect(proposalsOf(view([counted, extra], { targets: [target(1)] }))).toEqual([ { action: "shutdown", deviceId: "extra", reason: "never-leased-idle" }, ]); // A queued request the extra device serves keeps it: it is about to be granted. expect( - evaluate( + proposalsOf( view([counted, extra], { targets: [target(1)], waiting: [modelDemand("iPhone 17")] }), ), ).toEqual([]); @@ -242,7 +242,7 @@ describe("warm pool policy", () => { const claimed = device("claimed", "ready", { readyAt: 1 }); const booting = device("booting", "provisioning", { readyAt: 1 }); - expect(evaluate(view([leased, claimed, booting], { claimed: ["claimed"] }))).toEqual([]); + expect(proposalsOf(view([leased, claimed, booting], { claimed: ["claimed"] }))).toEqual([]); }); it("proposes nothing for a target when the running limit is full, and reports running-limit", () => { @@ -366,8 +366,8 @@ describe("warm pool policy", () => { ]); expect(result.targets[0]).toMatchObject({ booting: 1, ready: 0 }); const pad = { count: 1, spec: spec("iPad Pro") }; - expect(evaluate(view([idle], { limit: 1, targets: [target(1), pad] }))).toEqual([]); - expect(evaluate(view([idle], { limit: 1, targets: [pad, target(1)] }))).toEqual([]); + expect(proposalsOf(view([idle], { limit: 1, targets: [target(1), pad] }))).toEqual([]); + expect(proposalsOf(view([idle], { limit: 1, targets: [pad, target(1)] }))).toEqual([]); }); it("shuts down a copy past another configured kind's count, for a slot the reserve holds too, and fills every configured count before a learned peak", () => { @@ -378,22 +378,24 @@ describe("warm pool policy", () => { const both = [{ count: 1, spec: pad }, target(1)]; const make = { action: "provision", reason: "target", spec: kind, target: kind }; - expect(evaluate(view(pads, { limit: 2, targets: both }))).toEqual([ + expect(proposalsOf(view(pads, { limit: 2, targets: both }))).toEqual([ { action: "shutdown", deviceId: "b", reason: "displaced" }, make, ]); expect( - evaluate(view(pads, { limits: { ios: 3 }, reserve: { ios: 1 }, targets: both })), + proposalsOf(view(pads, { limits: { ios: 3 }, reserve: { ios: 1 }, targets: both })), ).toEqual([{ action: "shutdown", deviceId: "b", reason: "displaced" }, make]); expect( - evaluate(view([pads[0]!], { learned: [learnedOf(pad, 2, now)], limit: 2, targets: both })), + proposalsOf( + view([pads[0]!], { learned: [learnedOf(pad, 2, now)], limit: 2, targets: both }), + ), ).toEqual([make]); }); it("counts two targets resolving to one spec as one with the summed count", () => { const ready = ofKind("ready", "ready"); - const proposals = evaluate( + const proposals = proposalsOf( view([ready], { maxConcurrentBoots: 5, targets: [target(1), target(2)] }), ); @@ -407,7 +409,7 @@ describe("warm pool policy", () => { it("does not count a slim device toward a full target of the same model", () => { const slim = device("slim", "ready", { endedAgo: minute, spec: { ...kind, mode: "slim" } }); - const proposals = evaluate(view([slim], { targets: [target(1)] })); + const proposals = proposalsOf(view([slim], { targets: [target(1)] })); expect(proposals).toEqual([ { action: "provision", reason: "target", spec: kind, target: kind }, @@ -472,7 +474,7 @@ describe("warm pool policy", () => { expect(result.proposals).toEqual([]); expect(result.targets).toEqual([]); - expect(evaluate(view([stale]))).toEqual([ + expect(proposalsOf(view([stale]))).toEqual([ { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, ]); }); @@ -505,7 +507,9 @@ describe("warm pool policy", () => { const one = ofKind("one", "shutdown"); const two = ofKind("two", "shutdown"); - const proposals = evaluate(view([one, two], { maxConcurrentBoots: 3, targets: [target(3)] })); + const proposals = proposalsOf( + view([one, two], { maxConcurrentBoots: 3, targets: [target(3)] }), + ); expect(proposals).toEqual([ { action: "boot", deviceId: "one", reason: "target", target: kind }, @@ -592,7 +596,7 @@ describe("warm pool policy", () => { ); it("does not count a boot in flight for another kind toward the target's count", () => { - const result = evaluate( + const result = proposalsOf( view([], { inFlight: [spec("iPad Pro")], maxConcurrentBoots: 2, targets: [target(1)] }), ); @@ -604,7 +608,7 @@ describe("warm pool policy", () => { spec: spec("Pixel 9", { osVersion: "36", platform: "android" }), }); - expect(evaluate(view([], { limit: 1, targets: [target(1)] }))).toHaveLength(1); + expect(proposalsOf(view([], { limit: 1, targets: [target(1)] }))).toHaveLength(1); const global = plan( view([busyAndroid], { limit: 1, limits: { ios: 5 }, targets: [target(1)] }), ); @@ -621,14 +625,14 @@ describe("warm pool policy", () => { const machine = plan(view([], { limit: 2, reserve, targets: [target(1)] })); expect(machine.proposals).toEqual([]); expect(machine.targets[0]).toMatchObject({ short: "reserve" }); - expect(evaluate(view([], { limit: 3, reserve, targets: [target(1)] }))).toHaveLength(1); + expect(proposalsOf(view([], { limit: 3, reserve, targets: [target(1)] }))).toHaveLength(1); const platform = plan( view([], { limits: { ios: 1 }, reserve: { ios: 1 }, targets: [target(1)] }), ); expect(platform.targets[0]).toMatchObject({ short: "reserve" }); expect( - evaluate(view([], { limits: { ios: 2 }, reserve: { ios: 1 }, targets: [target(1)] })), + proposalsOf(view([], { limits: { ios: 2 }, reserve: { ios: 1 }, targets: [target(1)] })), ).toHaveLength(1); }); @@ -645,7 +649,7 @@ describe("warm pool policy", () => { it("frees the slot of a never-leased device it shuts down for a target of another kind in the same pass", () => { const stale = device("stale", "ready", { readyAt: 1, spec: spec("iPad Pro") }); - const proposals = evaluate(view([stale], { limit: 1, targets: [target(1)] })); + const proposals = proposalsOf(view([stale], { limit: 1, targets: [target(1)] })); expect(proposals).toEqual([ { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }, @@ -657,7 +661,7 @@ describe("warm pool policy", () => { const first = device("first", "ready", { readyAt: 1 }); const second = device("second", "ready", { readyAt: 1 }); - const proposals = evaluate(view([first, second], { limit: 1 })); + const proposals = proposalsOf(view([first, second], { limit: 1 })); expect(proposals).toEqual([ { action: "shutdown", deviceId: "first", reason: "over-budget" }, @@ -669,11 +673,13 @@ describe("warm pool policy", () => { const stale = device("stale", "ready", { readyAt: 1 }); const shutdown = { action: "shutdown", deviceId: "stale", reason: "never-leased-idle" }; - expect(evaluate(view([stale], { waiting: [modelDemand("iPhone 15")] }))).toEqual([]); + expect(proposalsOf(view([stale], { waiting: [modelDemand("iPhone 15")] }))).toEqual([]); expect( - evaluate(view([stale], { waiting: [{ ...modelDemand("iPhone 15"), inFlight: true }] })), + proposalsOf(view([stale], { waiting: [{ ...modelDemand("iPhone 15"), inFlight: true }] })), ).toEqual([shutdown]); - expect(evaluate(view([stale], { waiting: [modelDemand("iPad Pro")] }))).toEqual([shutdown]); + expect(proposalsOf(view([stale], { waiting: [modelDemand("iPad Pro")] }))).toEqual([ + shutdown, + ]); }); it("does not name a device with a lease record as targeted even when its state still reads ready", () => { diff --git a/src/core/warm-pool/policy.test.ts b/src/core/warm-pool/policy.test.ts index b0b8cd6b7..064e19ac4 100644 --- a/src/core/warm-pool/policy.test.ts +++ b/src/core/warm-pool/policy.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { VirtualDevice, WaitingDemand } from "../domain.js"; import { evaluate as plan, type WarmPolicyView } from "./policy.js"; import { - evaluate, + proposalsOf, minute, now, spec, @@ -28,15 +28,15 @@ describe("warm pool policy", () => { const shutDown = device("shut", "shutdown", { endedAgo: 20 * minute }); const three = [waiter("iPhone 15", true), waiter("iPhone 17"), waiter("iPhone 16")]; - expect(evaluate(view([busy], { maxConcurrentBoots: 3, waiting: three }))).toEqual([ + expect(proposalsOf(view([busy], { maxConcurrentBoots: 3, waiting: three }))).toEqual([ creation("iPhone 17"), creation("iPhone 16"), ]); - expect(evaluate(view([busy], { maxConcurrentBoots: 2, waiting: three }))).toEqual([ + expect(proposalsOf(view([busy], { maxConcurrentBoots: 2, waiting: three }))).toEqual([ creation("iPhone 17"), ]); expect( - evaluate( + proposalsOf( view([busy], { maxConcurrentBoots: 3, waiting: [waiter("iPhone 15"), waiter("iPhone 17"), waiter("iPhone 16")], @@ -50,7 +50,7 @@ describe("warm pool policy", () => { }), capacity: capacityOf([busy], { android: 0, global: 10 }), }; - expect(evaluate(refused)).toEqual([creation("iPhone 17")]); + expect(proposalsOf(refused)).toEqual([creation("iPhone 17")]); const atCap = view([busy], { inFlight: [spec("iPhone 15")], maxConcurrentBoots: 1, @@ -59,12 +59,14 @@ describe("warm pool policy", () => { }); expect(plan(atCap).proposals).toEqual([]); expect(plan(atCap).targets[0]?.short).toBeUndefined(); - expect(evaluate(view([shutDown], { waiting: [waiter("iPhone 15")] }))).toEqual([ + expect(proposalsOf(view([shutDown], { waiting: [waiter("iPhone 15")] }))).toEqual([ { action: "boot", deviceId: "shut", reason: "waiting-request" }, ]); - expect(evaluate(view([busy], { waiting: [waiter("iPhone 15")] }))).toEqual([]); + expect(proposalsOf(view([busy], { waiting: [waiter("iPhone 15")] }))).toEqual([]); expect( - evaluate(view([shutDown], { inFlight: [spec("iPad Pro")], waiting: [waiter("iPhone 15")] })), + proposalsOf( + view([shutDown], { inFlight: [spec("iPad Pro")], waiting: [waiter("iPhone 15")] }), + ), ).toEqual([]); }); @@ -81,17 +83,17 @@ describe("warm pool policy", () => { const two = [waiter("iPhone 15", true), waiter("iPhone 15"), waiter("iPhone 17")]; const base = { maxConcurrentBoots: 5, limit: 10 }; - expect(evaluate(view([ready("one")], { ...base, waiting: two }))).toEqual([ + expect(proposalsOf(view([ready("one")], { ...base, waiting: two }))).toEqual([ creation("iPhone 17"), ]); expect( - evaluate( + proposalsOf( view([ready("one")], { ...base, waiting: [waiter("iPhone 15"), waiter("iPhone 17")] }), ), ).toEqual([]); - expect(evaluate(view([ready("one"), ready("two")], { ...base, waiting: two }))).toEqual([]); + expect(proposalsOf(view([ready("one"), ready("two")], { ...base, waiting: two }))).toEqual([]); expect( - evaluate( + proposalsOf( view([], { ...base, onItsWay: [{ spec: spec("iPhone 15") }], @@ -100,7 +102,7 @@ describe("warm pool policy", () => { ), ).toEqual([creation("iPhone 17")]); expect( - evaluate( + proposalsOf( view([], { ...base, onItsWay: [{ spec: spec("iPhone 15") }, { deviceId: "other", spec: spec("iPhone 17") }], @@ -109,7 +111,7 @@ describe("warm pool policy", () => { ), ).toEqual([]); expect( - evaluate( + proposalsOf( view([ready("slim", { spec: spec("iPhone 15", { mode: "slim" }) })], { ...base, waiting: [waiter("iPhone 15", true), waiter("iPhone 17")], @@ -119,7 +121,7 @@ describe("warm pool policy", () => { // why: the head is the first waiter no ready idle device takes: with one ready device and two // waiters it serves, the second is the head and leasing creates for it. expect( - evaluate( + proposalsOf( view([ready("one")], { ...base, waiting: [waiter("iPhone 15"), waiter("iPhone 15")] }), ), ).toEqual([]); @@ -129,27 +131,27 @@ describe("warm pool policy", () => { const released = device("lse_15", "shutdown", { endedAgo: 90 * minute }); const waiting = [classDemand(), classDemand()]; - expect(evaluate(view([released], { waiting }))).toEqual([ + expect(proposalsOf(view([released], { waiting }))).toEqual([ { action: "boot", deviceId: "lse_15", reason: "waiting-request" }, ]); - expect(evaluate(view([released], { waiting, blocked: [spec("iPhone 15")] }))).toEqual([]); + expect(proposalsOf(view([released], { waiting, blocked: [spec("iPhone 15")] }))).toEqual([]); // why: the converger records the failure under the device's spec, which a class waiter's may differ from. const other = device("lse_17", "shutdown", { endedAgo: 90 * minute, spec: spec("iPhone 17"), }); - expect(evaluate(view([other], { waiting }))).toEqual([ + expect(proposalsOf(view([other], { waiting }))).toEqual([ { action: "boot", deviceId: "lse_17", reason: "waiting-request" }, ]); - expect(evaluate(view([other], { waiting, blocked: [spec("iPhone 17")] }))).toEqual([]); + expect(proposalsOf(view([other], { waiting, blocked: [spec("iPhone 17")] }))).toEqual([]); }); it("proposes a boot, not a shutdown, for a released iPhone 15 that serves a waiting class request at the running cap", () => { const released = device("lse_15", "shutdown", { endedAgo: 1_000 }); const leased = device("busy", "leased", { spec: spec("iPhone 17") }); - const proposals = evaluate(view([released, leased], { limit: 2, waiting: [classDemand()] })); + const proposals = proposalsOf(view([released, leased], { limit: 2, waiting: [classDemand()] })); expect(proposals).toEqual([{ action: "boot", deviceId: "lse_15", reason: "waiting-request" }]); }); @@ -158,7 +160,7 @@ describe("warm pool policy", () => { const ready = device("lse_15", "ready", { endedAgo: 1_000 }); const other = device("other", "ready", { endedAgo: 500_000, spec: spec("iPad Pro") }); - const proposals = evaluate(view([ready, other], { limit: 1, waiting: [classDemand()] })); + const proposals = proposalsOf(view([ready, other], { limit: 1, waiting: [classDemand()] })); // Over budget by one: the device that serves the waiter stays, the one that does not goes. expect(proposals).toEqual([{ action: "shutdown", deviceId: "other", reason: "over-budget" }]); @@ -171,9 +173,9 @@ describe("warm pool policy", () => { }); const full = device("full", "shutdown", { endedAgo: 1_000 }); - const forFull = evaluate(view([slim, full], { waiting: [classDemand("full")], limit: 5 })); + const forFull = proposalsOf(view([slim, full], { waiting: [classDemand("full")], limit: 5 })); // Full first: a check that refuses only slim-for-full would pick `full` here. - const forSlim = evaluate(view([full, slim], { waiting: [classDemand("slim")], limit: 5 })); + const forSlim = proposalsOf(view([full, slim], { waiting: [classDemand("slim")], limit: 5 })); expect(forFull.find((item) => item.reason === "waiting-request")).toEqual({ action: "boot", @@ -194,7 +196,9 @@ describe("warm pool policy", () => { spec: spec("iPhone 17"), }); - const proposals = evaluate(view([released, matching], { waiting: [modelDemand("iPhone 17")] })); + const proposals = proposalsOf( + view([released, matching], { waiting: [modelDemand("iPhone 17")] }), + ); expect(proposals).toEqual([{ action: "boot", deviceId: "lse_17", reason: "waiting-request" }]); }); @@ -204,7 +208,7 @@ describe("warm pool policy", () => { const older = device("b-older", "ready", { endedAgo: 60 * minute, spec: spec("iPad Pro") }); const newer = device("c-newer", "ready", { endedAgo: 5 * minute, spec: spec("iPad Pro") }); - const proposals = evaluate( + const proposals = proposalsOf( view([oldestServing, older, newer], { limit: 2, waiting: [classDemand()] }), ); @@ -218,7 +222,7 @@ describe("warm pool policy", () => { const second = device("second", "ready", { endedAgo: 70 * minute }); const third = device("third", "ready", { endedAgo: 60 * minute }); - const proposals = evaluate( + const proposals = proposalsOf( view([leased, claimed, first, second, third], { limit: 3, claimed: ["claimed"] }), ); @@ -230,7 +234,7 @@ describe("warm pool policy", () => { it("counts a reservation against the budget", () => { const only = device("only", "ready", { endedAgo: 1_000 }); - const proposals = evaluate(view([only], { limit: 1, reserved: 1 })); + const proposals = proposalsOf(view([only], { limit: 1, reserved: 1 })); expect(proposals).toEqual([{ action: "shutdown", deviceId: "only", reason: "over-budget" }]); }); @@ -239,8 +243,8 @@ describe("warm pool policy", () => { const first = device("first", "ready", { endedAgo: 80 * minute }); const second = device("second", "ready", { endedAgo: 70 * minute }); - expect(evaluate(view([first, second], { limit: 1, handoffInFlight: true }))).toEqual([]); - expect(evaluate(view([first, second], { limit: 1 }))).toEqual([ + expect(proposalsOf(view([first, second], { limit: 1, handoffInFlight: true }))).toEqual([]); + expect(proposalsOf(view([first, second], { limit: 1 }))).toEqual([ { action: "shutdown", deviceId: "first", reason: "over-budget" }, ]); }); @@ -254,7 +258,7 @@ describe("warm pool policy", () => { const leased = device("leased", "leased"); const shutdown = device("shutdown", "shutdown", { endedAgo: 1_000 }); - const proposals = evaluate( + const proposals = proposalsOf( view([ready, android, leased, shutdown], { enabled: false, waiting: [classDemand()] }), ); @@ -268,9 +272,9 @@ describe("warm pool policy", () => { const fresh = device("fresh", "shutdown", { endedAgo: 1_000 }); const later = device("later", "shutdown", { endedAgo: minute }); - expect(evaluate(view([fresh, later], { limit: 5 }))).toEqual([]); + expect(proposalsOf(view([fresh, later], { limit: 5 }))).toEqual([]); expect( - evaluate( + proposalsOf( view([fresh, later], { learned: [{ count: 1, lastLeasedAt: now, spec: fresh.spec }] }), ), ).toEqual([{ action: "boot", deviceId: "fresh", reason: "target", target: fresh.spec }]); @@ -281,7 +285,7 @@ describe("warm pool policy", () => { const serving = device("serving", "shutdown", { endedAgo: 9 * minute }); const busy = device("busy", "leased"); - const proposals = evaluate( + const proposals = proposalsOf( view([recent, serving, busy], { limit: 2, waiting: [classDemand()] }), ); @@ -297,10 +301,12 @@ describe("warm pool policy", () => { const learned = [learnedOf(recent.spec)]; const boot = { action: "boot", deviceId: "recent", reason: "target", target: recent.spec }; - expect(evaluate(view([recent, busy], { learned, limit: 2, waiting: [unserved] }))).toEqual([]); - expect(evaluate(view([recent, busy], { learned, limit: 2 }))).toEqual([boot]); + expect(proposalsOf(view([recent, busy], { learned, limit: 2, waiting: [unserved] }))).toEqual( + [], + ); + expect(proposalsOf(view([recent, busy], { learned, limit: 2 }))).toEqual([boot]); expect( - evaluate(view([recent, busy, served], { learned, limit: 3, waiting: [classDemand()] })), + proposalsOf(view([recent, busy, served], { learned, limit: 3, waiting: [classDemand()] })), ).toEqual([boot]); }); @@ -316,21 +322,23 @@ describe("warm pool policy", () => { const options = { maxConcurrentBoots: 3 }; expect( - evaluate(view([phone, busy], { ...options, waiting: [tabletHead, classDemand()] })), + proposalsOf(view([phone, busy], { ...options, waiting: [tabletHead, classDemand()] })), ).toEqual([boot("phone")]); expect( - evaluate( + proposalsOf( view([phone, busy], { ...options, waiting: [tabletHead, classDemand(), classDemand()] }), ).map((proposal) => proposal.action), ).toEqual(["boot", "provision"]); expect( - evaluate(view([tablet, busy], { ...options, waiting: [tabletHead, classDemand()] })), + proposalsOf(view([tablet, busy], { ...options, waiting: [tabletHead, classDemand()] })), ).toEqual([expect.objectContaining({ action: "provision" })]); expect( - evaluate(view([phone, tablet], { ...options, reset: ["phone"], waiting: [classDemand()] })), + proposalsOf( + view([phone, tablet], { ...options, reset: ["phone"], waiting: [classDemand()] }), + ), ).toEqual([boot("phone")]); expect( - evaluate( + proposalsOf( view([phone, busy], { ...options, reset: ["phone"], waiting: [tabletHead, classDemand()] }), ), ).toEqual([boot("phone")]); @@ -353,15 +361,17 @@ describe("warm pool policy", () => { }; }; - expect(evaluate(refusing([]))).toEqual([ + expect(proposalsOf(refusing([]))).toEqual([ { action: "boot", deviceId: "serving", reason: "waiting-request" }, ]); - expect(evaluate(refusing(["serving"]))).toEqual([]); + expect(proposalsOf(refusing(["serving"]))).toEqual([]); const learned = [learnedOf(recent.spec)]; - expect(evaluate(view([recent], { learned, limit: 3 }))).toEqual([ + expect(proposalsOf(view([recent], { learned, limit: 3 }))).toEqual([ { action: "boot", deviceId: "recent", reason: "target", target: recent.spec }, ]); - expect(evaluate(view([recent], { learned, limit: 3, refuseBoot: "ram-budget" }))).toEqual([]); + expect(proposalsOf(view([recent], { learned, limit: 3, refuseBoot: "ram-budget" }))).toEqual( + [], + ); }); it("does not boot back a device an operator reset shut down for a learned kind, though it was released a moment ago", () => { @@ -369,7 +379,7 @@ describe("warm pool policy", () => { const other = device("other", "shutdown", { endedAgo: 2_000 }); expect( - evaluate(view([reset, other], { learned: [learnedOf(reset.spec)], reset: ["reset"] })), + proposalsOf(view([reset, other], { learned: [learnedOf(reset.spec)], reset: ["reset"] })), ).toEqual([{ action: "boot", deviceId: "other", reason: "target", target: reset.spec }]); }); @@ -377,7 +387,7 @@ describe("warm pool policy", () => { const serving = device("serving", "ready", { endedAgo: 90 * minute }); const other = device("other", "ready", { endedAgo: 5 * minute, spec: spec("iPad Pro") }); - const proposals = evaluate( + const proposals = proposalsOf( view([serving, other], { limit: 1, waiting: [classDemand("full", "ios", true)] }), ); @@ -388,7 +398,7 @@ describe("warm pool policy", () => { const recent = device("recent", "shutdown", { endedAgo: 1_000 }); const busy = device("busy", "leased", { spec: spec("iPad Pro") }); - const proposals = evaluate( + const proposals = proposalsOf( view([recent, busy], { limit: 2, waiting: [classDemand("full", "ios", true)] }), ); @@ -407,15 +417,15 @@ describe("warm pool policy", () => { const newerIos = device("ios-new", "ready", { endedAgo: 2 * minute }); const devices = [oldestAndroid, newerAndroid, oldestIos, newerIos]; - const iosOver = evaluate({ + const iosOver = proposalsOf({ ...view(devices), capacity: capacityOf(devices, { android: 5, global: 5, ios: 1 }), }); - const androidOver = evaluate({ + const androidOver = proposalsOf({ ...view(devices), capacity: capacityOf(devices, { android: 1, global: 5, ios: 5 }), }); - const globalOver = evaluate({ + const globalOver = proposalsOf({ ...view(devices), capacity: capacityOf(devices, { android: 5, global: 3, ios: 5 }), }); @@ -433,7 +443,7 @@ describe("warm pool policy", () => { const leasedReady = device("leased-ready", "ready", { endedAgo: 90 * minute }); const free = device("free", "ready", { endedAgo: 1_000 }); - const proposals = evaluate(view([leasedReady, free], { limit: 1, leased: [leasedReady] })); + const proposals = proposalsOf(view([leasedReady, free], { limit: 1, leased: [leasedReady] })); expect(proposals).toEqual([{ action: "shutdown", deviceId: "free", reason: "over-budget" }]); }); @@ -441,7 +451,7 @@ describe("warm pool policy", () => { it("boots one device for two waiting requests it serves, once", () => { const serving = device("serving", "shutdown", { endedAgo: 1_000 }); - const proposals = evaluate(view([serving], { waiting: [classDemand(), classDemand()] })); + const proposals = proposalsOf(view([serving], { waiting: [classDemand(), classDemand()] })); expect(proposals).toEqual([{ action: "boot", deviceId: "serving", reason: "waiting-request" }]); }); @@ -451,7 +461,7 @@ describe("warm pool policy", () => { const busy = device("busy", "leased", { spec: spec("iPad Pro") }); const devices = [serving, busy]; - const proposals = evaluate({ + const proposals = proposalsOf({ ...view(devices, { waiting: [classDemand()] }), capacity: capacityOf(devices, { global: 5, ios: 1 }), }); @@ -465,11 +475,11 @@ describe("warm pool policy", () => { // The request wants a phone, which no device serves, and the only iOS slot is its own. const learned = [learnedOf(tablet.spec)]; - const held = evaluate({ + const held = proposalsOf({ ...view(devices, { learned, waiting: [classDemand()] }), capacity: capacityOf(devices, { global: 5, ios: 1 }), }); - const free = evaluate({ + const free = proposalsOf({ ...view(devices, { learned }), capacity: capacityOf(devices, { global: 5, ios: 1 }), }); @@ -484,7 +494,7 @@ describe("warm pool policy", () => { const serving = device("serving", "shutdown", { endedAgo: 1_000 }); const other = device("other", "shutdown", { endedAgo: 2_000, spec: spec("iPad Pro") }); - const proposals = evaluate( + const proposals = proposalsOf( view([serving, other], { learned: [learnedOf(serving.spec), learnedOf(other.spec)], maxConcurrentBoots: 3, @@ -504,11 +514,11 @@ describe("warm pool policy", () => { const devices = [first, second]; const learned = [learnedOf(first.spec, 2)]; - const globalTight = evaluate({ + const globalTight = proposalsOf({ ...view(devices, { learned, maxConcurrentBoots: 2 }), capacity: capacityOf(devices, { global: 1, ios: 5 }), }); - const platformTight = evaluate({ + const platformTight = proposalsOf({ ...view(devices, { learned, maxConcurrentBoots: 2 }), capacity: capacityOf(devices, { global: 5, ios: 1 }), }); @@ -532,7 +542,7 @@ describe("warm pool policy", () => { { android: 1, global: 5, ios: 5 }, ]) { expect( - evaluate({ ...view(devices), capacity: capacityOf(devices, limits) }), + proposalsOf({ ...view(devices), capacity: capacityOf(devices, limits) }), JSON.stringify(limits), ).toEqual([]); } @@ -548,9 +558,9 @@ describe("warm pool policy", () => { const free = device("free", "shutdown", { endedAgo: 2_000 }); const learned = [learnedOf(free.spec, 4)]; - expect(evaluate(view([busy, free], { learned, limit: 1 }))).toEqual([]); + expect(proposalsOf(view([busy, free], { learned, limit: 1 }))).toEqual([]); expect( - evaluate( + proposalsOf( view([busy, claimed, spent, free], { claimed: ["claimed"], learned, @@ -575,7 +585,7 @@ describe("warm pool policy", () => { const iosShut = device("ios-shut", "shutdown", { endedAgo: 2_000 }); const devices = [androidBusy, androidShut, iosShut]; - const proposals = evaluate({ + const proposals = proposalsOf({ ...view(devices, { learned: [learnedOf(androidShut.spec, 2), learnedOf(iosShut.spec)] }), capacity: capacityOf(devices, { android: 1, global: 5, ios: 5 }), }); @@ -595,10 +605,10 @@ describe("warm pool policy", () => { const three = idleIos(3); const two = idleIos(2); - expect(evaluate(view(three, { limits: { ios: 3 }, reserve: { ios: 1 } }))).toEqual([ + expect(proposalsOf(view(three, { limits: { ios: 3 }, reserve: { ios: 1 } }))).toEqual([ { action: "shutdown", deviceId: "ios-2", reason: "over-budget" }, ]); - expect(evaluate(view(two, { limits: { ios: 3 }, reserve: { ios: 1 } }))).toEqual([]); + expect(proposalsOf(view(two, { limits: { ios: 3 }, reserve: { ios: 1 } }))).toEqual([]); }); it("takes the sum of the platform reserves off the global budget", () => { @@ -610,8 +620,8 @@ describe("warm pool policy", () => { const reserve = { android: 1, ios: 1 }; // Global limit 4 minus the two reserves leaves room for two idle devices, not three. - expect(evaluate(view(idle(3), { limit: 4, reserve }))).toHaveLength(1); - expect(evaluate(view(idle(2), { limit: 4, reserve }))).toEqual([]); + expect(proposalsOf(view(idle(3), { limit: 4, reserve }))).toHaveLength(1); + expect(proposalsOf(view(idle(2), { limit: 4, reserve }))).toEqual([]); }); it("proposes every idle device of a platform for shutdown when the reserve is above its limit, and none of the other", () => { @@ -621,7 +631,7 @@ describe("warm pool policy", () => { device("android-0", "ready", { endedAgo: minute, spec: androidSpec }), ]; - const proposals = evaluate(view(devices, { limits: { ios: 2 }, reserve: { ios: 3 } })); + const proposals = proposalsOf(view(devices, { limits: { ios: 2 }, reserve: { ios: 3 } })); expect( proposals.map((proposal) => ("deviceId" in proposal ? proposal.deviceId : "")).sort(), @@ -637,7 +647,7 @@ describe("warm pool policy", () => { // iOS limit 1 with reserve 4 holds one slot of the global 3, leaving room for two Androids. expect( - evaluate(view(devices, { limit: 3, limits: { ios: 1 }, reserve: { ios: 4 } })), + proposalsOf(view(devices, { limit: 3, limits: { ios: 1 }, reserve: { ios: 4 } })), ).toEqual([]); }); @@ -649,8 +659,8 @@ describe("warm pool policy", () => { // Global limit 3 with one running leaves two, and the two reserves hold both. const learned = [learnedOf(released.spec)]; - expect(evaluate(view([released, running], { learned, limit: 3, reserve }))).toEqual([]); - expect(evaluate(view([released, running], { learned, limit: 3 }))).toEqual([ + expect(proposalsOf(view([released, running], { learned, limit: 3, reserve }))).toEqual([]); + expect(proposalsOf(view([released, running], { learned, limit: 3 }))).toEqual([ { action: "boot", deviceId: "released", reason: "target", target: released.spec }, ]); }); @@ -661,11 +671,11 @@ describe("warm pool policy", () => { const learned = [learnedOf(released.spec, 2)]; expect( - evaluate( + proposalsOf( view([released, ...running], { learned, limits: { ios: 2 }, reserve: { ios: 1 } }), ), ).toEqual([]); - expect(evaluate(view([released, ...running], { learned, limits: { ios: 2 } }))).toEqual([ + expect(proposalsOf(view([released, ...running], { learned, limits: { ios: 2 } }))).toEqual([ { action: "boot", deviceId: "released", reason: "target", target: released.spec }, ]); }); @@ -674,7 +684,7 @@ describe("warm pool policy", () => { const released = device("released", "shutdown", { endedAgo: minute }); expect( - evaluate( + proposalsOf( view([released], { limits: { ios: 1 }, reserve: { ios: 1 }, diff --git a/src/core/warm-pool/test-policy.ts b/src/core/warm-pool/test-policy.ts index 27c9ce1ca..831f1a61f 100644 --- a/src/core/warm-pool/test-policy.ts +++ b/src/core/warm-pool/test-policy.ts @@ -9,7 +9,7 @@ import type { import { sameSpec } from "../domain.js"; import { evaluate as plan, type ResolvedTarget, type WarmPolicyView } from "./policy.js"; -export const evaluate = (policyView: WarmPolicyView) => plan(policyView).proposals; +export const proposalsOf = (policyView: WarmPolicyView) => plan(policyView).proposals; export const minute = 60_000; export const now = 1_000 * minute; diff --git a/src/daemon/optional-modules-start.test.ts b/src/daemon/optional-modules-start.test.ts index 721bd0406..8b7ca278a 100644 --- a/src/daemon/optional-modules-start.test.ts +++ b/src/daemon/optional-modules-start.test.ts @@ -25,9 +25,9 @@ import { leaseAndRelease, prepare, servers, - sleep, start, } from "./test-optional-modules.js"; +import { sleep } from "./test-relay-support.js"; afterEach(async () => { await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test").catch(() => undefined))); diff --git a/src/daemon/optional-modules-stop.test.ts b/src/daemon/optional-modules-stop.test.ts index 106208b85..e6af4b5de 100644 --- a/src/daemon/optional-modules-stop.test.ts +++ b/src/daemon/optional-modules-stop.test.ts @@ -12,9 +12,9 @@ import { directories, ios, servers, - sleep, start, } from "./test-optional-modules.js"; +import { sleep } from "./test-relay-support.js"; afterEach(async () => { await Promise.all(daemons.splice(0).map((daemon) => daemon.stop("test").catch(() => undefined))); diff --git a/src/daemon/test-optional-modules.ts b/src/daemon/test-optional-modules.ts index d0f02f35b..4b66da9f1 100644 --- a/src/daemon/test-optional-modules.ts +++ b/src/daemon/test-optional-modules.ts @@ -157,5 +157,3 @@ export async function leaseAndRelease(daemon: DaemonServer): Promise { await daemon.dispatch("lease.release", { leaseId: grant.lease.id }, agent); return grant.lease.id; } - -export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/src/drivers/android/index.ts b/src/drivers/android/index.ts index 79e042e58..c4c8c8bcc 100644 --- a/src/drivers/android/index.ts +++ b/src/drivers/android/index.ts @@ -752,6 +752,7 @@ export class AndroidDriver implements Driver { : SNAPSHOT_RECLAIM_ESTIMATE_MS; } } + // fallow-ignore-next-line unused-class-member -- called through the Driver interface, and by its tests through a test-* harness. async toolVersions(): Promise { return this.#systemImages.toolVersions(); } diff --git a/src/gateway/worker-registry.ts b/src/gateway/worker-registry.ts index a9951b38c..3b1471d2d 100644 --- a/src/gateway/worker-registry.ts +++ b/src/gateway/worker-registry.ts @@ -145,6 +145,7 @@ export class WorkerRegistry { * Likewise a view with no `devices` carries the physical devices of the last list read from it, none when that list had none. * Never shown to an operator: the view itself says "not read yet". */ + // fallow-ignore-next-line unused-class-member -- called through the gateway's views port, and by its tests through the test-* harness. routingViews(): readonly WorkerView[] { return this.views().map((view) => { const catalog = this.#lastCatalog.get(view.id); @@ -188,6 +189,7 @@ export class WorkerRegistry { * or any other listener: this registry's own correctness must never depend on what a * subscriber does with the notification. */ + // fallow-ignore-next-line unused-class-member -- called through the gateway's views port, and by its tests through the test-* harness. onViewsChanged(listener: () => void): () => void { this.#viewsChangedListeners.add(listener); return () => { diff --git a/src/history/history-prune.test.ts b/src/history/history-prune.test.ts index 9c6591345..ea524dece 100644 --- a/src/history/history-prune.test.ts +++ b/src/history/history-prune.test.ts @@ -6,7 +6,6 @@ import { type EventEnvelope } from "../bus/index.js"; import { plainProjection } from "./testing.js"; import { NOW, - HOUR, directories, rig, stopping, @@ -15,6 +14,7 @@ import { settle, sizeOf, } from "./test-history.js"; +import { HOUR } from "./testing.js"; afterEach(async () => { await Promise.all( diff --git a/src/history/history-start-recovery.test.ts b/src/history/history-start-recovery.test.ts index f6879b64c..54e2378f1 100644 --- a/src/history/history-start-recovery.test.ts +++ b/src/history/history-start-recovery.test.ts @@ -7,7 +7,6 @@ import { SqliteUnavailableError, type SqliteOpener } from "../ports/index.js"; import { ProbeSqliteOpener } from "./testing.js"; import { NOW, - HOUR, JUNK, directories, type Rig, @@ -20,6 +19,7 @@ import { readHistoryOf, countRows, } from "./test-history.js"; +import { HOUR } from "./testing.js"; afterEach(async () => { await Promise.all( diff --git a/src/history/test-history.ts b/src/history/test-history.ts index df1c408e1..8e4830f63 100644 --- a/src/history/test-history.ts +++ b/src/history/test-history.ts @@ -1,3 +1,4 @@ +import { HOUR } from "./testing.js"; import { mkdtemp, readdir, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -20,7 +21,6 @@ import { import { plainProjection } from "./testing.js"; export const NOW = 1_760_605_920_000; -export const HOUR = 3_600_000; export const JUNK = "this is not a sqlite database ".repeat(20); export const directories: string[] = []; diff --git a/src/http/notices.ts b/src/http/notices.ts index 982b68395..9c99663a6 100644 --- a/src/http/notices.ts +++ b/src/http/notices.ts @@ -54,6 +54,7 @@ export class LeaseNoticeBuffer { * Live feed for a lease's SSE stream: flushes whatever is already buffered first, so a * client that connects after a notice fired doesn't miss it, then pushes future notices. */ + // fallow-ignore-next-line unused-class-member -- called through the app's notice port, and by its tests through a test-* harness. subscribe(leaseId: string, listener: (notice: LeaseNotice) => void): () => void { for (const notice of this.drain(leaseId)) listener(notice); const listeners = this.#listeners.get(leaseId) ?? new Set(); diff --git a/src/http/tracker.ts b/src/http/tracker.ts index eee2ad281..3ab383558 100644 --- a/src/http/tracker.ts +++ b/src/http/tracker.ts @@ -364,6 +364,7 @@ export class LeaseRequestTracker { return { kind: "not-cancellable" }; } + // fallow-ignore-next-line unused-class-member -- called through the app's tracker, and by its tests through a test-* harness. requestIdForLease(leaseId: string): string | undefined { return this.options.requests.requestIdForLease(leaseId); } From dc10f6e931e2f64387615d4d0bc06d2110e8a9d6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:09:13 +0100 Subject: [PATCH 80/83] docs(manuals): name the moved parts by paths that resolve, within the manuals' size budget --- src/auth/AGENTS.md | 2 +- src/contract/AGENTS.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auth/AGENTS.md b/src/auth/AGENTS.md index 153ac45d3..f67e24e4a 100644 --- a/src/auth/AGENTS.md +++ b/src/auth/AGENTS.md @@ -39,4 +39,4 @@ Socket: 1. A directory under `strategies/`, and one line in `strategies/index.ts`: its place in the list is its priority. 2. A fixture in `strategies/conformance.test.ts`. -3. If it owns a config key under `http`: the mirror in the contract's `configSchema`. `src/core/config-validators.ts` validates and `config-defaults.ts` defaults the key from the definition. +3. If it owns a config key under `http`: the mirror in the contract's `configSchema`. `src/core/config.ts` validates and defaults the key from the definition. diff --git a/src/contract/AGENTS.md b/src/contract/AGENTS.md index d5ee04b02..2aaad4436 100644 --- a/src/contract/AGENTS.md +++ b/src/contract/AGENTS.md @@ -43,11 +43,11 @@ Errors and views: ## Adding an operation -1. The `operations-*.ts` file of its family (lease request, leases, admin, fleet): `defineOperation` (`operation-definition.ts`), with `authorize` when it acts on one lease. Add it to `OPERATIONS` in `operations.ts`. +1. Its family's `operations-*.ts`: `defineOperation`, with `authorize` when it acts on one lease. Add it to `OPERATIONS` (`operations.ts`). 2. `operations.test.ts`: a row in the role matrix and the effect matrix. 3. `protocol.ts`: `PROTOCOL_VERSION_RANGE` + 1. -4. Handlers in `src/daemon/dispatcher.ts` (the table; the handler goes in its family's `dispatcher-*.ts` part) and `src/gateway/dispatcher.ts`. A gateway-only operation answers `UNSUPPORTED_IN_WORKER_MODE` on a worker; the reverse, `UNSUPPORTED_IN_GATEWAY_MODE`. -5. Client: a method on `SimlockClient` or `SimlockAdminClient` (declared in `src/simlock-client/client-api.ts`, implemented in `client-impl.ts`), types in `src/simlock-client/types.ts`, re-exported from `src/client/index.ts` or `src/admin/index.ts`. +4. Handlers in a `src/daemon/dispatcher-*.ts` part and `src/gateway/dispatcher.ts`. A gateway-only operation answers `UNSUPPORTED_IN_WORKER_MODE` on a worker; the reverse, `UNSUPPORTED_IN_GATEWAY_MODE`. +5. Client: a method on `SimlockClient` or `SimlockAdminClient` (`src/simlock-client/client-api.ts`), types in `src/simlock-client/types.ts`, re-exported from `src/client/index.ts` or `src/admin/index.ts`. 6. Frontends as needed: CLI, HTTP, MCP only for an agent session's need. 7. A new error code: `ErrorDetailsMap` and `ERROR_TABLE` (`errors.ts`). 8. Docs: the website pages that describe it, at least [errors.mdx](../../website/src/content/docs/docs/reference/errors.mdx) (documentation rule 9). From 73fc87c1fae0c05ec1272c152d35837d6a6bcabd Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 08:35:47 +0100 Subject: [PATCH 81/83] fix(docs-surface): the config surface covers the parts config.ts was split into The gate watched only src/core/config.ts, so after the split a change to the config type, defaults or validators passed with no website page changed. --- .agents/scripts/docs-surface-check.mjs | 11 +++++++++-- src/docs-surface-check.test.ts | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.agents/scripts/docs-surface-check.mjs b/.agents/scripts/docs-surface-check.mjs index 560f78e8f..e3189d38e 100644 --- a/.agents/scripts/docs-surface-check.mjs +++ b/.agents/scripts/docs-surface-check.mjs @@ -5,7 +5,8 @@ // one without the other prints the surfaces it changed and the exit code is 1. The surface is: // // cli src/cli/tree.ts, the command tree -// config src/core/config.ts, the config type, defaults and validator +// config src/core/config.ts and the parts it re-exports (config-types.ts, -defaults.ts, +// -validators.ts, -role-checks.ts, -helpers.ts): the config type, defaults and validator // errors the ERROR_TABLE object in src/contract/errors.ts // mcp each `registerTool(` name and definition object in src/mcp/server.ts, and the // tool schemas in src/mcp/contracts.ts @@ -50,7 +51,13 @@ const tools = (text) => const SURFACES = [ { name: "cli", files: ["src/cli/tree.ts"], parts: whole }, - { name: "config", files: ["src/core/config.ts"], parts: whole }, + { + name: "config", + files: ["", "-types", "-defaults", "-validators", "-role-checks", "-helpers"].map( + (part) => `src/core/config${part}.ts`, + ), + parts: whole, + }, { name: "errors", files: ["src/contract/errors.ts"], diff --git a/src/docs-surface-check.test.ts b/src/docs-surface-check.test.ts index 8d62cd7ba..7ce1a8a71 100644 --- a/src/docs-surface-check.test.ts +++ b/src/docs-surface-check.test.ts @@ -72,6 +72,7 @@ describe("docs-surface-check.mjs", () => { it.each([ ["cli", edit("src/cli/tree.ts", '"lease" }', '"lease", nuke: "nuke" }')], ["config", edit("src/core/config.ts", "ttl: 1", "ttl: 2")], + ["config", { "src/core/config-defaults.ts": "export const DEFAULT_TTL = 2;\n" }], ["errors", edit("src/contract/errors.ts", "exit: 2", "exit: 3")], ["mcp", edit("src/mcp/server.ts", '"List devices"', '"List the devices"')], ["mcp", edit("src/mcp/contracts.ts", "z.object({})", "z.object({ id: z.string() })")], From abc46bfded7d46b3cc5ee5a11108c98489588287 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 08:35:52 +0100 Subject: [PATCH 82/83] docs: pointers name the files the split moved code to, and edited comments say why Comments and flow docs still named pre-split locations. An edited comment must follow architecture rule 17, so those blocks now say why without pointing elsewhere, or are gone. --- docs/internal/flows/device-exec.md | 2 +- docs/internal/flows/device-selection.md | 4 ++-- src/contract/operations-leases.ts | 2 +- src/contract/protocol.ts | 2 +- src/contract/roles.ts | 2 +- src/core/config.test.ts | 16 ++------------ src/daemon/dispatcher-exec.test.ts | 2 +- src/daemon/error-code.ts | 29 +++++-------------------- src/daemon/gateway-fleet.e2e.test.ts | 12 +++------- src/daemon/worker-composition.ts | 2 +- src/gateway/dispatcher.ts | 4 ++-- src/gateway/fleet-dispatch.ts | 2 +- src/gateway/test-worker-registry.ts | 4 ++-- src/http/errors.ts | 6 ++--- src/lease-policy/index.ts | 7 ++---- src/mcp/session.ts | 2 +- src/ports/node-process-runner.ts | 2 +- src/ports/process-runner.test.ts | 12 ++++------ src/simlock-client/types.ts | 12 ++++------ src/simlock-client/wire.ts | 24 ++++++-------------- 20 files changed, 45 insertions(+), 103 deletions(-) diff --git a/docs/internal/flows/device-exec.md b/docs/internal/flows/device-exec.md index 416067de9..94c8f69c8 100644 --- a/docs/internal/flows/device-exec.md +++ b/docs/internal/flows/device-exec.md @@ -1,7 +1,7 @@ # device.exec Modules: [daemon](../../../src/daemon/AGENTS.md), [gateway](../../../src/gateway/AGENTS.md). -Contract: `src/contract/operations.ts` `deviceExec` — role `agent`, effect `write`. +Contract: `src/contract/operations-leases.ts` `deviceExec` — role `agent`, effect `write`. Input (strict): `{ leaseId, tool, args, stdin?, stdinFile?, requesterId? }`. Output: `{ exitCode }`. Runs the command on the machine that owns the device and streams its output back. diff --git a/docs/internal/flows/device-selection.md b/docs/internal/flows/device-selection.md index 1d3cb1c16..5a3135c3b 100644 --- a/docs/internal/flows/device-selection.md +++ b/docs/internal/flows/device-selection.md @@ -11,7 +11,7 @@ How one lease request becomes one concrete device, virtual or physical. | Source | Where | Read when | |---|---|---| | Request: `model` or `class`, `osVersion` (exact or range), `mode`, `imageTag`, `physical`, `allowDownload`, `noWait` | `contract/operations.ts` → `DeviceRequest` | Per request | -| Download permission | `downloads.policy` (`on-request`), `daemon/dispatcher.ts` | Per request, before leasing | +| Download permission | `downloads.policy` (`on-request`), `daemon/dispatcher-leases.ts` | Per request, before leasing | | Default mode | `ios.defaultMode` (`full`); Android always `full` | Per request with no `mode` | | Operator preference lists | `ios.defaultModels`, `android.defaultModels` | Merged once at start | | Built-in preference lists | `Driver.defaultModels` of each driver | Merged once at start, after the operator's | @@ -31,7 +31,7 @@ Never read: the sdkmanager catalog. A preference list is merged in `daemon/worke ## Virtual: request → spec -In `LeaseAcquisitionCoordinator` (`leasing/lease-acquisition-coordinator.ts`), no lock held: +In `AcquisitionSpecResolver` (`leasing/acquisition-spec-resolver.ts`), a part of `LeaseAcquisitionCoordinator`, no lock held: 1. `physical: true` → skip to the physical branch. 2. Mode = request's, else the platform default. diff --git a/src/contract/operations-leases.ts b/src/contract/operations-leases.ts index 8390b4b2f..e95aeceb2 100644 --- a/src/contract/operations-leases.ts +++ b/src/contract/operations-leases.ts @@ -29,7 +29,7 @@ export const leaseCancel = defineOperation({ * *lease's* recorded `ownerId` from the registry; a pending, not-yet-granted request has no * lease yet to look up -- `pendingRequestOwner` is the wait-queue equivalent). An omitted * `requesterId` defaults to the principal the same way the handler's own default does (see - * `dispatcher.ts`'s `#leaseCancel`), so it always resolves to a request this principal owns. + * `dispatcher-leases.ts`'s `leaseCancel`), so it always resolves to a request this principal owns. * A `requesterId` with no pending request resolves `pendingRequestOwner` to `undefined`, * which is treated as authorized (same convention as `ownsLease`) so the handler's own * `not-found` surfaces instead of a misleading `FORBIDDEN`. diff --git a/src/contract/protocol.ts b/src/contract/protocol.ts index 8085c4edd..6dbbc7898 100644 --- a/src/contract/protocol.ts +++ b/src/contract/protocol.ts @@ -68,7 +68,7 @@ export const helloRequestSchema = z * `hello` request supplied, or the daemon's own default (today: `defaultRequesterId`) when it * omitted one. Without this the client cannot learn its own principal in that fallback case, and * would otherwise have to guess -- see the abort-authorization defect this closes in - * `simlock-client/client.ts`. + * `simlock-client/client-impl.ts`. */ export const helloReplySchema = z.object({ protocolVersion: z.number().int(), diff --git a/src/contract/roles.ts b/src/contract/roles.ts index ec51ac465..4a12707ab 100644 --- a/src/contract/roles.ts +++ b/src/contract/roles.ts @@ -49,7 +49,7 @@ export interface AuthorizeContext { * Builds an `authorize` hook that requires the calling session to either be `admin` (ADR §4: * "admin bypasses") or own the resource identified by `getId(input)`. This is the one * `authorize` hook this contract declares -- `lease.renew` and `lease.release` both use it (see - * `operations.ts`); `lease.list`'s ownership is a filter over many records, not a single-id + * `operations-leases.ts`); `lease.list`'s ownership is a filter over many records, not a single-id * check, so it is not a candidate for this helper and is handled entirely in its handler. */ export function ownsLease( diff --git a/src/core/config.test.ts b/src/core/config.test.ts index a6f71ed0d..aa5672a7f 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -417,20 +417,8 @@ describe("loadConfig", () => { }); describe("configSchema agrees with the config the daemon actually loads", () => { - /** - * `config.get` declares `configSchema` as its output (`contract/operations.ts`), and zod's - * object mode strips unknown keys -- so a key this schema does not list is not a type error - * anywhere, it simply vanishes on the way to the caller. - * - * Round 6 review found `gateway.leaseRequestTimeoutMs` doing exactly that: declared on - * `Config`, validated by `loadConfig`, documented in CONFIGURATION.md as inspectable via - * `simlock config get`, and silently dropped. Nothing referenced `configSchema` from any test, - * so the drift was invisible in both directions. - * - * This compares key sets per block rather than asserting one key, so the next block that gains - * a field fails here instead of shipping a `config get` that quietly omits it (testing rule 4: - * a test that enforces a rule has to see everything the rule covers). - */ + /** why: zod's object mode strips a key `configSchema` does not list, so a config key missing + * from it vanishes from `config get` with no type error anywhere. */ it("strips no key from any block of a freshly loaded config", async () => { const config = await loadConfig({ configPath, diff --git a/src/daemon/dispatcher-exec.test.ts b/src/daemon/dispatcher-exec.test.ts index df68e4d45..6bcb0ac7b 100644 --- a/src/daemon/dispatcher-exec.test.ts +++ b/src/daemon/dispatcher-exec.test.ts @@ -644,7 +644,7 @@ describe("Dispatcher: device.exec", () => { // already exited and closed its pipe while a chunk's delivery to a slow consumer is still // outstanding" -- exactly the state a real OS pipe can be in (verified against real Node: // `close` can fire in the same tick as the `data` event for a chunk this process runner - // just paused on). `#deviceExec` (`server.ts`) and the HTTP exec route (`app.ts`) both just + // just paused on). `#execDevice` (`server-requests.ts`) and the HTTP exec route (`app-lease-routes.ts`) both just // hand their transport's own backpressured `onOutput` straight to this same dispatcher // call, so a real reproduction here against a real child process is the one place this // defect -- and its fix, in `NodeStreamingProcessHandle` -- is actually exercised, for diff --git a/src/daemon/error-code.ts b/src/daemon/error-code.ts index 8323613d7..b3933787a 100644 --- a/src/daemon/error-code.ts +++ b/src/daemon/error-code.ts @@ -1,23 +1,8 @@ /** - * The single place that turns a thrown core/daemon error into a `SimlockErrorCode`. ADR 0003 - * §7: "CLI exit codes and HTTP status codes are columns of the same error table, not second - * mappings." Both transports -- `DaemonServer`'s socket frame handling (`errorCode` in - * `server.ts`) and the HTTP gateway (`mapError` in `../http/errors.ts`) -- call `classifyError` - * to get the *code*, then read whichever column they need (`cliExitCode`/`httpStatus`) off - * `ERROR_TABLE` themselves. Before this module existed, each transport hand-wrote its own - * `instanceof` chain for these errors and the two silently drifted (see the ADR 0003-09 review, - * finding B5): `InsufficientDiskSpaceError`, `LicenseNotAcceptedError`, `StartupFailedError`, - * `DoctorUnavailableError`, and `NukeUnavailableError` all mapped correctly over the socket but - * fell through to `INTERNAL` over HTTP. Adding a new core error now means adding one branch - * here, not one in each transport. - * - * `DispatchError`'s own `code` is used verbatim (trusted by construction -- see `dispatcher.ts`, - * every call site passes a real `SimlockErrorCode`) rather than re-classified, since it is - * already the shared dispatcher's own protocol-shaped rejection. - * - * Deliberately excludes anything that is socket-framing-specific (`ProtocolError`, thrown only - * while parsing a request frame before a `Session` exists -- it never reaches the shared - * dispatcher, so it never reaches HTTP either) -- that stays local to `server.ts`. + * why: both transports classify a thrown error here and read their own column off + * `ERROR_TABLE`, so a core error maps the same over the socket and over HTTP; per-transport + * `instanceof` chains drifted apart before. `ProtocolError` is left out: it is thrown only while + * parsing a frame before a `Session` exists, so it never reaches HTTP. */ import { BaselineMismatchError, @@ -84,10 +69,8 @@ export class StartupFailedError extends Error { */ // fallow-ignore-next-line complexity -- one exhaustive classifier is the point (ADR §7). export function classifyError(error: unknown): SimlockErrorCode | undefined { - // `DispatchError` is `Dispatcher`'s own protocol-shaped rejection (bad input, role/authorize - // failure, unknown operation, a rejected credential) -- its `code` is already a - // `SimlockErrorCode` by construction (see `dispatcher.ts`), so it is used verbatim rather than - // re-derived. + // why: a `DispatchError` carries a `SimlockErrorCode` by construction, so its code is used + // verbatim rather than re-derived. if (error instanceof DispatchError) { return error.code as SimlockErrorCode; } diff --git a/src/daemon/gateway-fleet.e2e.test.ts b/src/daemon/gateway-fleet.e2e.test.ts index 5be91d631..975b7ece5 100644 --- a/src/daemon/gateway-fleet.e2e.test.ts +++ b/src/daemon/gateway-fleet.e2e.test.ts @@ -574,15 +574,9 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { const workerBId = beforeDrain.find((worker) => worker.label === "worker-b")?.id; if (workerBId === undefined) throw new Error("worker-b never connected"); - // Lease worker-a's own device -- the fleet client below must still be able to renew this - // exact lease after everything that follows. - // P2 (round 1 review, #119): `requesterId` set explicitly, distinct from the session - // principal -- `agentSession({ principal: "fleet-owner" })` alone makes `ownerId` and - // `requesterId` the same string (`dispatcher.ts` defaults both to `session.principal`), so - // the §27a assertion below (owner authorizes, requester does not) could not tell the two - // fields apart: mutating `lease-index.ts` to re-derive `ownerId` from `requesterId` on - // rebuild would still pass. With a distinct `requesterId`, only a genuinely separate - // `ownerId` on the rebuilt entry can authorize the renew below. + // why: a `requesterId` distinct from the session principal, because with both defaulted to + // `session.principal` the renew below would still pass if the rebuilt index re-derived + // `ownerId` from `requesterId`. const grant = await firstGateway.dispatch( "lease.request", { diff --git a/src/daemon/worker-composition.ts b/src/daemon/worker-composition.ts index b30360edf..5c6a80569 100644 --- a/src/daemon/worker-composition.ts +++ b/src/daemon/worker-composition.ts @@ -425,7 +425,7 @@ export async function composeWorker( // could return having stopped nothing (because `stopHttpGateway` isn't assigned yet, the // gateway still being mid-`start()`), and `#stop()` would go on to settle and dispose while // the gateway finishes binding and starts accepting requests against a daemon already being - // torn down -- see `server.ts`'s `stopAuxiliary` doc: it must be shut off before lease/queue + // torn down -- see `server-options.ts`'s `stopAuxiliary` doc: it must be shut off before lease/queue // teardown begins. When HTTP is disabled there is nothing to wait for, so this resolves // immediately. let resolveGatewayStarted: (() => void) | undefined; diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 4b2417447..b3a1b0826 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -681,8 +681,8 @@ export class GatewayDispatcher { * and its `lease.maxTtlMs` caps what its own clients may ask for, both before anything is * dispatched". `owner` (§27a, narrowed round 3 review H3) is read only from `session. * isGatewayUplink` -- never merely `role === "admin"`, which HTTP's `operator` token also maps - * onto (`dispatcher-session.ts`), the same over-wide gate `daemon/dispatcher.ts`'s own - * `#leaseRequest` used to share with this one. + * onto (`dispatcher-session.ts`), the same over-wide gate `daemon/dispatcher-leases.ts`'s + * `leaseRequest` used to share with this one. * * Unlike the worker, nothing ever forwards *into* a gateway's own front door the way this * gateway forwards into a worker -- there is no "gateway of gateways" in ADR 0005, so no diff --git a/src/gateway/fleet-dispatch.ts b/src/gateway/fleet-dispatch.ts index 5e3645921..6afff9466 100644 --- a/src/gateway/fleet-dispatch.ts +++ b/src/gateway/fleet-dispatch.ts @@ -370,7 +370,7 @@ export class FleetDispatch { // connection (`WorkerLink#connect`, via `acceptUplink` on the worker's own end, see // `session.isGatewayUplink`'s doc). The worker stores this verbatim as the lease's // `ownerId` instead of deriving it from the connection -- see - // `daemon/dispatcher.ts`'s `#leaseRequest`. + // `daemon/dispatcher-leases.ts`'s `leaseRequest`. owner: waiter.options.ownerId, // why: ADR 0009 §3: a download is never triggered through a gateway. allowDownload: false, diff --git a/src/gateway/test-worker-registry.ts b/src/gateway/test-worker-registry.ts index 3b14df9b2..97d1a171c 100644 --- a/src/gateway/test-worker-registry.ts +++ b/src/gateway/test-worker-registry.ts @@ -5,8 +5,8 @@ import { WorkerRegistry } from "./worker-registry.js"; export const RETENTION_MS = 24 * 60 * 60_000; -/** Matches `core/config.ts`'s own default, so a test that does not care about ADR 0005 §15's - * warning gets a cap no fixture's `ttlMs` ever comes near. */ +/** why: the config default's cap, so no fixture's `ttlMs` comes near it and a test that does not + * care about the max-TTL warning never sees one. */ const DEFAULT_LEASE_MAX_TTL_MS = 4 * 60 * 60_000; /** Matches `service.test.ts`'s `principal: "gw:instance-1"` -- ADR 0005 §14/§27's own-lease diff --git a/src/http/errors.ts b/src/http/errors.ts index 831f958d5..5b36ffc1d 100644 --- a/src/http/errors.ts +++ b/src/http/errors.ts @@ -105,10 +105,8 @@ export interface MappedError { readonly extra?: Record; } -/** Looks a code up in `ERROR_TABLE`, defensively -- every code `classifyError` returns is a - * real key by construction, except `DispatchError`'s (a runtime-supplied string, trusted but - * not verified -- see `dispatcher.ts`); this is the one guard against that ever indexing to - * `undefined` and throwing instead of degrading to `INTERNAL`. */ +/** why: `DispatchError`'s code is a runtime string, trusted but not verified, so a code missing + * from the table degrades to `INTERNAL` instead of indexing to `undefined` and throwing. */ function tableEntry(code: SimlockErrorCode): (typeof ERROR_TABLE)[SimlockErrorCode] { return ( (ERROR_TABLE as Record)[code] ?? diff --git a/src/lease-policy/index.ts b/src/lease-policy/index.ts index 0d60578c8..8bb8d50c3 100644 --- a/src/lease-policy/index.ts +++ b/src/lease-policy/index.ts @@ -178,11 +178,8 @@ type Attempt = | { readonly attemptId: number; readonly ok: false; readonly error: unknown }; /** - * A rejection that answers "this lease is not yours to renew" rather than "that attempt did - * not work". Both codes are the daemon's final word: `UNKNOWN_LEASE` means the record is gone - * (expired, released, force-released), `FORBIDDEN` that it belongs to another principal - * (`ownsLease` in `src/contract/operations.ts`). Retrying either just prints the same answer - * until the deadline. + * why: both codes are the daemon's final word -- `UNKNOWN_LEASE` the record is gone, `FORBIDDEN` + * it belongs to another principal -- so retrying either only repeats the answer until the deadline. */ function isLeaseGone(error: unknown): boolean { return isSimlockError(error) && (error.code === "UNKNOWN_LEASE" || error.code === "FORBIDDEN"); diff --git a/src/mcp/session.ts b/src/mcp/session.ts index d466ee083..51867f9ca 100644 --- a/src/mcp/session.ts +++ b/src/mcp/session.ts @@ -242,7 +242,7 @@ export class McpSession { /** * why: This session's current lease, or an explicit "no lease held" result -- one `lease.list` * call (ADR 0003 §9), never a cache of lease state. `lease.list` filters by **owner principal - * only** (see `src/daemon/dispatcher.ts`'s `lease.list` handler), so it can return leases + * only** (see `src/daemon/dispatcher-leases.ts`'s `lease.list` handler), so it can return leases * this connection never requested: one a `simlock lease --detach` left behind under the same * principal (`SIMLOCK_AGENT_ID` or the agent session's id), or one left over from an earlier session under that * principal. Taking `leases[0]` unconditionally would report a lease this session neither diff --git a/src/ports/node-process-runner.ts b/src/ports/node-process-runner.ts index a7e0a4300..058317aba 100644 --- a/src/ports/node-process-runner.ts +++ b/src/ports/node-process-runner.ts @@ -292,7 +292,7 @@ export class ExecOutputDeliveryStalledError extends Error { * grandchild with **no** pending delivery hitting the same deadline still settles as before: * nothing of the command's own output is waiting on anyone in that case, only unrelated noise * this handle was never going to forward to a live consumer anyway (see `#execDevice` in - * `daemon/server.ts`, torn down by then). + * `daemon/server-requests.ts`, torn down by then). */ class NodeStreamingProcessHandle implements StreamingProcessHandle { readonly pid: number; diff --git a/src/ports/process-runner.test.ts b/src/ports/process-runner.test.ts index 35d96aeb5..49f9b615a 100644 --- a/src/ports/process-runner.test.ts +++ b/src/ports/process-runner.test.ts @@ -422,14 +422,10 @@ describe("NodeProcessRunner: spawnStreaming", () => { }, 6_000); it("does not crash the process when onChunk throws synchronously -- treated as a failed delivery instead", async () => { - // `onChunk` is typed `void | Promise`, but nothing stops a transport's own - // implementation from throwing synchronously. Called bare inside a `data` listener (round - // 4 hardening), that throw becomes an uncaughtException that takes the whole daemon down - // over one bad chunk -- latent today because both real callers (`daemon/server.ts`, - // `http/app.ts`) happen to be `async`, but nothing enforces that. A `process.on - // ("uncaughtException", ...)` listener is the only way to observe the pre-fix crash from - // inside a test without actually taking this process down, so this registers one, asserts - // it never fires, and removes it again in `finally` regardless of outcome. + // why: nothing stops a transport's `onChunk` from throwing synchronously, and inside a `data` + // listener that is an uncaughtException that takes the daemon down; a `process.on + // ("uncaughtException", ...)` listener is the only way to see it without taking this process + // down, so it is registered here and removed in `finally` whatever the outcome. const uncaught: unknown[] = []; const onUncaughtException = (error: unknown): void => { uncaught.push(error); diff --git a/src/simlock-client/types.ts b/src/simlock-client/types.ts index 1dd38c822..05b9ac28f 100644 --- a/src/simlock-client/types.ts +++ b/src/simlock-client/types.ts @@ -1,10 +1,6 @@ /** - * Public types for `simlock/client` and `simlock/admin` (ADR 0003 §10). Every type here is - * `z.infer`red from a contract schema (`src/contract/operations.ts`, `schemas.ts`, `pushes.ts`) - * rather than hand-declared, so a schema change is a compile-time type change here too, never a - * silent drift. Nothing in this file imports from `src/core`, `src/daemon`, or `src/drivers` -- - * see `no-core-leak.test.ts`, which asserts the compiled `.d.ts` for these two entry points - * never mentions a core-private type name. + * why: every type here is `z.infer`red from a contract schema rather than hand-declared, so a + * schema change is a compile-time change here, never a silent drift. */ import type { z } from "zod"; @@ -179,8 +175,8 @@ export interface InstallComponentOnWorkersOptions { } export interface RequestLeaseOptions { - /** ADR §10's one stateful client behaviour -- see `client.ts`'s `#requestLeaseWithAbort` for - * the four distinct behaviours this drives. */ + /** why: aborting sends `lease.cancel` rather than just dropping the wait, so a queued request is + * withdrawn on the daemon instead of being granted later; an abort after the grant is ignored. */ readonly signal?: AbortSignal; readonly onProgress?: (progress: LeaseProgress) => void; } diff --git a/src/simlock-client/wire.ts b/src/simlock-client/wire.ts index c866586a9..fef8b59d8 100644 --- a/src/simlock-client/wire.ts +++ b/src/simlock-client/wire.ts @@ -1,9 +1,3 @@ -/** - * The typed client's wire layer (ADR 0003 §10). One connection, one frame-id space, no - * reconnect and no retry -- see the module comment on `client.ts` for the policy this - * implements. This file owns only framing, push routing, and connection-death handling; every - * typed method surface lives in `client.ts`. - */ import type { IpcConnection } from "../ports/index.js"; import { fromWireError, @@ -88,10 +82,8 @@ export type LeaseScopedPush = }; /** - * Everything below `client.ts`'s typed method surface: frame ids, pending-call bookkeeping, - * push routing and de-duplication, and connection-death fan-out. Deliberately has no idea what - * a "lease" or a "grant" is beyond the push correlation keys the contract declares -- that - * vocabulary lives in `client.ts`. + * why: knows nothing of a "lease" or a "grant" beyond the push correlation keys the contract + * declares, so the typed client above it owns that vocabulary. */ export class SimlockWire { readonly #connection: IpcConnection; @@ -107,7 +99,7 @@ export class SimlockWire { * "never fires onLeaseLost for a release the same client asked for") is the daemon's job, * not this class's: `DaemonServer` marks the lease id in its own per-connection * `selfInitiatedReleases` set right before dispatching `lease.release`/`lease.release-all` - * and clears it in a `finally` regardless of outcome (`daemon/server.ts`; + * and clears it in a `finally` regardless of outcome (`daemon/server-requests.ts`; * `daemon/server-pushes.ts`'s `notifyLeaseLost` reads it), so the push is never sent to the * releasing connection in the first place. Mirroring that bookkeeping here too was redundant * on a successful release and actively harmful on a failed one: marked before the call and @@ -212,9 +204,8 @@ export class SimlockWire { return fromWireError(error.code, error.message, error.details); } - /** One request/response round trip for an already role/shape-agnostic operation name. - * `client.ts` is responsible for input/output schema validation; this only moves bytes and - * correlates frame ids. */ + /** why: role- and shape-agnostic: the typed client validates input and output, so this only + * moves bytes and correlates frame ids. */ call(type: string, payload: unknown, hooks: CallHooks = {}): Promise { return this.#send(type, payload, hooks); } @@ -441,9 +432,8 @@ export class SimlockWire { } } -/** Internal wire-level rejection: `{code, message, details}` straight off the wire, not yet - * mapped through the contract's error table. `client.ts`/`hello()` do that mapping -- kept - * separate from `SimlockError` so a mid-mapping bug cannot masquerade as a real contract error. */ +/** why: kept apart from `SimlockError` until it is mapped through the contract's error table, so a + * mid-mapping bug cannot masquerade as a real contract error. */ export class WireCallError extends Error { constructor( readonly code: string, From f5305a5ab62586a7ffbf0d5fdf2186dd652f8f97 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 09:24:56 +0100 Subject: [PATCH 83/83] fix(docs-surface): the config surface covers config-alerts.ts too --- .agents/scripts/docs-surface-check.mjs | 5 +++-- src/docs-surface-check.test.ts | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.agents/scripts/docs-surface-check.mjs b/.agents/scripts/docs-surface-check.mjs index e3189d38e..f5bb7a618 100644 --- a/.agents/scripts/docs-surface-check.mjs +++ b/.agents/scripts/docs-surface-check.mjs @@ -6,7 +6,8 @@ // // cli src/cli/tree.ts, the command tree // config src/core/config.ts and the parts it re-exports (config-types.ts, -defaults.ts, -// -validators.ts, -role-checks.ts, -helpers.ts): the config type, defaults and validator +// -validators.ts, -role-checks.ts, -helpers.ts, -alerts.ts): the config type, defaults +// and validator // errors the ERROR_TABLE object in src/contract/errors.ts // mcp each `registerTool(` name and definition object in src/mcp/server.ts, and the // tool schemas in src/mcp/contracts.ts @@ -53,7 +54,7 @@ const SURFACES = [ { name: "cli", files: ["src/cli/tree.ts"], parts: whole }, { name: "config", - files: ["", "-types", "-defaults", "-validators", "-role-checks", "-helpers"].map( + files: ["", "-types", "-defaults", "-validators", "-role-checks", "-helpers", "-alerts"].map( (part) => `src/core/config${part}.ts`, ), parts: whole, diff --git a/src/docs-surface-check.test.ts b/src/docs-surface-check.test.ts index 7ce1a8a71..b8c988201 100644 --- a/src/docs-surface-check.test.ts +++ b/src/docs-surface-check.test.ts @@ -73,6 +73,7 @@ describe("docs-surface-check.mjs", () => { ["cli", edit("src/cli/tree.ts", '"lease" }', '"lease", nuke: "nuke" }')], ["config", edit("src/core/config.ts", "ttl: 1", "ttl: 2")], ["config", { "src/core/config-defaults.ts": "export const DEFAULT_TTL = 2;\n" }], + ["config", { "src/core/config-alerts.ts": "export const ALERT_EVENTS = [];\n" }], ["errors", edit("src/contract/errors.ts", "exit: 2", "exit: 3")], ["mcp", edit("src/mcp/server.ts", '"List devices"', '"List the devices"')], ["mcp", edit("src/mcp/contracts.ts", "z.object({})", "z.object({ id: z.string() })")],